diff --git a/.gitignore b/.gitignore index 1e6e178e746fadfbfb5e0f02a692d87b8b57a302..704b2e741598a3b32fb91987b25a51edaedf8fff 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,6 @@ debug/ release/ target/ debs/ -deps/ rpms/ mac/ *.pyc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b873e47b7405bd73b8cb9ef5f90e58c72409977b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,28 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: check-json + - id: end-of-file-fixer + - id: trailing-whitespace + +repos: + - repo: https://github.com/psf/black + rev: stable + hooks: + - id: black + +repos: + - repo: https://github.com/pocc/pre-commit-hooks + rev: master + hooks: + - id: cppcheck + args: ["--error-exitcode=0"] + +repos: + - repo: https://github.com/crate-ci/typos + rev: v1.15.7 + hooks: + - id: typos + diff --git a/CMakeLists.txt b/CMakeLists.txt index fb2b306f65467361dd5612f5bf1e6629a7a0c306..5048287cf6967a7e609fb559bfd6fdf352482ec4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,11 +15,15 @@ SET(TD_COMMUNITY_DIR ${PROJECT_SOURCE_DIR}) set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake") set(TD_CONTRIB_DIR "${TD_SOURCE_DIR}/contrib") + + + include(${TD_SUPPORT_DIR}/cmake.platform) include(${TD_SUPPORT_DIR}/cmake.define) include(${TD_SUPPORT_DIR}/cmake.options) include(${TD_SUPPORT_DIR}/cmake.version) + # contrib add_subdirectory(contrib) diff --git a/CONTRIBUTING-CN.md b/CONTRIBUTING-CN.md index efaa2077fe05a7c9f4b08098fcb74da8ad3a97cd..887cff0978ee4263b10ea704fd01da3b3f02f457 100644 --- a/CONTRIBUTING-CN.md +++ b/CONTRIBUTING-CN.md @@ -18,7 +18,7 @@ 注意：修改文档的分支要以`docs/`为开头，以免进行不必要的测试。 4. 创建pull request，将自己的分支合并到开发分支`3.0`，我们开发团队将尽快审核。 -如遇任何问题，请添加官方微信TDengineECO。我们的团队会帮忙解决。 +如遇任何问题，请添加官方微信 tdengine1。我们的团队会帮忙解决。 ## 给贡献者的礼品 @@ -48,4 +48,4 @@ TDengine 社区致力于让更多的开发者理解和使用它。 ## 联系我们 -如果您有什么问题需要解决，或者有什么问题需要解答，可以添加微信：TDengineECO +如果您有什么问题需要解决，或者有什么问题需要解答，可以添加微信：tdengine1。 diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 55bd5466ed4abe40cfaedc634b75a59b43668759..f4dcdb242ee51ca711a123fb1b56d7e1840eabfa 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -314,7 +314,7 @@ def pre_test_build_win() { cd %WIN_CONNECTOR_ROOT% python.exe -m pip install --upgrade pip python -m pip uninstall taospy -y - python -m pip install taospy==2.7.6 + python -m pip install taospy==2.7.10 xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 ''' return 1 diff --git a/README-CN.md b/README-CN.md index a6dfefc47ac98db6d222021923849be6ac2a8549..2b1790f4bbd5cb9862c62c450cb6cb484526cbb0 100644 --- a/README-CN.md +++ b/README-CN.md @@ -15,7 +15,7 @@ [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) -简体中文 | [English](README.md) | 很多职位正在热招中，请看[这里](https://www.taosdata.com/cn/careers/) +简体中文 | [English](README.md) | [TDengine 云服务](https://cloud.taosdata.com/?utm_medium=cn&utm_source=github) | 很多职位正在热招中，请看[这里](https://www.taosdata.com/cn/careers/) # TDengine 简介 @@ -39,7 +39,7 @@ TDengine 是一款开源、高性能、云原生的时序数据库 (Time-Series # 构建 -TDengine 目前可以在 Linux、 Windows、macOS 等平台上安装和运行。任何 OS 的应用也可以选择 taosAdapter 的 RESTful 接口连接服务端 taosd。CPU 支持 X64/ARM64，后续会支持 MIPS64、Alpha64、ARM32、RISC-V 等 CPU 架构。 +TDengine 目前可以在 Linux、 Windows、macOS 等平台上安装和运行。任何 OS 的应用也可以选择 taosAdapter 的 RESTful 接口连接服务端 taosd。CPU 支持 X64/ARM64，后续会支持 MIPS64、Alpha64、ARM32、RISC-V 等 CPU 架构。目前不支持使用交叉编译器构建。 用户可根据需求选择通过源码、[容器](https://docs.taosdata.com/get-started/docker/)、[安装包](https://docs.taosdata.com/get-started/package/)或[Kubernetes](https://docs.taosdata.com/deployment/k8s/)来安装。本快速指南仅适用于通过源码安装。 @@ -68,14 +68,14 @@ sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-d ```bash sudo yum install epel-release sudo yum update -sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel +sudo yum install -y gcc gcc-c++ make cmake3 gflags git openssl-devel sudo ln -sf /usr/bin/cmake3 /usr/bin/cmake ``` -### CentOS 8 & Fedora +### CentOS 8/Fedora/Rocky Linux ```bash -sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel +sudo dnf install -y gcc gcc-c++ gflags make cmake epel-release git openssl-devel ``` #### 在 CentOS 上构建 taosTools 安装依赖软件 @@ -88,7 +88,7 @@ sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel ``` -#### CentOS 8/Rocky Linux +#### CentOS 8/Fedora/Rocky Linux ``` sudo yum install -y epel-release @@ -101,7 +101,7 @@ sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson 若 powertools 安装失败，可以尝试改用： ``` -sudo yum config-manager --set-enabled Powertools +sudo yum config-manager --set-enabled powertools ``` #### CentOS + devtoolset @@ -117,7 +117,7 @@ scl enable devtoolset-9 -- bash ### macOS ``` -brew install argp-standalone pkgconfig +brew install argp-standalone gflags pkgconfig ``` ### 设置 golang 开发环境 @@ -175,7 +175,7 @@ cd TDengine ```bash mkdir debug cd debug -cmake .. -DBUILD_TOOLS=true +cmake .. -DBUILD_TOOLS=true -DBUILD_CONTRIB=true make ``` @@ -352,4 +352,4 @@ TDengine 提供了丰富的应用程序开发接口，其中包括 C/C++、Java # 加入技术交流群 -TDengine 官方社群「物联网大数据群」对外开放，欢迎您加入讨论。搜索微信号 "tdengine1"，加小 T 为好友，即可入群。 +TDengine 官方社群「物联网大数据群」对外开放，欢迎您加入讨论。搜索微信号 "tdengine"，加小 T 为好友，即可入群。 diff --git a/README.md b/README.md index a088404c85c414e46a7b9a056c3d89aaa65e87a8..a8c20ea3f606e2b68b76c5c203bb02b20b588105 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ For user manual, system design and architecture, please refer to [TDengine Docum # Building -At the moment, TDengine server supports running on Linux/Windows/macOS systems. Any application can also choose the RESTful interface provided by taosAdapter to connect the taosd service . TDengine supports X64/ARM64 CPU, and it will support MIPS64, Alpha64, ARM32, RISC-V and other CPU architectures in the future. +At the moment, TDengine server supports running on Linux/Windows/macOS systems. Any application can also choose the RESTful interface provided by taosAdapter to connect the taosd service . TDengine supports X64/ARM64 CPU, and it will support MIPS64, Alpha64, ARM32, RISC-V and other CPU architectures in the future. Right now we don't support build with cross-compiling environment. You can choose to install through source code, [container](https://docs.tdengine.com/get-started/docker/), [installation package](https://docs.tdengine.com/get-started/package/) or [Kubernetes](https://docs.tdengine.com/deployment/k8s/). This quick guide only applies to installing from source. @@ -76,14 +76,14 @@ sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-d ```bash sudo yum install epel-release sudo yum update -sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel +sudo yum install -y gcc gcc-c++ make cmake3 gflags git openssl-devel sudo ln -sf /usr/bin/cmake3 /usr/bin/cmake ``` -### CentOS 8 & Fedora +### CentOS 8/Fedora/Rocky Linux ```bash -sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel +sudo dnf install -y gcc gcc-c++ make cmake epel-release gflags git openssl-devel ``` #### Install build dependencies for taosTools on CentOS @@ -94,7 +94,7 @@ sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel ``` -#### CentOS 8/Rocky Linux +#### CentOS 8/Fedora/Rocky Linux ``` sudo yum install -y epel-release @@ -124,7 +124,7 @@ scl enable devtoolset-9 -- bash ### macOS ``` -brew install argp-standalone pkgconfig +brew install argp-standalone gflags pkgconfig ``` ### Setup golang environment @@ -183,7 +183,7 @@ It equals to execute following commands: ```bash mkdir debug cd debug -cmake .. -DBUILD_TOOLS=true +cmake .. -DBUILD_TOOLS=true -DBUILD_CONTRIB=true make ``` diff --git a/build.sh b/build.sh index 78f08afa7a772e2a9f05178244f2443e0ec45925..04ca7a11a0d8907b63848504beeb6ff96db73dd3 100755 --- a/build.sh +++ b/build.sh @@ -4,5 +4,5 @@ if [ ! -d debug ]; then mkdir debug || echo -e "failed to make directory for build" fi -cd debug && cmake .. -DBUILD_TOOLS=true && make +cd debug && cmake .. -DBUILD_TOOLS=true -DBUILD_CONTRIB=true && make diff --git a/cmake/cmake.define b/cmake/cmake.define index f55a9bdabc79e31f129b2184144c9472572d5454..cf7f45099414af86a6fe8bccfbbd8bb477c5b711 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0) -set(CMAKE_VERBOSE_MAKEFILE OFF) +set(CMAKE_VERBOSE_MAKEFILE ON) set(TD_BUILD_TAOSA_INTERNAL FALSE) #set output directory @@ -115,15 +115,6 @@ ELSE () SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}") ENDIF () - IF (${BUILD_SANITIZER}) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") - MESSAGE(STATUS "Compile with Address Sanitizer!") - ELSE () - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reserved-user-defined-literal -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") - ENDIF () - # disable all assert IF ((${DISABLE_ASSERT} MATCHES "true") OR (${DISABLE_ASSERTS} MATCHES "true")) ADD_DEFINITIONS(-DDISABLE_ASSERT) @@ -165,4 +156,20 @@ ELSE () MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED") ENDIF() + # build mode + SET(CMAKE_C_FLAGS_REL "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -O3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") + SET(CMAKE_CXX_FLAGS_REL "${CMAKE_CXX_FLAGS} -Werror -Wno-reserved-user-defined-literal -Wno-literal-suffix -Werror=return-type -fPIC -O3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") + + IF (${BUILD_SANITIZER}) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0") + MESSAGE(STATUS "Compile with Address Sanitizer!") + ELSEIF (${BUILD_RELEASE}) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_REL}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_REL}") + ELSE () + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -g3 -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reserved-user-defined-literal -g3 -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") + ENDIF () + ENDIF () diff --git a/cmake/cmake.options b/cmake/cmake.options index 4ec9d18e08580a735598b647ef65188b46fdbc61..fa0b88841519305ff30b758356e66cf041d8d900 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -64,12 +64,25 @@ IF(${TD_WINDOWS}) ON ) + MESSAGE("build geos Win32") + option( + BUILD_GEOS + "If build geos on Windows" + ON + ) + ELSEIF (TD_DARWIN_64) IF(${BUILD_TEST}) add_definitions(-DCOMPILER_SUPPORTS_CXX13) ENDIF () ENDIF () +option( + BUILD_GEOS + "If build geos on Windows" + ON + ) + option( BUILD_SHARED_LIBS "" @@ -171,3 +184,14 @@ option( ON ) +option( + BUILD_RELEASE + "If build release version" + OFF +) + +option( + BUILD_CONTRIB + "If build thirdpart from source" + OFF +) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index ba747c6134c7032239b6d1d7b87186be66257069..76ac6ba004674386672d45302649b134a61d4cfd 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -121,6 +121,12 @@ IF ("${CPUTYPE}" STREQUAL "") SET(TD_LOONGARCH_64 TRUE) ADD_DEFINITIONS("-D_TD_LOONGARCH_") ADD_DEFINITIONS("-D_TD_LOONGARCH_64") + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") + SET(PLATFORM_ARCH_STR "mips") + MESSAGE(STATUS "input cpuType: mips64") + SET(TD_MIPS_64 TRUE) + ADD_DEFINITIONS("-D_TD_MIPS_") + ADD_DEFINITIONS("-D_TD_MIPS_64") ENDIF () ELSE () # if generate ARM version: @@ -172,5 +178,17 @@ ENDIF() MESSAGE(STATUS "Platform arch:" ${PLATFORM_ARCH_STR}) +set(TD_DEPS_DIR "x86") +if (TD_LINUX) + IF (TD_ARM_64 OR TD_ARM_32) + set(TD_DEPS_DIR "arm") + ELSEIF (TD_MIPS_64) + set(TD_DEPS_DIR "mips") + ELSE() + set(TD_DEPS_DIR "x86") + ENDIF() +endif() +MESSAGE(STATUS "DEPS_DIR: " ${TD_DEPS_DIR}) + MESSAGE("C Compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_C_COMPILER_VERSION})") MESSAGE("CXX Compiler: ${CMAKE_CXX_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_CXX_COMPILER_VERSION})") diff --git a/cmake/cmake.version b/cmake/cmake.version index 3166a0695cea5132fdd509da130548079add2103..a6bf90fa3cb270245b206d81358cb76912e5b739 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.4.1") + SET(TD_VER_NUMBER "3.1.1.0.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/cmake/geos_CMakeLists.txt.in b/cmake/geos_CMakeLists.txt.in new file mode 100644 index 0000000000000000000000000000000000000000..f939ccead045fcab4c3aee14e86bb014ae39e8cb --- /dev/null +++ b/cmake/geos_CMakeLists.txt.in @@ -0,0 +1,12 @@ + +# geos +ExternalProject_Add(geos + GIT_REPOSITORY https://github.com/libgeos/geos.git + GIT_TAG 3.12.0 + SOURCE_DIR "${TD_CONTRIB_DIR}/geos" + BINARY_DIR "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + ) diff --git a/cmake/rocksdb_CMakeLists.txt.in b/cmake/rocksdb_CMakeLists.txt.in index 7d9f49d3fa20874b19c6f4e4e318c0a9da26d4e9..45599d82e3a8750f8b6e4823c0f8eb8dc8a79f24 100644 --- a/cmake/rocksdb_CMakeLists.txt.in +++ b/cmake/rocksdb_CMakeLists.txt.in @@ -1,11 +1,29 @@ # rocksdb -ExternalProject_Add(rocksdb - GIT_REPOSITORY https://github.com/taosdata-contrib/rocksdb.git - GIT_TAG v6.23.3 - SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" +if (${BUILD_CONTRIB}) + ExternalProject_Add(rocksdb + URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.1.1.tar.gz + URL_HASH MD5=3b4c97ee45df9c8a5517308d31ab008b + DOWNLOAD_NO_PROGRESS 1 + DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" + SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" ) +else() + if (NOT ${TD_LINUX}) + ExternalProject_Add(rocksdb + URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.1.1.tar.gz + URL_HASH MD5=3b4c97ee45df9c8a5517308d31ab008b + DOWNLOAD_NO_PROGRESS 1 + DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" + SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + ) + endif() +endif() diff --git a/cmake/stub_CMakeLists.txt.in b/cmake/stub_CMakeLists.txt.in index cf175aab422ce4710b48ae501fa65b59481b772e..96cc924d811230a7b7f9a76a5edbd5f758dccb43 100644 --- a/cmake/stub_CMakeLists.txt.in +++ b/cmake/stub_CMakeLists.txt.in @@ -2,6 +2,7 @@ # stub ExternalProject_Add(stub GIT_REPOSITORY https://github.com/coolxv/cpp-stub.git + GIT_TAG 5e903b8e GIT_SUBMODULES "src" SOURCE_DIR "${TD_CONTRIB_DIR}/cpp-stub" BINARY_DIR "${TD_CONTRIB_DIR}/cpp-stub/src" diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index c67918351de7b25d18403314886bd5227f8fa6c6..13826a1a7487c0c16f3f80c934d2d406a2ed4c7f 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 565ca21 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index d9d2f120690e714122756a36821a6fe6398151c2..9bbda8309fcbecea01a8daf24ebb1361cfaffa05 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 4378702 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 536d4eae8e49bf85bdd43551a9c93948770ff409..c60fd33b16187840d2cc27feb8321e14b801ed5d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -77,11 +77,23 @@ if(${BUILD_WITH_LEVELDB}) cat("${TD_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_WITH_LEVELDB}) -# rocksdb -if(${BUILD_WITH_ROCKSDB}) - cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - add_definitions(-DUSE_ROCKSDB) -endif(${BUILD_WITH_ROCKSDB}) +if (${BUILD_CONTRIB}) + if(${BUILD_WITH_ROCKSDB}) + cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + add_definitions(-DUSE_ROCKSDB) + endif() +else() + if (NOT ${TD_LINUX}) + if(${BUILD_WITH_ROCKSDB}) + cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + add_definitions(-DUSE_ROCKSDB) + endif(${BUILD_WITH_ROCKSDB}) + else() + if(${BUILD_WITH_ROCKSDB}) + add_definitions(-DUSE_ROCKSDB) + endif(${BUILD_WITH_ROCKSDB}) + endif() +endif() # canonical-raft if(${BUILD_WITH_CRAFT}) @@ -134,6 +146,11 @@ if(${BUILD_ADDR2LINE}) endif(NOT ${TD_WINDOWS}) endif(${BUILD_ADDR2LINE}) +# geos +if(${BUILD_GEOS}) + cat("${TD_SUPPORT_DIR}/geos_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + # download dependencies configure_file(${CONTRIB_TMP_FILE} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -170,8 +187,8 @@ if(${BUILD_TEST}) PUBLIC $ ) endif(${TD_DARWIN}) - - + + endif(${BUILD_TEST}) # cJson @@ -222,55 +239,113 @@ endif(${BUILD_WITH_LEVELDB}) # rocksdb # To support rocksdb build on ubuntu: sudo apt-get install libgflags-dev -if(${BUILD_WITH_ROCKSDB}) +if (${BUILD_WITH_UV}) if(${TD_LINUX}) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=unused-function -Wno-errno=unused-private-field -Wno-error=unused-result") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_REL}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_REL}") + IF ("${CMAKE_BUILD_TYPE}" STREQUAL "") + SET(CMAKE_BUILD_TYPE Release) + endif() endif(${TD_LINUX}) - - if(${TD_DARWIN}) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") - endif(${TD_DARWIN}) - - if (${TD_WINDOWS}) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") - endif(${TD_WINDOWS}) - - - if(${TD_DARWIN}) - option(HAVE_THREAD_LOCAL "" OFF) - option(WITH_IOSTATS_CONTEXT "" OFF) - option(WITH_PERF_CONTEXT "" OFF) - endif(${TD_DARWIN}) - - if(${TD_WINDOWS}) - option(WITH_JNI "" ON) - endif(${TD_WINDOWS}) - - if(${TD_WINDOWS}) - option(WITH_MD_LIBRARY "build with MD" OFF) - set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) - endif(${TD_WINDOWS}) - - - option(WITH_FALLOCATE "" OFF) - option(WITH_JEMALLOC "" OFF) - option(WITH_GFLAGS "" OFF) - option(PORTABLE "" ON) - option(WITH_LIBURING "" OFF) - option(FAIL_ON_WARNINGS OFF) - - option(WITH_TESTS "" OFF) - option(WITH_BENCHMARK_TOOLS "" OFF) - option(WITH_TOOLS "" OFF) - option(WITH_LIBURING "" OFF) - - option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) - add_subdirectory(rocksdb EXCLUDE_FROM_ALL) - target_include_directories( - rocksdb - PUBLIC $ - ) -endif(${BUILD_WITH_ROCKSDB}) +endif (${BUILD_WITH_UV}) + +if (${BUILD_WITH_ROCKSDB}) + if (${BUILD_CONTRIB}) + if(${TD_LINUX}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_REL} -Wno-error=maybe-uninitialized -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=unused-function -Wno-errno=unused-private-field -Wno-error=unused-result") + if ("${CMAKE_BUILD_TYPE}" STREQUAL "") + SET(CMAKE_BUILD_TYPE Release) + endif() + endif(${TD_LINUX}) + MESSAGE(STATUS "CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS}) + + if(${TD_DARWIN}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") + endif(${TD_DARWIN}) + + if (${TD_DARWIN_ARM64}) + set(HAS_ARMV8_CRC true) + endif(${TD_DARWIN_ARM64}) + + if (${TD_WINDOWS}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") + option(WITH_JNI "" OFF) + option(WITH_MD_LIBRARY "build with MD" OFF) + set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) + endif(${TD_WINDOWS}) + + + if(${TD_DARWIN}) + option(HAVE_THREAD_LOCAL "" OFF) + option(WITH_IOSTATS_CONTEXT "" OFF) + option(WITH_PERF_CONTEXT "" OFF) + endif(${TD_DARWIN}) + + option(WITH_FALLOCATE "" OFF) + option(WITH_JEMALLOC "" OFF) + option(WITH_GFLAGS "" OFF) + option(PORTABLE "" ON) + option(WITH_LIBURING "" OFF) + option(FAIL_ON_WARNINGS OFF) + + option(WITH_TESTS "" OFF) + option(WITH_BENCHMARK_TOOLS "" OFF) + option(WITH_TOOLS "" OFF) + option(WITH_LIBURING "" OFF) + + option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) + add_subdirectory(rocksdb EXCLUDE_FROM_ALL) + target_include_directories( + rocksdb + PUBLIC $ + ) + else() + if (NOT ${TD_LINUX}) + MESSAGE(STATUS "CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS}) + if(${TD_DARWIN}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") + endif(${TD_DARWIN}) + + if (${TD_DARWIN_ARM64}) + set(HAS_ARMV8_CRC true) + endif(${TD_DARWIN_ARM64}) + + if (${TD_WINDOWS}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") + option(WITH_JNI "" OFF) + option(WITH_MD_LIBRARY "build with MD" OFF) + set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) + endif(${TD_WINDOWS}) + + + if(${TD_DARWIN}) + option(HAVE_THREAD_LOCAL "" OFF) + option(WITH_IOSTATS_CONTEXT "" OFF) + option(WITH_PERF_CONTEXT "" OFF) + endif(${TD_DARWIN}) + + option(WITH_FALLOCATE "" OFF) + option(WITH_JEMALLOC "" OFF) + option(WITH_GFLAGS "" OFF) + option(PORTABLE "" ON) + option(WITH_LIBURING "" OFF) + option(FAIL_ON_WARNINGS OFF) + + option(WITH_TESTS "" OFF) + option(WITH_BENCHMARK_TOOLS "" OFF) + option(WITH_TOOLS "" OFF) + option(WITH_LIBURING "" OFF) + + option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) + add_subdirectory(rocksdb EXCLUDE_FROM_ALL) + target_include_directories( + rocksdb + PUBLIC $ + ) + endif() + + endif() +endif() # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev @@ -278,10 +353,10 @@ if(${BUILD_WITH_LUCENE}) option(ENABLE_TEST "Enable the tests" OFF) add_subdirectory(lucene EXCLUDE_FROM_ALL) target_include_directories( - lucene++ + lucene++ PUBLIC $ - ) - + ) + endif(${BUILD_WITH_LUCENE}) # NuRaft @@ -341,7 +416,7 @@ if(${BUILD_MSVCREGEX}) target_include_directories(msvcregex PRIVATE "msvcregex" ) - target_link_libraries(msvcregex + target_link_libraries(msvcregex INTERFACE Shell32 ) SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex) @@ -401,8 +476,8 @@ if(${BUILD_WITH_BDB}) IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/bdb/libdb.a" INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/bdb" ) - target_link_libraries(bdb - INTERFACE pthread + target_link_libraries(bdb + INTERFACE pthread ) endif(${BUILD_WITH_BDB}) @@ -414,12 +489,12 @@ if(${BUILD_WITH_SQLITE}) IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/sqlite/.libs/libsqlite3.a" INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/sqlite" ) - target_link_libraries(sqlite - INTERFACE m - INTERFACE pthread + target_link_libraries(sqlite + INTERFACE m + INTERFACE pthread ) if(NOT TD_WINDOWS) - target_link_libraries(sqlite + target_link_libraries(sqlite INTERFACE dl ) endif(NOT TD_WINDOWS) @@ -427,22 +502,22 @@ endif(${BUILD_WITH_SQLITE}) # addr2line if(${BUILD_ADDR2LINE}) - if(NOT ${TD_WINDOWS}) - check_include_file( "sys/types.h" HAVE_SYS_TYPES_H) - check_include_file( "sys/stat.h" HAVE_SYS_STAT_H ) - check_include_file( "inttypes.h" HAVE_INTTYPES_H ) - check_include_file( "stddef.h" HAVE_STDDEF_H ) - check_include_file( "stdlib.h" HAVE_STDLIB_H ) - check_include_file( "string.h" HAVE_STRING_H ) - check_include_file( "memory.h" HAVE_MEMORY_H ) - check_include_file( "strings.h" HAVE_STRINGS_H ) + if(NOT ${TD_WINDOWS}) + check_include_file( "sys/types.h" HAVE_SYS_TYPES_H) + check_include_file( "sys/stat.h" HAVE_SYS_STAT_H ) + check_include_file( "inttypes.h" HAVE_INTTYPES_H ) + check_include_file( "stddef.h" HAVE_STDDEF_H ) + check_include_file( "stdlib.h" HAVE_STDLIB_H ) + check_include_file( "string.h" HAVE_STRING_H ) + check_include_file( "memory.h" HAVE_MEMORY_H ) + check_include_file( "strings.h" HAVE_STRINGS_H ) check_include_file( "stdint.h" HAVE_STDINT_H ) check_include_file( "unistd.h" HAVE_UNISTD_H ) check_include_file( "sgidefs.h" HAVE_SGIDEFS_H ) check_include_file( "stdafx.h" HAVE_STDAFX_H ) - check_include_file( "elf.h" HAVE_ELF_H ) - check_include_file( "libelf.h" HAVE_LIBELF_H ) - check_include_file( "libelf/libelf.h" HAVE_LIBELF_LIBELF_H) + check_include_file( "elf.h" HAVE_ELF_H ) + check_include_file( "libelf.h" HAVE_LIBELF_H ) + check_include_file( "libelf/libelf.h" HAVE_LIBELF_LIBELF_H) check_include_file( "alloca.h" HAVE_ALLOCA_H ) check_include_file( "elfaccess.h" HAVE_ELFACCESS_H) check_include_file( "sys/elf_386.h" HAVE_SYS_ELF_386_H ) @@ -450,7 +525,7 @@ if(${BUILD_ADDR2LINE}) check_include_file( "sys/elf_sparc.h" HAVE_SYS_ELF_SPARC_H) check_include_file( "sys/ia64/elf.h" HAVE_SYS_IA64_ELF_H ) set(VERSION 0.3.1) - set(PACKAGE_VERSION "\"${VERSION}\"") + set(PACKAGE_VERSION "\"${VERSION}\"") configure_file(libdwarf/cmake/config.h.cmake config.h) file(GLOB_RECURSE LIBDWARF_SOURCES "libdwarf/src/lib/libdwarf/*.c") add_library(libdwarf STATIC ${LIBDWARF_SOURCES}) @@ -470,6 +545,23 @@ if(${BUILD_ADDR2LINE}) endif(NOT ${TD_WINDOWS}) endif(${BUILD_ADDR2LINE}) +# geos +if(${BUILD_GEOS}) + if(${TD_LINUX}) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_REL}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_REL}") + IF ("${CMAKE_BUILD_TYPE}" STREQUAL "") + SET(CMAKE_BUILD_TYPE Release) + endif() + endif(${TD_LINUX}) + option(BUILD_SHARED_LIBS "Build GEOS with shared libraries" OFF) + add_subdirectory(geos EXCLUDE_FROM_ALL) + unset(CMAKE_CXX_STANDARD CACHE) # undo libgeos's setting of global CMAKE_CXX_STANDARD + target_include_directories( + geos_c + PUBLIC $ + ) +endif(${BUILD_GEOS}) # ================================================================================================ # Build test diff --git a/deps/arm/rocksdb_static/librocksdb.a b/deps/arm/rocksdb_static/librocksdb.a new file mode 100644 index 0000000000000000000000000000000000000000..dc7f37b0f308431c96327d7a3a61f612673e3437 Binary files /dev/null and b/deps/arm/rocksdb_static/librocksdb.a differ diff --git a/deps/arm/rocksdb_static/rocksdb/c.h b/deps/arm/rocksdb_static/rocksdb/c.h new file mode 100644 index 0000000000000000000000000000000000000000..1ba7fabefe92f674ecbd27a91bfbb57b0ed17d23 --- /dev/null +++ b/deps/arm/rocksdb_static/rocksdb/c.h @@ -0,0 +1,2844 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +/* Copyright (c) 2011 The LevelDB Authors. All rights reserved. + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. See the AUTHORS file for names of contributors. + + C bindings for rocksdb. May be useful as a stable ABI that can be + used by programs that keep rocksdb in a shared library, or for + a JNI api. + + Does not support: + . getters for the option types + . custom comparators that implement key shortening + . capturing post-write-snapshot + . custom iter, db, env, cache implementations using just the C bindings + + Some conventions: + + (1) We expose just opaque struct pointers and functions to clients. + This allows us to change internal representations without having to + recompile clients. + + (2) For simplicity, there is no equivalent to the Slice type. Instead, + the caller has to pass the pointer and length as separate + arguments. + + (3) Errors are represented by a null-terminated c string. NULL + means no error. All operations that can raise an error are passed + a "char** errptr" as the last argument. One of the following must + be true on entry: + *errptr == NULL + *errptr points to a malloc()ed null-terminated error message + On success, a leveldb routine leaves *errptr unchanged. + On failure, leveldb frees the old value of *errptr and + set *errptr to a malloc()ed error message. + + (4) Bools have the type unsigned char (0 == false; rest == true) + + (5) All of the pointer arguments must be non-NULL. +*/ + +#pragma once + +#ifdef _WIN32 +#ifdef ROCKSDB_DLL +#ifdef ROCKSDB_LIBRARY_EXPORTS +#define ROCKSDB_LIBRARY_API __declspec(dllexport) +#else +#define ROCKSDB_LIBRARY_API __declspec(dllimport) +#endif +#else +#define ROCKSDB_LIBRARY_API +#endif +#else +#define ROCKSDB_LIBRARY_API +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include + +/* Exported types */ + +typedef struct rocksdb_t rocksdb_t; +typedef struct rocksdb_backup_engine_t rocksdb_backup_engine_t; +typedef struct rocksdb_backup_engine_info_t rocksdb_backup_engine_info_t; +typedef struct rocksdb_backup_engine_options_t rocksdb_backup_engine_options_t; +typedef struct rocksdb_restore_options_t rocksdb_restore_options_t; +typedef struct rocksdb_memory_allocator_t rocksdb_memory_allocator_t; +typedef struct rocksdb_lru_cache_options_t rocksdb_lru_cache_options_t; +typedef struct rocksdb_hyper_clock_cache_options_t + rocksdb_hyper_clock_cache_options_t; +typedef struct rocksdb_cache_t rocksdb_cache_t; +typedef struct rocksdb_compactionfilter_t rocksdb_compactionfilter_t; +typedef struct rocksdb_compactionfiltercontext_t + rocksdb_compactionfiltercontext_t; +typedef struct rocksdb_compactionfilterfactory_t + rocksdb_compactionfilterfactory_t; +typedef struct rocksdb_comparator_t rocksdb_comparator_t; +typedef struct rocksdb_dbpath_t rocksdb_dbpath_t; +typedef struct rocksdb_env_t rocksdb_env_t; +typedef struct rocksdb_fifo_compaction_options_t + rocksdb_fifo_compaction_options_t; +typedef struct rocksdb_filelock_t rocksdb_filelock_t; +typedef struct rocksdb_filterpolicy_t rocksdb_filterpolicy_t; +typedef struct rocksdb_flushoptions_t rocksdb_flushoptions_t; +typedef struct rocksdb_iterator_t rocksdb_iterator_t; +typedef struct rocksdb_logger_t rocksdb_logger_t; +typedef struct rocksdb_mergeoperator_t rocksdb_mergeoperator_t; +typedef struct rocksdb_options_t rocksdb_options_t; +typedef struct rocksdb_compactoptions_t rocksdb_compactoptions_t; +typedef struct rocksdb_block_based_table_options_t + rocksdb_block_based_table_options_t; +typedef struct rocksdb_cuckoo_table_options_t rocksdb_cuckoo_table_options_t; +typedef struct rocksdb_randomfile_t rocksdb_randomfile_t; +typedef struct rocksdb_readoptions_t rocksdb_readoptions_t; +typedef struct rocksdb_seqfile_t rocksdb_seqfile_t; +typedef struct rocksdb_slicetransform_t rocksdb_slicetransform_t; +typedef struct rocksdb_snapshot_t rocksdb_snapshot_t; +typedef struct rocksdb_writablefile_t rocksdb_writablefile_t; +typedef struct rocksdb_writebatch_t rocksdb_writebatch_t; +typedef struct rocksdb_writebatch_wi_t rocksdb_writebatch_wi_t; +typedef struct rocksdb_writeoptions_t rocksdb_writeoptions_t; +typedef struct rocksdb_universal_compaction_options_t + rocksdb_universal_compaction_options_t; +typedef struct rocksdb_livefiles_t rocksdb_livefiles_t; +typedef struct rocksdb_column_family_handle_t rocksdb_column_family_handle_t; +typedef struct rocksdb_column_family_metadata_t + rocksdb_column_family_metadata_t; +typedef struct rocksdb_level_metadata_t rocksdb_level_metadata_t; +typedef struct rocksdb_sst_file_metadata_t rocksdb_sst_file_metadata_t; +typedef struct rocksdb_envoptions_t rocksdb_envoptions_t; +typedef struct rocksdb_ingestexternalfileoptions_t + rocksdb_ingestexternalfileoptions_t; +typedef struct rocksdb_sstfilewriter_t rocksdb_sstfilewriter_t; +typedef struct rocksdb_ratelimiter_t rocksdb_ratelimiter_t; +typedef struct rocksdb_perfcontext_t rocksdb_perfcontext_t; +typedef struct rocksdb_pinnableslice_t rocksdb_pinnableslice_t; +typedef struct rocksdb_transactiondb_options_t rocksdb_transactiondb_options_t; +typedef struct rocksdb_transactiondb_t rocksdb_transactiondb_t; +typedef struct rocksdb_transaction_options_t rocksdb_transaction_options_t; +typedef struct rocksdb_optimistictransactiondb_t + rocksdb_optimistictransactiondb_t; +typedef struct rocksdb_optimistictransaction_options_t + rocksdb_optimistictransaction_options_t; +typedef struct rocksdb_transaction_t rocksdb_transaction_t; +typedef struct rocksdb_checkpoint_t rocksdb_checkpoint_t; +typedef struct rocksdb_wal_iterator_t rocksdb_wal_iterator_t; +typedef struct rocksdb_wal_readoptions_t rocksdb_wal_readoptions_t; +typedef struct rocksdb_memory_consumers_t rocksdb_memory_consumers_t; +typedef struct rocksdb_memory_usage_t rocksdb_memory_usage_t; + +/* DB operations */ + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open( + const rocksdb_options_t* options, const char* name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_with_ttl( + const rocksdb_options_t* options, const char* name, int ttl, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_for_read_only( + const rocksdb_options_t* options, const char* name, + unsigned char error_if_wal_file_exists, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_as_secondary( + const rocksdb_options_t* options, const char* name, + const char* secondary_path, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_backup_engine_t* rocksdb_backup_engine_open( + const rocksdb_options_t* options, const char* path, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_backup_engine_t* +rocksdb_backup_engine_open_opts(const rocksdb_backup_engine_options_t* options, + rocksdb_env_t* env, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_create_new_backup( + rocksdb_backup_engine_t* be, rocksdb_t* db, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_create_new_backup_flush( + rocksdb_backup_engine_t* be, rocksdb_t* db, + unsigned char flush_before_backup, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_purge_old_backups( + rocksdb_backup_engine_t* be, uint32_t num_backups_to_keep, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_restore_options_t* +rocksdb_restore_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_restore_options_destroy( + rocksdb_restore_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_restore_options_set_keep_log_files( + rocksdb_restore_options_t* opt, int v); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_verify_backup( + rocksdb_backup_engine_t* be, uint32_t backup_id, char** errptr); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_restore_db_from_latest_backup( + rocksdb_backup_engine_t* be, const char* db_dir, const char* wal_dir, + const rocksdb_restore_options_t* restore_options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_restore_db_from_backup( + rocksdb_backup_engine_t* be, const char* db_dir, const char* wal_dir, + const rocksdb_restore_options_t* restore_options, const uint32_t backup_id, + char** errptr); + +extern ROCKSDB_LIBRARY_API const rocksdb_backup_engine_info_t* +rocksdb_backup_engine_get_backup_info(rocksdb_backup_engine_t* be); + +extern ROCKSDB_LIBRARY_API int rocksdb_backup_engine_info_count( + const rocksdb_backup_engine_info_t* info); + +extern ROCKSDB_LIBRARY_API int64_t rocksdb_backup_engine_info_timestamp( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API uint32_t rocksdb_backup_engine_info_backup_id( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API uint64_t rocksdb_backup_engine_info_size( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API uint32_t rocksdb_backup_engine_info_number_files( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_info_destroy( + const rocksdb_backup_engine_info_t* info); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_close( + rocksdb_backup_engine_t* be); + +extern ROCKSDB_LIBRARY_API void rocksdb_put_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* ts, size_t tslen, const char* val, size_t vallen, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_put_cf_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* ts, size_t tslen, const char* val, size_t vallen, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_cf_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete_cf_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_increase_full_history_ts_low( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* ts_low, size_t ts_lowlen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_full_history_ts_low( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + size_t* ts_lowlen, char** errptr); + +/* BackupEngineOptions */ + +extern ROCKSDB_LIBRARY_API rocksdb_backup_engine_options_t* +rocksdb_backup_engine_options_create(const char* backup_dir); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_set_backup_dir( + rocksdb_backup_engine_options_t* options, const char* backup_dir); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_set_env( + rocksdb_backup_engine_options_t* options, rocksdb_env_t* env); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_share_table_files( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_backup_engine_options_get_share_table_files( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_set_sync( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_backup_engine_options_get_sync( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_destroy_old_data( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_backup_engine_options_get_destroy_old_data( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_backup_log_files( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_backup_engine_options_get_backup_log_files( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_backup_rate_limit( + rocksdb_backup_engine_options_t* options, uint64_t limit); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_backup_engine_options_get_backup_rate_limit( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_restore_rate_limit( + rocksdb_backup_engine_options_t* options, uint64_t limit); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_backup_engine_options_get_restore_rate_limit( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_max_background_operations( + rocksdb_backup_engine_options_t* options, int val); + +extern ROCKSDB_LIBRARY_API int +rocksdb_backup_engine_options_get_max_background_operations( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_callback_trigger_interval_size( + rocksdb_backup_engine_options_t* options, uint64_t size); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_backup_engine_options_get_callback_trigger_interval_size( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_max_valid_backups_to_open( + rocksdb_backup_engine_options_t* options, int val); + +extern ROCKSDB_LIBRARY_API int +rocksdb_backup_engine_options_get_max_valid_backups_to_open( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_share_files_with_checksum_naming( + rocksdb_backup_engine_options_t* options, int val); + +extern ROCKSDB_LIBRARY_API int +rocksdb_backup_engine_options_get_share_files_with_checksum_naming( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_destroy( + rocksdb_backup_engine_options_t*); + +/* Checkpoint */ + +extern ROCKSDB_LIBRARY_API rocksdb_checkpoint_t* +rocksdb_checkpoint_object_create(rocksdb_t* db, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_checkpoint_create( + rocksdb_checkpoint_t* checkpoint, const char* checkpoint_dir, + uint64_t log_size_for_flush, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_checkpoint_object_destroy( + rocksdb_checkpoint_t* checkpoint); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_and_trim_history( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char* trim_ts, + size_t trim_tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_column_families( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_column_families_with_ttl( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, const int* ttls, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* +rocksdb_open_for_read_only_column_families( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, + unsigned char error_if_wal_file_exists, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_as_secondary_column_families( + const rocksdb_options_t* options, const char* name, + const char* secondary_path, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API char** rocksdb_list_column_families( + const rocksdb_options_t* options, const char* name, size_t* lencf, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_list_column_families_destroy( + char** list, size_t len); + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* +rocksdb_create_column_family(rocksdb_t* db, + const rocksdb_options_t* column_family_options, + const char* column_family_name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* +rocksdb_create_column_family_with_ttl( + rocksdb_t* db, const rocksdb_options_t* column_family_options, + const char* column_family_name, int ttl, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_drop_column_family( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_column_family_handle_destroy( + rocksdb_column_family_handle_t*); + +extern ROCKSDB_LIBRARY_API uint32_t +rocksdb_column_family_handle_get_id(rocksdb_column_family_handle_t* handle); + +extern ROCKSDB_LIBRARY_API char* rocksdb_column_family_handle_get_name( + rocksdb_column_family_handle_t* handle, size_t* name_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_close(rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_put( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_put_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_range_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* start_key, + size_t start_key_len, const char* end_key, size_t end_key_len, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_merge( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_merge_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_write( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_writebatch_t* batch, char** errptr); + +/* Returns NULL if not found. A malloc()ed array otherwise. + Stores the length of the array in *vallen. */ +extern ROCKSDB_LIBRARY_API char* rocksdb_get( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t keylen, size_t* vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t keylen, size_t* vallen, char** ts, size_t* tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_cf_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** ts, size_t* tslen, char** errptr); + +// if values_list[i] == NULL and errs[i] == NULL, +// then we got status.IsNotFound(), which we will not return. +// all errors except status status.ok() and status.IsNotFound() are returned. +// +// errs, values_list and values_list_sizes must be num_keys in length, +// allocated by the caller. +// errs is a list of strings as opposed to the conventional one error, +// where errs[i] is the status for retrieval of keys_list[i]. +// each non-NULL errs entry is a malloc()ed, null terminated string. +// each non-NULL values_list entry is a malloc()ed array, with +// the length for each stored in values_list_sizes[i]. +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get( + rocksdb_t* db, const rocksdb_readoptions_t* options, size_t num_keys, + const char* const* keys_list, const size_t* keys_list_sizes, + char** values_list, size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, size_t num_keys, + const char* const* keys_list, const size_t* keys_list_sizes, + char** values_list, size_t* values_list_sizes, char** timestamp_list, + size_t* timestamp_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_cf_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** timestamps_list, + size_t* timestamps_list_sizes, char** errs); + +// The MultiGet API that improves performance by batching operations +// in the read path for greater efficiency. Currently, only the block based +// table format with full filters are supported. Other table formats such +// as plain table, block based table with block based filters and +// partitioned indexes will still work, but will not get any performance +// benefits. +// +// Note that all the keys passed to this API are restricted to a single +// column family. +// +// Parameters - +// db - the RocksDB instance. +// options - ReadOptions +// column_family - ColumnFamilyHandle* that the keys belong to. All the keys +// passed to the API are restricted to a single column family +// num_keys - Number of keys to lookup +// keys_list - Pointer to C style array of keys with num_keys elements +// keys_list_sizes - Pointer to C style array of the size of corresponding key +// in key_list with num_keys elements. +// values - Pointer to C style array of PinnableSlices with num_keys elements +// statuses - Pointer to C style array of Status with num_keys elements +// sorted_input - If true, it means the input keys are already sorted by key +// order, so the MultiGet() API doesn't have to sort them +// again. If false, the keys will be copied and sorted +// internally by the API - the input array will not be +// modified +extern ROCKSDB_LIBRARY_API void rocksdb_batched_multi_get_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, size_t num_keys, + const char* const* keys_list, const size_t* keys_list_sizes, + rocksdb_pinnableslice_t** values, char** errs, const bool sorted_input); + +// The value is only allocated (using malloc) and returned if it is found and +// value_found isn't NULL. In that case the user is responsible for freeing it. +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_key_may_exist( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t key_len, char** value, size_t* val_len, const char* timestamp, + size_t timestamp_len, unsigned char* value_found); + +// The value is only allocated (using malloc) and returned if it is found and +// value_found isn't NULL. In that case the user is responsible for freeing it. +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_key_may_exist_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t key_len, char** value, size_t* val_len, const char* timestamp, + size_t timestamp_len, unsigned char* value_found); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* rocksdb_create_iterator( + rocksdb_t* db, const rocksdb_readoptions_t* options); + +extern ROCKSDB_LIBRARY_API rocksdb_wal_iterator_t* rocksdb_get_updates_since( + rocksdb_t* db, uint64_t seq_number, + const rocksdb_wal_readoptions_t* options, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* rocksdb_create_iterator_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API void rocksdb_create_iterators( + rocksdb_t* db, rocksdb_readoptions_t* opts, + rocksdb_column_family_handle_t** column_families, + rocksdb_iterator_t** iterators, size_t size, char** errptr); + +extern ROCKSDB_LIBRARY_API const rocksdb_snapshot_t* rocksdb_create_snapshot( + rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_release_snapshot( + rocksdb_t* db, const rocksdb_snapshot_t* snapshot); + +/* Returns NULL if property name is unknown. + Else returns a pointer to a malloc()-ed null-terminated value. */ +extern ROCKSDB_LIBRARY_API char* rocksdb_property_value(rocksdb_t* db, + const char* propname); +/* returns 0 on success, -1 otherwise */ +extern ROCKSDB_LIBRARY_API int rocksdb_property_int(rocksdb_t* db, + const char* propname, + uint64_t* out_val); + +/* returns 0 on success, -1 otherwise */ +extern ROCKSDB_LIBRARY_API int rocksdb_property_int_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* propname, uint64_t* out_val); + +extern ROCKSDB_LIBRARY_API char* rocksdb_property_value_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* propname); + +extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes( + rocksdb_t* db, int num_ranges, const char* const* range_start_key, + const size_t* range_start_key_len, const char* const* range_limit_key, + const size_t* range_limit_key_len, uint64_t* sizes, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + int num_ranges, const char* const* range_start_key, + const size_t* range_start_key_len, const char* const* range_limit_key, + const size_t* range_limit_key_len, uint64_t* sizes, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range(rocksdb_t* db, + const char* start_key, + size_t start_key_len, + const char* limit_key, + size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* limit_key, + size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_suggest_compact_range( + rocksdb_t* db, const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_suggest_compact_range_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* limit_key, + size_t limit_key_len, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range_opt( + rocksdb_t* db, rocksdb_compactoptions_t* opt, const char* start_key, + size_t start_key_len, const char* limit_key, size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range_cf_opt( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + rocksdb_compactoptions_t* opt, const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_file(rocksdb_t* db, + const char* name); + +extern ROCKSDB_LIBRARY_API const rocksdb_livefiles_t* rocksdb_livefiles( + rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush( + rocksdb_t* db, const rocksdb_flushoptions_t* options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush_cf( + rocksdb_t* db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t* column_family, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush_cfs( + rocksdb_t* db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t** column_family, int num_column_families, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush_wal(rocksdb_t* db, + unsigned char sync, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_disable_file_deletions(rocksdb_t* db, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_enable_file_deletions( + rocksdb_t* db, unsigned char force, char** errptr); + +/* Management operations */ + +extern ROCKSDB_LIBRARY_API void rocksdb_destroy_db( + const rocksdb_options_t* options, const char* name, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_repair_db( + const rocksdb_options_t* options, const char* name, char** errptr); + +/* Iterator */ + +extern ROCKSDB_LIBRARY_API void rocksdb_iter_destroy(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_iter_valid( + const rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek_to_first(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek_to_last(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek(rocksdb_iterator_t*, + const char* k, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek_for_prev(rocksdb_iterator_t*, + const char* k, + size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_next(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_prev(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API const char* rocksdb_iter_key( + const rocksdb_iterator_t*, size_t* klen); +extern ROCKSDB_LIBRARY_API const char* rocksdb_iter_value( + const rocksdb_iterator_t*, size_t* vlen); +extern ROCKSDB_LIBRARY_API const char* rocksdb_iter_timestamp( + const rocksdb_iterator_t*, size_t* tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_get_error( + const rocksdb_iterator_t*, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_wal_iter_next( + rocksdb_wal_iterator_t* iter); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_wal_iter_valid( + const rocksdb_wal_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_wal_iter_status( + const rocksdb_wal_iterator_t* iter, char** errptr); +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* rocksdb_wal_iter_get_batch( + const rocksdb_wal_iterator_t* iter, uint64_t* seq); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_get_latest_sequence_number(rocksdb_t* db); +extern ROCKSDB_LIBRARY_API void rocksdb_wal_iter_destroy( + const rocksdb_wal_iterator_t* iter); + +/* Write batch */ + +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* rocksdb_writebatch_create( + void); +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* rocksdb_writebatch_create_from( + const char* rep, size_t size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_destroy( + rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_clear(rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API int rocksdb_writebatch_count(rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put(rocksdb_writebatch_t*, + const char* key, + size_t klen, + const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put_cf( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put_cf_with_ts( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen, const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_putv( + rocksdb_writebatch_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_putv_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_merge(rocksdb_writebatch_t*, + const char* key, + size_t klen, + const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_merge_cf( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_mergev( + rocksdb_writebatch_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_mergev_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete(rocksdb_writebatch_t*, + const char* key, + size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_singledelete( + rocksdb_writebatch_t* b, const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_cf( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_cf_with_ts( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_singledelete_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_singledelete_cf_with_ts( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev( + rocksdb_writebatch_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_range( + rocksdb_writebatch_t* b, const char* start_key, size_t start_key_len, + const char* end_key, size_t end_key_len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_range_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* end_key, + size_t end_key_len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_rangev( + rocksdb_writebatch_t* b, int num_keys, const char* const* start_keys_list, + const size_t* start_keys_list_sizes, const char* const* end_keys_list, + const size_t* end_keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_rangev_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* start_keys_list, + const size_t* start_keys_list_sizes, const char* const* end_keys_list, + const size_t* end_keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put_log_data( + rocksdb_writebatch_t*, const char* blob, size_t len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_iterate( + rocksdb_writebatch_t*, void* state, + void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), + void (*deleted)(void*, const char* k, size_t klen)); +extern ROCKSDB_LIBRARY_API const char* rocksdb_writebatch_data( + rocksdb_writebatch_t*, size_t* size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_set_save_point( + rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_rollback_to_save_point( + rocksdb_writebatch_t*, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_pop_save_point( + rocksdb_writebatch_t*, char** errptr); + +/* Write batch with index */ + +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_wi_t* +rocksdb_writebatch_wi_create(size_t reserved_bytes, + unsigned char overwrite_keys); +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_wi_t* +rocksdb_writebatch_wi_create_from(const char* rep, size_t size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_destroy( + rocksdb_writebatch_wi_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_clear( + rocksdb_writebatch_wi_t*); +extern ROCKSDB_LIBRARY_API int rocksdb_writebatch_wi_count( + rocksdb_writebatch_wi_t* b); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_put( + rocksdb_writebatch_wi_t*, const char* key, size_t klen, const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_put_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_putv( + rocksdb_writebatch_wi_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_putv_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_merge( + rocksdb_writebatch_wi_t*, const char* key, size_t klen, const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_merge_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_mergev( + rocksdb_writebatch_wi_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_mergev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete( + rocksdb_writebatch_wi_t*, const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_singledelete( + rocksdb_writebatch_wi_t*, const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_singledelete_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_deletev( + rocksdb_writebatch_wi_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_deletev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes); +// DO NOT USE - rocksdb_writebatch_wi_delete_range is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_range( + rocksdb_writebatch_wi_t* b, const char* start_key, size_t start_key_len, + const char* end_key, size_t end_key_len); +// DO NOT USE - rocksdb_writebatch_wi_delete_range_cf is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_range_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* end_key, + size_t end_key_len); +// DO NOT USE - rocksdb_writebatch_wi_delete_rangev is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_rangev( + rocksdb_writebatch_wi_t* b, int num_keys, + const char* const* start_keys_list, const size_t* start_keys_list_sizes, + const char* const* end_keys_list, const size_t* end_keys_list_sizes); +// DO NOT USE - rocksdb_writebatch_wi_delete_rangev_cf is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_rangev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* start_keys_list, + const size_t* start_keys_list_sizes, const char* const* end_keys_list, + const size_t* end_keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_put_log_data( + rocksdb_writebatch_wi_t*, const char* blob, size_t len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_iterate( + rocksdb_writebatch_wi_t* b, void* state, + void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), + void (*deleted)(void*, const char* k, size_t klen)); +extern ROCKSDB_LIBRARY_API const char* rocksdb_writebatch_wi_data( + rocksdb_writebatch_wi_t* b, size_t* size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_set_save_point( + rocksdb_writebatch_wi_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_rollback_to_save_point( + rocksdb_writebatch_wi_t*, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch( + rocksdb_writebatch_wi_t* wbwi, const rocksdb_options_t* options, + const char* key, size_t keylen, size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch_cf( + rocksdb_writebatch_wi_t* wbwi, const rocksdb_options_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch_and_db( + rocksdb_writebatch_wi_t* wbwi, rocksdb_t* db, + const rocksdb_readoptions_t* options, const char* key, size_t keylen, + size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch_and_db_cf( + rocksdb_writebatch_wi_t* wbwi, rocksdb_t* db, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_write_writebatch_wi( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_writebatch_wi_t* wbwi, char** errptr); +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_writebatch_wi_create_iterator_with_base( + rocksdb_writebatch_wi_t* wbwi, rocksdb_iterator_t* base_iterator); +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_writebatch_wi_create_iterator_with_base_cf( + rocksdb_writebatch_wi_t* wbwi, rocksdb_iterator_t* base_iterator, + rocksdb_column_family_handle_t* cf); + +/* Options utils */ + +// Load the latest rocksdb options from the specified db_path. +// +// On success, num_column_families will be updated with a non-zero +// number indicating the number of column families. +// The returned db_options, column_family_names, and column_family_options +// should be released via rocksdb_load_latest_options_destroy(). +// +// On error, a non-null errptr that includes the error message will be +// returned. db_options, column_family_names, and column_family_options +// will be set to NULL. +extern ROCKSDB_LIBRARY_API void rocksdb_load_latest_options( + const char* db_path, rocksdb_env_t* env, bool ignore_unknown_options, + rocksdb_cache_t* cache, rocksdb_options_t** db_options, + size_t* num_column_families, char*** column_family_names, + rocksdb_options_t*** column_family_options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_load_latest_options_destroy( + rocksdb_options_t* db_options, char** list_column_family_names, + rocksdb_options_t** list_column_family_options, size_t len); + +/* Block based table options */ + +extern ROCKSDB_LIBRARY_API rocksdb_block_based_table_options_t* +rocksdb_block_based_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_destroy( + rocksdb_block_based_table_options_t* options); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_checksum( + rocksdb_block_based_table_options_t*, char); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_block_size( + rocksdb_block_based_table_options_t* options, size_t block_size); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_block_size_deviation( + rocksdb_block_based_table_options_t* options, int block_size_deviation); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_block_restart_interval( + rocksdb_block_based_table_options_t* options, int block_restart_interval); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_index_block_restart_interval( + rocksdb_block_based_table_options_t* options, + int index_block_restart_interval); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_metadata_block_size( + rocksdb_block_based_table_options_t* options, uint64_t metadata_block_size); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_partition_filters( + rocksdb_block_based_table_options_t* options, + unsigned char partition_filters); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_optimize_filters_for_memory( + rocksdb_block_based_table_options_t* options, + unsigned char optimize_filters_for_memory); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_use_delta_encoding( + rocksdb_block_based_table_options_t* options, + unsigned char use_delta_encoding); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_filter_policy( + rocksdb_block_based_table_options_t* options, + rocksdb_filterpolicy_t* filter_policy); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_no_block_cache( + rocksdb_block_based_table_options_t* options, unsigned char no_block_cache); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_block_cache( + rocksdb_block_based_table_options_t* options, rocksdb_cache_t* block_cache); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_whole_key_filtering( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_format_version( + rocksdb_block_based_table_options_t*, int); +enum { + rocksdb_block_based_table_index_type_binary_search = 0, + rocksdb_block_based_table_index_type_hash_search = 1, + rocksdb_block_based_table_index_type_two_level_index_search = 2, +}; +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_index_type( + rocksdb_block_based_table_options_t*, int); // uses one of the above enums +enum { + rocksdb_block_based_table_data_block_index_type_binary_search = 0, + rocksdb_block_based_table_data_block_index_type_binary_search_and_hash = 1, +}; +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_data_block_index_type( + rocksdb_block_based_table_options_t*, int); // uses one of the above enums +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_data_block_hash_ratio( + rocksdb_block_based_table_options_t* options, double v); +// rocksdb_block_based_options_set_hash_index_allow_collision() +// is removed since BlockBasedTableOptions.hash_index_allow_collision() +// is removed +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_cache_index_and_filter_blocks( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_pin_top_level_index_and_filter( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_block_based_table_factory( + rocksdb_options_t* opt, rocksdb_block_based_table_options_t* table_options); + +/* Cuckoo table options */ + +extern ROCKSDB_LIBRARY_API rocksdb_cuckoo_table_options_t* +rocksdb_cuckoo_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_destroy( + rocksdb_cuckoo_table_options_t* options); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_hash_ratio( + rocksdb_cuckoo_table_options_t* options, double v); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_max_search_depth( + rocksdb_cuckoo_table_options_t* options, uint32_t v); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_cuckoo_block_size( + rocksdb_cuckoo_table_options_t* options, uint32_t v); +extern ROCKSDB_LIBRARY_API void +rocksdb_cuckoo_options_set_identity_as_first_hash( + rocksdb_cuckoo_table_options_t* options, unsigned char v); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_use_module_hash( + rocksdb_cuckoo_table_options_t* options, unsigned char v); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_cuckoo_table_factory( + rocksdb_options_t* opt, rocksdb_cuckoo_table_options_t* table_options); + +/* Options */ +extern ROCKSDB_LIBRARY_API void rocksdb_set_options(rocksdb_t* db, int count, + const char* const keys[], + const char* const values[], + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_set_options_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, int count, + const char* const keys[], const char* const values[], char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_options_t* rocksdb_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_options_destroy(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API rocksdb_options_t* rocksdb_options_create_copy( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_increase_parallelism( + rocksdb_options_t* opt, int total_threads); +extern ROCKSDB_LIBRARY_API void rocksdb_options_optimize_for_point_lookup( + rocksdb_options_t* opt, uint64_t block_cache_size_mb); +extern ROCKSDB_LIBRARY_API void rocksdb_options_optimize_level_style_compaction( + rocksdb_options_t* opt, uint64_t memtable_memory_budget); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_optimize_universal_style_compaction( + rocksdb_options_t* opt, uint64_t memtable_memory_budget); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_allow_ingest_behind( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_allow_ingest_behind(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compaction_filter( + rocksdb_options_t*, rocksdb_compactionfilter_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compaction_filter_factory( + rocksdb_options_t*, rocksdb_compactionfilterfactory_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_compaction_readahead_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_compaction_readahead_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_comparator( + rocksdb_options_t*, rocksdb_comparator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_merge_operator( + rocksdb_options_t*, rocksdb_mergeoperator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_uint64add_merge_operator( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compression_per_level( + rocksdb_options_t* opt, const int* level_values, size_t num_levels); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_create_if_missing( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_create_if_missing( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_create_missing_column_families(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_create_missing_column_families(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_error_if_exists( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_error_if_exists( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_paranoid_checks( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_paranoid_checks( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_db_paths( + rocksdb_options_t*, const rocksdb_dbpath_t** path_values, size_t num_paths); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_env(rocksdb_options_t*, + rocksdb_env_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_info_log(rocksdb_options_t*, + rocksdb_logger_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_info_log_level( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_info_log_level( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_write_buffer_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_write_buffer_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_db_write_buffer_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_db_write_buffer_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_open_files( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_open_files( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_file_opening_threads( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_file_opening_threads( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_total_wal_size( + rocksdb_options_t* opt, uint64_t n); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_total_wal_size(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compression_options( + rocksdb_options_t*, int, int, int, int); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_zstd_max_train_bytes(rocksdb_options_t*, + int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_compression_options_zstd_max_train_bytes( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_use_zstd_dict_trainer( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_compression_options_use_zstd_dict_trainer( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_parallel_threads(rocksdb_options_t*, + int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_compression_options_parallel_threads( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_max_dict_buffer_bytes( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_compression_options_max_dict_buffer_bytes( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options(rocksdb_options_t*, int, int, + int, int, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options_zstd_max_train_bytes( + rocksdb_options_t*, int, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options_use_zstd_dict_trainer( + rocksdb_options_t*, unsigned char, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_bottommost_compression_options_use_zstd_dict_trainer( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options_max_dict_buffer_bytes( + rocksdb_options_t*, uint64_t, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_prefix_extractor( + rocksdb_options_t*, rocksdb_slicetransform_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_num_levels( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_num_levels( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_level0_file_num_compaction_trigger(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_level0_file_num_compaction_trigger(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_level0_slowdown_writes_trigger(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_level0_slowdown_writes_trigger(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_level0_stop_writes_trigger( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_level0_stop_writes_trigger( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_target_file_size_base( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_target_file_size_base(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_target_file_size_multiplier( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_target_file_size_multiplier( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_bytes_for_level_base( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_bytes_for_level_base(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_level_compaction_dynamic_level_bytes(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_level_compaction_dynamic_level_bytes(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_bytes_for_level_multiplier(rocksdb_options_t*, double); +extern ROCKSDB_LIBRARY_API double +rocksdb_options_get_max_bytes_for_level_multiplier(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_bytes_for_level_multiplier_additional( + rocksdb_options_t*, int* level_values, size_t num_levels); +extern ROCKSDB_LIBRARY_API void rocksdb_options_enable_statistics( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt, + unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_skip_stats_update_on_db_open(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open( + rocksdb_options_t* opt, unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open( + rocksdb_options_t* opt); + +/* Blob Options Settings */ +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_blob_files( + rocksdb_options_t* opt, unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_enable_blob_files( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_min_blob_size( + rocksdb_options_t* opt, uint64_t val); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_min_blob_size(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_file_size( + rocksdb_options_t* opt, uint64_t val); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_blob_file_size(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_compression_type( + rocksdb_options_t* opt, int val); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_blob_compression_type( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_blob_gc( + rocksdb_options_t* opt, unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_enable_blob_gc( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_gc_age_cutoff( + rocksdb_options_t* opt, double val); +extern ROCKSDB_LIBRARY_API double rocksdb_options_get_blob_gc_age_cutoff( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_gc_force_threshold( + rocksdb_options_t* opt, double val); +extern ROCKSDB_LIBRARY_API double rocksdb_options_get_blob_gc_force_threshold( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_blob_compaction_readahead_size(rocksdb_options_t* opt, + uint64_t val); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_blob_compaction_readahead_size(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_file_starting_level( + rocksdb_options_t* opt, int val); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_blob_file_starting_level( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_cache( + rocksdb_options_t* opt, rocksdb_cache_t* blob_cache); + +enum { + rocksdb_prepopulate_blob_disable = 0, + rocksdb_prepopulate_blob_flush_only = 1 +}; + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_prepopulate_blob_cache( + rocksdb_options_t* opt, int val); + +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_prepopulate_blob_cache( + rocksdb_options_t* opt); + +/* returns a pointer to a malloc()-ed, null terminated string */ +extern ROCKSDB_LIBRARY_API char* rocksdb_options_statistics_get_string( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_write_buffer_number( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_write_buffer_number( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_min_write_buffer_number_to_merge(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_min_write_buffer_number_to_merge(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_write_buffer_number_to_maintain(rocksdb_options_t*, + int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_max_write_buffer_number_to_maintain(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_write_buffer_size_to_maintain(rocksdb_options_t*, + int64_t); +extern ROCKSDB_LIBRARY_API int64_t +rocksdb_options_get_max_write_buffer_size_to_maintain(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_pipelined_write( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_enable_pipelined_write(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_unordered_write( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_unordered_write( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_subcompactions( + rocksdb_options_t*, uint32_t); +extern ROCKSDB_LIBRARY_API uint32_t +rocksdb_options_get_max_subcompactions(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_background_jobs( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_background_jobs( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_background_compactions( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_background_compactions( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_background_flushes( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_background_flushes( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_log_file_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_max_log_file_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_log_file_time_to_roll( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_log_file_time_to_roll(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_keep_log_file_num( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_keep_log_file_num(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_recycle_log_file_num( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_recycle_log_file_num(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_soft_pending_compaction_bytes_limit(rocksdb_options_t* opt, + size_t v); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_soft_pending_compaction_bytes_limit(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_hard_pending_compaction_bytes_limit(rocksdb_options_t* opt, + size_t v); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_hard_pending_compaction_bytes_limit(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_manifest_file_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_max_manifest_file_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_table_cache_numshardbits( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_table_cache_numshardbits( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_arena_block_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_arena_block_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_use_fsync( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_use_fsync( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_db_log_dir( + rocksdb_options_t*, const char*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_dir(rocksdb_options_t*, + const char*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_WAL_ttl_seconds( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_WAL_ttl_seconds(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_WAL_size_limit_MB( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_WAL_size_limit_MB(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_manifest_preallocation_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_manifest_preallocation_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_allow_mmap_reads( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_allow_mmap_reads( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_allow_mmap_writes( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_allow_mmap_writes( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_use_direct_reads( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_use_direct_reads( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_use_direct_io_for_flush_and_compaction(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_use_direct_io_for_flush_and_compaction(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_is_fd_close_on_exec( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_is_fd_close_on_exec(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_stats_dump_period_sec( + rocksdb_options_t*, unsigned int); +extern ROCKSDB_LIBRARY_API unsigned int +rocksdb_options_get_stats_dump_period_sec(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_stats_persist_period_sec( + rocksdb_options_t*, unsigned int); +extern ROCKSDB_LIBRARY_API unsigned int +rocksdb_options_get_stats_persist_period_sec(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_advise_random_on_open( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_advise_random_on_open(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_access_hint_on_compaction_start(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_access_hint_on_compaction_start(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_use_adaptive_mutex( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_use_adaptive_mutex( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bytes_per_sync( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_bytes_per_sync(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_bytes_per_sync( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_wal_bytes_per_sync(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_writable_file_max_buffer_size(rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_writable_file_max_buffer_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_allow_concurrent_memtable_write(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_allow_concurrent_memtable_write(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_enable_write_thread_adaptive_yield(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_enable_write_thread_adaptive_yield(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_sequential_skip_in_iterations(rocksdb_options_t*, + uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_sequential_skip_in_iterations(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_disable_auto_compactions( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_disable_auto_compactions(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_optimize_filters_for_hits( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_optimize_filters_for_hits(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_delete_obsolete_files_period_micros(rocksdb_options_t*, + uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_delete_obsolete_files_period_micros(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_prepare_for_bulk_load( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_memtable_vector_rep( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_memtable_prefix_bloom_size_ratio(rocksdb_options_t*, + double); +extern ROCKSDB_LIBRARY_API double +rocksdb_options_get_memtable_prefix_bloom_size_ratio(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_compaction_bytes( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_compaction_bytes(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_hash_skip_list_rep( + rocksdb_options_t*, size_t, int32_t, int32_t); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_hash_link_list_rep( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_plain_table_factory( + rocksdb_options_t*, uint32_t, int, double, size_t); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_min_level_to_compress( + rocksdb_options_t* opt, int level); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_memtable_huge_page_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_memtable_huge_page_size(rocksdb_options_t*); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_successive_merges( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_max_successive_merges(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bloom_locality( + rocksdb_options_t*, uint32_t); +extern ROCKSDB_LIBRARY_API uint32_t +rocksdb_options_get_bloom_locality(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_inplace_update_support( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_inplace_update_support(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_inplace_update_num_locks( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_inplace_update_num_locks(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_report_bg_io_stats( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_report_bg_io_stats( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_avoid_unnecessary_blocking_io(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_avoid_unnecessary_blocking_io(rocksdb_options_t*); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_experimental_mempurge_threshold(rocksdb_options_t*, double); +extern ROCKSDB_LIBRARY_API double +rocksdb_options_get_experimental_mempurge_threshold(rocksdb_options_t*); + +enum { + rocksdb_tolerate_corrupted_tail_records_recovery = 0, + rocksdb_absolute_consistency_recovery = 1, + rocksdb_point_in_time_recovery = 2, + rocksdb_skip_any_corrupted_records_recovery = 3 +}; +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_recovery_mode( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_wal_recovery_mode( + rocksdb_options_t*); + +enum { + rocksdb_no_compression = 0, + rocksdb_snappy_compression = 1, + rocksdb_zlib_compression = 2, + rocksdb_bz2_compression = 3, + rocksdb_lz4_compression = 4, + rocksdb_lz4hc_compression = 5, + rocksdb_xpress_compression = 6, + rocksdb_zstd_compression = 7 +}; +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compression( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_compression( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bottommost_compression( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_bottommost_compression( + rocksdb_options_t*); + +enum { + rocksdb_level_compaction = 0, + rocksdb_universal_compaction = 1, + rocksdb_fifo_compaction = 2 +}; +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compaction_style( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_compaction_style( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_universal_compaction_options( + rocksdb_options_t*, rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_fifo_compaction_options( + rocksdb_options_t* opt, rocksdb_fifo_compaction_options_t* fifo); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_ratelimiter( + rocksdb_options_t* opt, rocksdb_ratelimiter_t* limiter); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_atomic_flush( + rocksdb_options_t* opt, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_atomic_flush( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_row_cache( + rocksdb_options_t* opt, rocksdb_cache_t* cache); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_add_compact_on_deletion_collector_factory( + rocksdb_options_t*, size_t window_size, size_t num_dels_trigger); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_manual_wal_flush( + rocksdb_options_t* opt, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_manual_wal_flush( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_compression( + rocksdb_options_t* opt, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_wal_compression( + rocksdb_options_t* opt); + +/* RateLimiter */ +extern ROCKSDB_LIBRARY_API rocksdb_ratelimiter_t* rocksdb_ratelimiter_create( + int64_t rate_bytes_per_sec, int64_t refill_period_us, int32_t fairness); +extern ROCKSDB_LIBRARY_API void rocksdb_ratelimiter_destroy( + rocksdb_ratelimiter_t*); + +/* PerfContext */ +enum { + rocksdb_uninitialized = 0, + rocksdb_disable = 1, + rocksdb_enable_count = 2, + rocksdb_enable_time_except_for_mutex = 3, + rocksdb_enable_time = 4, + rocksdb_out_of_bounds = 5 +}; + +enum { + rocksdb_user_key_comparison_count = 0, + rocksdb_block_cache_hit_count, + rocksdb_block_read_count, + rocksdb_block_read_byte, + rocksdb_block_read_time, + rocksdb_block_checksum_time, + rocksdb_block_decompress_time, + rocksdb_get_read_bytes, + rocksdb_multiget_read_bytes, + rocksdb_iter_read_bytes, + rocksdb_internal_key_skipped_count, + rocksdb_internal_delete_skipped_count, + rocksdb_internal_recent_skipped_count, + rocksdb_internal_merge_count, + rocksdb_get_snapshot_time, + rocksdb_get_from_memtable_time, + rocksdb_get_from_memtable_count, + rocksdb_get_post_process_time, + rocksdb_get_from_output_files_time, + rocksdb_seek_on_memtable_time, + rocksdb_seek_on_memtable_count, + rocksdb_next_on_memtable_count, + rocksdb_prev_on_memtable_count, + rocksdb_seek_child_seek_time, + rocksdb_seek_child_seek_count, + rocksdb_seek_min_heap_time, + rocksdb_seek_max_heap_time, + rocksdb_seek_internal_seek_time, + rocksdb_find_next_user_entry_time, + rocksdb_write_wal_time, + rocksdb_write_memtable_time, + rocksdb_write_delay_time, + rocksdb_write_pre_and_post_process_time, + rocksdb_db_mutex_lock_nanos, + rocksdb_db_condition_wait_nanos, + rocksdb_merge_operator_time_nanos, + rocksdb_read_index_block_nanos, + rocksdb_read_filter_block_nanos, + rocksdb_new_table_block_iter_nanos, + rocksdb_new_table_iterator_nanos, + rocksdb_block_seek_nanos, + rocksdb_find_table_nanos, + rocksdb_bloom_memtable_hit_count, + rocksdb_bloom_memtable_miss_count, + rocksdb_bloom_sst_hit_count, + rocksdb_bloom_sst_miss_count, + rocksdb_key_lock_wait_time, + rocksdb_key_lock_wait_count, + rocksdb_env_new_sequential_file_nanos, + rocksdb_env_new_random_access_file_nanos, + rocksdb_env_new_writable_file_nanos, + rocksdb_env_reuse_writable_file_nanos, + rocksdb_env_new_random_rw_file_nanos, + rocksdb_env_new_directory_nanos, + rocksdb_env_file_exists_nanos, + rocksdb_env_get_children_nanos, + rocksdb_env_get_children_file_attributes_nanos, + rocksdb_env_delete_file_nanos, + rocksdb_env_create_dir_nanos, + rocksdb_env_create_dir_if_missing_nanos, + rocksdb_env_delete_dir_nanos, + rocksdb_env_get_file_size_nanos, + rocksdb_env_get_file_modification_time_nanos, + rocksdb_env_rename_file_nanos, + rocksdb_env_link_file_nanos, + rocksdb_env_lock_file_nanos, + rocksdb_env_unlock_file_nanos, + rocksdb_env_new_logger_nanos, + rocksdb_number_async_seek, + rocksdb_blob_cache_hit_count, + rocksdb_blob_read_count, + rocksdb_blob_read_byte, + rocksdb_blob_read_time, + rocksdb_blob_checksum_time, + rocksdb_blob_decompress_time, + rocksdb_internal_range_del_reseek_count, + rocksdb_total_metric_count = 78 +}; + +extern ROCKSDB_LIBRARY_API void rocksdb_set_perf_level(int); +extern ROCKSDB_LIBRARY_API rocksdb_perfcontext_t* rocksdb_perfcontext_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_perfcontext_reset( + rocksdb_perfcontext_t* context); +extern ROCKSDB_LIBRARY_API char* rocksdb_perfcontext_report( + rocksdb_perfcontext_t* context, unsigned char exclude_zero_counters); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_perfcontext_metric(rocksdb_perfcontext_t* context, int metric); +extern ROCKSDB_LIBRARY_API void rocksdb_perfcontext_destroy( + rocksdb_perfcontext_t* context); + +/* Compaction Filter */ + +extern ROCKSDB_LIBRARY_API rocksdb_compactionfilter_t* +rocksdb_compactionfilter_create( + void* state, void (*destructor)(void*), + unsigned char (*filter)(void*, int level, const char* key, + size_t key_length, const char* existing_value, + size_t value_length, char** new_value, + size_t* new_value_length, + unsigned char* value_changed), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_compactionfilter_set_ignore_snapshots( + rocksdb_compactionfilter_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_compactionfilter_destroy( + rocksdb_compactionfilter_t*); + +/* Compaction Filter Context */ + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactionfiltercontext_is_full_compaction( + rocksdb_compactionfiltercontext_t* context); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactionfiltercontext_is_manual_compaction( + rocksdb_compactionfiltercontext_t* context); + +/* Compaction Filter Factory */ + +extern ROCKSDB_LIBRARY_API rocksdb_compactionfilterfactory_t* +rocksdb_compactionfilterfactory_create( + void* state, void (*destructor)(void*), + rocksdb_compactionfilter_t* (*create_compaction_filter)( + void*, rocksdb_compactionfiltercontext_t* context), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_compactionfilterfactory_destroy( + rocksdb_compactionfilterfactory_t*); + +/* Comparator */ + +extern ROCKSDB_LIBRARY_API rocksdb_comparator_t* rocksdb_comparator_create( + void* state, void (*destructor)(void*), + int (*compare)(void*, const char* a, size_t alen, const char* b, + size_t blen), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_comparator_destroy( + rocksdb_comparator_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_comparator_t* +rocksdb_comparator_with_ts_create( + void* state, void (*destructor)(void*), + int (*compare)(void*, const char* a, size_t alen, const char* b, + size_t blen), + int (*compare_ts)(void*, const char* a_ts, size_t a_tslen, const char* b_ts, + size_t b_tslen), + int (*compare_without_ts)(void*, const char* a, size_t alen, + unsigned char a_has_ts, const char* b, + size_t blen, unsigned char b_has_ts), + const char* (*name)(void*), size_t timestamp_size); + +/* Filter policy */ + +extern ROCKSDB_LIBRARY_API void rocksdb_filterpolicy_destroy( + rocksdb_filterpolicy_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_bloom(double bits_per_key); +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_bloom_full(double bits_per_key); +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_ribbon(double bloom_equivalent_bits_per_key); +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_ribbon_hybrid(double bloom_equivalent_bits_per_key, + int bloom_before_level); + +/* Merge Operator */ + +extern ROCKSDB_LIBRARY_API rocksdb_mergeoperator_t* +rocksdb_mergeoperator_create( + void* state, void (*destructor)(void*), + char* (*full_merge)(void*, const char* key, size_t key_length, + const char* existing_value, + size_t existing_value_length, + const char* const* operands_list, + const size_t* operands_list_length, int num_operands, + unsigned char* success, size_t* new_value_length), + char* (*partial_merge)(void*, const char* key, size_t key_length, + const char* const* operands_list, + const size_t* operands_list_length, int num_operands, + unsigned char* success, size_t* new_value_length), + void (*delete_value)(void*, const char* value, size_t value_length), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_mergeoperator_destroy( + rocksdb_mergeoperator_t*); + +/* Read options */ + +extern ROCKSDB_LIBRARY_API rocksdb_readoptions_t* rocksdb_readoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_destroy( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_verify_checksums( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_verify_checksums(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_fill_cache( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_fill_cache( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_snapshot( + rocksdb_readoptions_t*, const rocksdb_snapshot_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iterate_upper_bound( + rocksdb_readoptions_t*, const char* key, size_t keylen); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iterate_lower_bound( + rocksdb_readoptions_t*, const char* key, size_t keylen); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_read_tier( + rocksdb_readoptions_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_readoptions_get_read_tier( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_tailing( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_tailing( + rocksdb_readoptions_t*); +// The functionality that this option controlled has been removed. +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_managed( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_readahead_size( + rocksdb_readoptions_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_readoptions_get_readahead_size(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_prefix_same_as_start( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_prefix_same_as_start(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_pin_data( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_pin_data( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_total_order_seek( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_total_order_seek(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_readoptions_set_max_skippable_internal_keys(rocksdb_readoptions_t*, + uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_readoptions_get_max_skippable_internal_keys(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_readoptions_set_background_purge_on_iterator_cleanup( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_background_purge_on_iterator_cleanup( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_ignore_range_deletions( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_ignore_range_deletions(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_deadline( + rocksdb_readoptions_t*, uint64_t microseconds); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_readoptions_get_deadline(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_io_timeout( + rocksdb_readoptions_t*, uint64_t microseconds); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_readoptions_get_io_timeout(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_async_io( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_async_io( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_timestamp( + rocksdb_readoptions_t*, const char* ts, size_t tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iter_start_ts( + rocksdb_readoptions_t*, const char* ts, size_t tslen); + +/* Write options */ + +extern ROCKSDB_LIBRARY_API rocksdb_writeoptions_t* rocksdb_writeoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_destroy( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_set_sync( + rocksdb_writeoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_sync( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_disable_WAL( + rocksdb_writeoptions_t* opt, int disable); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_disable_WAL( + rocksdb_writeoptions_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_writeoptions_set_ignore_missing_column_families(rocksdb_writeoptions_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_writeoptions_get_ignore_missing_column_families( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_set_no_slowdown( + rocksdb_writeoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_no_slowdown( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_set_low_pri( + rocksdb_writeoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_low_pri( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_writeoptions_set_memtable_insert_hint_per_batch(rocksdb_writeoptions_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_writeoptions_get_memtable_insert_hint_per_batch( + rocksdb_writeoptions_t*); + +/* Compact range options */ + +extern ROCKSDB_LIBRARY_API rocksdb_compactoptions_t* +rocksdb_compactoptions_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_destroy( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_compactoptions_set_exclusive_manual_compaction( + rocksdb_compactoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactoptions_get_exclusive_manual_compaction( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_compactoptions_set_bottommost_level_compaction( + rocksdb_compactoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactoptions_get_bottommost_level_compaction( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_set_change_level( + rocksdb_compactoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactoptions_get_change_level(rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_set_target_level( + rocksdb_compactoptions_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_compactoptions_get_target_level( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_set_full_history_ts_low( + rocksdb_compactoptions_t*, char* ts, size_t tslen); + +/* Flush options */ + +extern ROCKSDB_LIBRARY_API rocksdb_flushoptions_t* rocksdb_flushoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_flushoptions_destroy( + rocksdb_flushoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_flushoptions_set_wait( + rocksdb_flushoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_flushoptions_get_wait( + rocksdb_flushoptions_t*); + +/* Memory allocator */ + +extern ROCKSDB_LIBRARY_API rocksdb_memory_allocator_t* +rocksdb_jemalloc_nodump_allocator_create(char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_allocator_destroy( + rocksdb_memory_allocator_t*); + +/* Cache */ + +extern ROCKSDB_LIBRARY_API rocksdb_lru_cache_options_t* +rocksdb_lru_cache_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_destroy( + rocksdb_lru_cache_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_capacity( + rocksdb_lru_cache_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_num_shard_bits( + rocksdb_lru_cache_options_t*, int); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_memory_allocator( + rocksdb_lru_cache_options_t*, rocksdb_memory_allocator_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_lru( + size_t capacity); +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* +rocksdb_cache_create_lru_with_strict_capacity_limit(size_t capacity); +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_lru_opts( + rocksdb_lru_cache_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_cache_destroy(rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API void rocksdb_cache_disown_data( + rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API void rocksdb_cache_set_capacity( + rocksdb_cache_t* cache, size_t capacity); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_cache_get_capacity(rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_cache_get_usage(rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_cache_get_pinned_usage(rocksdb_cache_t* cache); + +/* HyperClockCache */ +extern ROCKSDB_LIBRARY_API rocksdb_hyper_clock_cache_options_t* +rocksdb_hyper_clock_cache_options_create(size_t capacity, + size_t estimated_entry_charge); +extern ROCKSDB_LIBRARY_API void rocksdb_hyper_clock_cache_options_destroy( + rocksdb_hyper_clock_cache_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_hyper_clock_cache_options_set_capacity( + rocksdb_hyper_clock_cache_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void +rocksdb_hyper_clock_cache_options_set_estimated_entry_charge( + rocksdb_hyper_clock_cache_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void +rocksdb_hyper_clock_cache_options_set_num_shard_bits( + rocksdb_hyper_clock_cache_options_t*, int); +extern ROCKSDB_LIBRARY_API void +rocksdb_hyper_clock_cache_options_set_memory_allocator( + rocksdb_hyper_clock_cache_options_t*, rocksdb_memory_allocator_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_hyper_clock( + size_t capacity, size_t estimated_entry_charge); +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* +rocksdb_cache_create_hyper_clock_opts(rocksdb_hyper_clock_cache_options_t*); + +/* DBPath */ + +extern ROCKSDB_LIBRARY_API rocksdb_dbpath_t* rocksdb_dbpath_create( + const char* path, uint64_t target_size); +extern ROCKSDB_LIBRARY_API void rocksdb_dbpath_destroy(rocksdb_dbpath_t*); + +/* Env */ + +extern ROCKSDB_LIBRARY_API rocksdb_env_t* rocksdb_create_default_env(void); +extern ROCKSDB_LIBRARY_API rocksdb_env_t* rocksdb_create_mem_env(void); +extern ROCKSDB_LIBRARY_API void rocksdb_env_set_background_threads( + rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int rocksdb_env_get_background_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_set_high_priority_background_threads(rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int rocksdb_env_get_high_priority_background_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_set_low_priority_background_threads( + rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int rocksdb_env_get_low_priority_background_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_set_bottom_priority_background_threads(rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int +rocksdb_env_get_bottom_priority_background_threads(rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_join_all_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_lower_thread_pool_io_priority( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_lower_high_priority_thread_pool_io_priority(rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_lower_thread_pool_cpu_priority( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_lower_high_priority_thread_pool_cpu_priority(rocksdb_env_t* env); + +extern ROCKSDB_LIBRARY_API void rocksdb_env_destroy(rocksdb_env_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_envoptions_t* rocksdb_envoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_envoptions_destroy( + rocksdb_envoptions_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_create_dir_if_missing( + rocksdb_env_t* env, const char* path, char** errptr); + +/* SstFile */ + +extern ROCKSDB_LIBRARY_API rocksdb_sstfilewriter_t* +rocksdb_sstfilewriter_create(const rocksdb_envoptions_t* env, + const rocksdb_options_t* io_options); +extern ROCKSDB_LIBRARY_API rocksdb_sstfilewriter_t* +rocksdb_sstfilewriter_create_with_comparator( + const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options, + const rocksdb_comparator_t* comparator); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_open( + rocksdb_sstfilewriter_t* writer, const char* name, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_add( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* val, size_t vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_put( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* val, size_t vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_put_with_ts( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* ts, size_t tslen, const char* val, size_t vallen, + char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_merge( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* val, size_t vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete_with_ts( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* ts, size_t tslen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete_range( + rocksdb_sstfilewriter_t* writer, const char* begin_key, size_t begin_keylen, + const char* end_key, size_t end_keylen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_finish( + rocksdb_sstfilewriter_t* writer, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_file_size( + rocksdb_sstfilewriter_t* writer, uint64_t* file_size); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_destroy( + rocksdb_sstfilewriter_t* writer); +extern ROCKSDB_LIBRARY_API rocksdb_ingestexternalfileoptions_t* +rocksdb_ingestexternalfileoptions_create(void); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_move_files( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char move_files); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_snapshot_consistency( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char snapshot_consistency); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_allow_global_seqno( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char allow_global_seqno); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_allow_blocking_flush( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char allow_blocking_flush); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_ingest_behind( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char ingest_behind); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_fail_if_not_bottommost_level( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char fail_if_not_bottommost_level); + +extern ROCKSDB_LIBRARY_API void rocksdb_ingestexternalfileoptions_destroy( + rocksdb_ingestexternalfileoptions_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_ingest_external_file( + rocksdb_t* db, const char* const* file_list, const size_t list_len, + const rocksdb_ingestexternalfileoptions_t* opt, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_ingest_external_file_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, + const char* const* file_list, const size_t list_len, + const rocksdb_ingestexternalfileoptions_t* opt, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_try_catch_up_with_primary( + rocksdb_t* db, char** errptr); + +/* SliceTransform */ + +extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t* +rocksdb_slicetransform_create( + void* state, void (*destructor)(void*), + char* (*transform)(void*, const char* key, size_t length, + size_t* dst_length), + unsigned char (*in_domain)(void*, const char* key, size_t length), + unsigned char (*in_range)(void*, const char* key, size_t length), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t* + rocksdb_slicetransform_create_fixed_prefix(size_t); +extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t* +rocksdb_slicetransform_create_noop(void); +extern ROCKSDB_LIBRARY_API void rocksdb_slicetransform_destroy( + rocksdb_slicetransform_t*); + +/* Universal Compaction options */ + +enum { + rocksdb_similar_size_compaction_stop_style = 0, + rocksdb_total_size_compaction_stop_style = 1 +}; + +extern ROCKSDB_LIBRARY_API rocksdb_universal_compaction_options_t* +rocksdb_universal_compaction_options_create(void); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_size_ratio( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_size_ratio( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_min_merge_width( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_min_merge_width( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_max_merge_width( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_max_merge_width( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_max_size_amplification_percent( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_max_size_amplification_percent( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_compression_size_percent( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_compression_size_percent( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_stop_style( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_stop_style( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_universal_compaction_options_destroy( + rocksdb_universal_compaction_options_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_fifo_compaction_options_t* +rocksdb_fifo_compaction_options_create(void); +extern ROCKSDB_LIBRARY_API void +rocksdb_fifo_compaction_options_set_allow_compaction( + rocksdb_fifo_compaction_options_t* fifo_opts, unsigned char allow_compaction); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_fifo_compaction_options_get_allow_compaction( + rocksdb_fifo_compaction_options_t* fifo_opts); +extern ROCKSDB_LIBRARY_API void +rocksdb_fifo_compaction_options_set_max_table_files_size( + rocksdb_fifo_compaction_options_t* fifo_opts, uint64_t size); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_fifo_compaction_options_get_max_table_files_size( + rocksdb_fifo_compaction_options_t* fifo_opts); +extern ROCKSDB_LIBRARY_API void rocksdb_fifo_compaction_options_destroy( + rocksdb_fifo_compaction_options_t* fifo_opts); + +extern ROCKSDB_LIBRARY_API int rocksdb_livefiles_count( + const rocksdb_livefiles_t*); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_column_family_name( + const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_name( + const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API int rocksdb_livefiles_level( + const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_livefiles_size(const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_smallestkey( + const rocksdb_livefiles_t*, int index, size_t* size); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_largestkey( + const rocksdb_livefiles_t*, int index, size_t* size); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_livefiles_entries(const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_livefiles_deletions(const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API void rocksdb_livefiles_destroy( + const rocksdb_livefiles_t*); + +/* Utility Helpers */ + +extern ROCKSDB_LIBRARY_API void rocksdb_get_options_from_string( + const rocksdb_options_t* base_options, const char* opts_str, + rocksdb_options_t* new_options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_file_in_range( + rocksdb_t* db, const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_file_in_range_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* limit_key, + size_t limit_key_len, char** errptr); + +/* MetaData */ + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_metadata_t* +rocksdb_get_column_family_metadata(rocksdb_t* db); + +/** + * Returns the rocksdb_column_family_metadata_t of the specified + * column family. + * + * Note that the caller is responsible to release the returned memory + * using rocksdb_column_family_metadata_destroy. + */ +extern ROCKSDB_LIBRARY_API rocksdb_column_family_metadata_t* +rocksdb_get_column_family_metadata_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API void rocksdb_column_family_metadata_destroy( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API uint64_t rocksdb_column_family_metadata_get_size( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API size_t rocksdb_column_family_metadata_get_file_count( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API char* rocksdb_column_family_metadata_get_name( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API size_t +rocksdb_column_family_metadata_get_level_count( + rocksdb_column_family_metadata_t* cf_meta); + +/** + * Returns the rocksdb_level_metadata_t of the ith level from the specified + * column family metadata. + * + * If the specified i is greater than or equal to the number of levels + * in the specified column family, then NULL will be returned. + * + * Note that the caller is responsible to release the returned memory + * using rocksdb_level_metadata_destroy before releasing its parent + * rocksdb_column_family_metadata_t. + */ +extern ROCKSDB_LIBRARY_API rocksdb_level_metadata_t* +rocksdb_column_family_metadata_get_level_metadata( + rocksdb_column_family_metadata_t* cf_meta, size_t i); + +/** + * Releases the specified rocksdb_level_metadata_t. + * + * Note that the specified rocksdb_level_metadata_t must be released + * before the release of its parent rocksdb_column_family_metadata_t. + */ +extern ROCKSDB_LIBRARY_API void rocksdb_level_metadata_destroy( + rocksdb_level_metadata_t* level_meta); + +extern ROCKSDB_LIBRARY_API int rocksdb_level_metadata_get_level( + rocksdb_level_metadata_t* level_meta); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_level_metadata_get_size(rocksdb_level_metadata_t* level_meta); + +extern ROCKSDB_LIBRARY_API size_t +rocksdb_level_metadata_get_file_count(rocksdb_level_metadata_t* level_meta); + +/** + * Returns the sst_file_metadata_t of the ith file from the specified level + * metadata. + * + * If the specified i is greater than or equal to the number of files + * in the specified level, then NULL will be returned. + * + * Note that the caller is responsible to release the returned memory + * using rocksdb_sst_file_metadata_destroy before releasing its + * parent rocksdb_level_metadata_t. + */ +extern ROCKSDB_LIBRARY_API rocksdb_sst_file_metadata_t* +rocksdb_level_metadata_get_sst_file_metadata( + rocksdb_level_metadata_t* level_meta, size_t i); + +/** + * Releases the specified rocksdb_sst_file_metadata_t. + * + * Note that the specified rocksdb_sst_file_metadata_t must be released + * before the release of its parent rocksdb_level_metadata_t. + */ +extern ROCKSDB_LIBRARY_API void rocksdb_sst_file_metadata_destroy( + rocksdb_sst_file_metadata_t* file_meta); + +extern ROCKSDB_LIBRARY_API char* +rocksdb_sst_file_metadata_get_relative_filename( + rocksdb_sst_file_metadata_t* file_meta); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_sst_file_metadata_get_size(rocksdb_sst_file_metadata_t* file_meta); + +/** + * Returns the smallest key of the specified sst file. + * The caller is responsible for releasing the returned memory. + * + * @param file_meta the metadata of an SST file to obtain its smallest key. + * @param len the out value which will contain the length of the returned key + * after the function call. + */ +extern ROCKSDB_LIBRARY_API char* rocksdb_sst_file_metadata_get_smallestkey( + rocksdb_sst_file_metadata_t* file_meta, size_t* len); + +/** + * Returns the smallest key of the specified sst file. + * The caller is responsible for releasing the returned memory. + * + * @param file_meta the metadata of an SST file to obtain its smallest key. + * @param len the out value which will contain the length of the returned key + * after the function call. + */ +extern ROCKSDB_LIBRARY_API char* rocksdb_sst_file_metadata_get_largestkey( + rocksdb_sst_file_metadata_t* file_meta, size_t* len); + +/* Transactions */ + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* +rocksdb_transactiondb_create_column_family( + rocksdb_transactiondb_t* txn_db, + const rocksdb_options_t* column_family_options, + const char* column_family_name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_transactiondb_t* rocksdb_transactiondb_open( + const rocksdb_options_t* options, + const rocksdb_transactiondb_options_t* txn_db_options, const char* name, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_transactiondb_t* +rocksdb_transactiondb_open_column_families( + const rocksdb_options_t* options, + const rocksdb_transactiondb_options_t* txn_db_options, const char* name, + int num_column_families, const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API const rocksdb_snapshot_t* +rocksdb_transactiondb_create_snapshot(rocksdb_transactiondb_t* txn_db); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_release_snapshot( + rocksdb_transactiondb_t* txn_db, const rocksdb_snapshot_t* snapshot); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transactiondb_property_value( + rocksdb_transactiondb_t* db, const char* propname); + +extern ROCKSDB_LIBRARY_API int rocksdb_transactiondb_property_int( + rocksdb_transactiondb_t* db, const char* propname, uint64_t* out_val); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_t* rocksdb_transaction_begin( + rocksdb_transactiondb_t* txn_db, + const rocksdb_writeoptions_t* write_options, + const rocksdb_transaction_options_t* txn_options, + rocksdb_transaction_t* old_txn); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_t** +rocksdb_transactiondb_get_prepared_transactions(rocksdb_transactiondb_t* txn_db, + size_t* cnt); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_set_name( + rocksdb_transaction_t* txn, const char* name, size_t name_len, + char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_name( + rocksdb_transaction_t* txn, size_t* name_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_prepare( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_commit( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rollback( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_set_savepoint( + rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rollback_to_savepoint( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_destroy( + rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_wi_t* +rocksdb_transaction_get_writebatch_wi(rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rebuild_from_writebatch( + rocksdb_transaction_t* txn, rocksdb_writebatch_t* writebatch, + char** errptr); + +// This rocksdb_writebatch_wi_t should be freed with rocksdb_free +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rebuild_from_writebatch_wi( + rocksdb_transaction_t* txn, rocksdb_writebatch_wi_t* wi, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_set_commit_timestamp( + rocksdb_transaction_t* txn, uint64_t commit_timestamp); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transaction_set_read_timestamp_for_validation( + rocksdb_transaction_t* txn, uint64_t read_timestamp); + +// This snapshot should be freed using rocksdb_free +extern ROCKSDB_LIBRARY_API const rocksdb_snapshot_t* +rocksdb_transaction_get_snapshot(rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const char* key, size_t klen, size_t* vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + size_t* vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned_cf(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_for_update( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const char* key, size_t klen, size_t* vlen, unsigned char exclusive, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned_for_update(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, + unsigned char exclusive, + char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_for_update_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + size_t* vlen, unsigned char exclusive, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned_for_update_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + unsigned char exclusive, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_multi_get( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_multi_get_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transactiondb_get( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + const char* key, size_t klen, size_t* vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transactiondb_get_pinned(rocksdb_transactiondb_t* txn_db, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transactiondb_get_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transactiondb_get_pinned_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_multi_get( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_multi_get_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_put( + rocksdb_transaction_t* txn, const char* key, size_t klen, const char* val, + size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_put_cf( + rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_put( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_put_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_write( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_writebatch_t* batch, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_merge( + rocksdb_transaction_t* txn, const char* key, size_t klen, const char* val, + size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_merge_cf( + rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_merge( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_merge_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_delete( + rocksdb_transaction_t* txn, const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_delete_cf( + rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_delete( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_delete_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transaction_create_iterator(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transaction_create_iterator_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transactiondb_create_iterator(rocksdb_transactiondb_t* txn_db, + const rocksdb_readoptions_t* options); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transactiondb_create_iterator_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_close( + rocksdb_transactiondb_t* txn_db); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush( + rocksdb_transactiondb_t* txn_db, const rocksdb_flushoptions_t* options, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t* column_family, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush_cfs( + rocksdb_transactiondb_t* txn_db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t** column_families, int num_column_families, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush_wal( + rocksdb_transactiondb_t* txn_db, unsigned char sync, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_checkpoint_t* +rocksdb_transactiondb_checkpoint_object_create(rocksdb_transactiondb_t* txn_db, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_optimistictransactiondb_t* +rocksdb_optimistictransactiondb_open(const rocksdb_options_t* options, + const char* name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_optimistictransactiondb_t* +rocksdb_optimistictransactiondb_open_column_families( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* +rocksdb_optimistictransactiondb_get_base_db( + rocksdb_optimistictransactiondb_t* otxn_db); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransactiondb_close_base_db( + rocksdb_t* base_db); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_t* +rocksdb_optimistictransaction_begin( + rocksdb_optimistictransactiondb_t* otxn_db, + const rocksdb_writeoptions_t* write_options, + const rocksdb_optimistictransaction_options_t* otxn_options, + rocksdb_transaction_t* old_txn); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransactiondb_write( + rocksdb_optimistictransactiondb_t* otxn_db, + const rocksdb_writeoptions_t* options, rocksdb_writebatch_t* batch, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransactiondb_close( + rocksdb_optimistictransactiondb_t* otxn_db); + +extern ROCKSDB_LIBRARY_API rocksdb_checkpoint_t* +rocksdb_optimistictransactiondb_checkpoint_object_create( + rocksdb_optimistictransactiondb_t* otxn_db, char** errptr); + +/* Transaction Options */ + +extern ROCKSDB_LIBRARY_API rocksdb_transactiondb_options_t* +rocksdb_transactiondb_options_create(void); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_options_destroy( + rocksdb_transactiondb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_options_set_max_num_locks( + rocksdb_transactiondb_options_t* opt, int64_t max_num_locks); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_options_set_num_stripes( + rocksdb_transactiondb_options_t* opt, size_t num_stripes); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transactiondb_options_set_transaction_lock_timeout( + rocksdb_transactiondb_options_t* opt, int64_t txn_lock_timeout); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transactiondb_options_set_default_lock_timeout( + rocksdb_transactiondb_options_t* opt, int64_t default_lock_timeout); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_options_t* +rocksdb_transaction_options_create(void); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_destroy( + rocksdb_transaction_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_set_snapshot( + rocksdb_transaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_deadlock_detect( + rocksdb_transaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_lock_timeout( + rocksdb_transaction_options_t* opt, int64_t lock_timeout); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_expiration( + rocksdb_transaction_options_t* opt, int64_t expiration); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transaction_options_set_deadlock_detect_depth( + rocksdb_transaction_options_t* opt, int64_t depth); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transaction_options_set_max_write_batch_size( + rocksdb_transaction_options_t* opt, size_t size); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_skip_prepare( + rocksdb_transaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API rocksdb_optimistictransaction_options_t* +rocksdb_optimistictransaction_options_create(void); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransaction_options_destroy( + rocksdb_optimistictransaction_options_t* opt); + +extern ROCKSDB_LIBRARY_API void +rocksdb_optimistictransaction_options_set_set_snapshot( + rocksdb_optimistictransaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API char* rocksdb_optimistictransactiondb_property_value( + rocksdb_optimistictransactiondb_t* db, const char* propname); + +extern ROCKSDB_LIBRARY_API int rocksdb_optimistictransactiondb_property_int( + rocksdb_optimistictransactiondb_t* db, const char* propname, + uint64_t* out_val); + +// referring to convention (3), this should be used by client +// to free memory that was malloc()ed +extern ROCKSDB_LIBRARY_API void rocksdb_free(void* ptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* rocksdb_get_pinned( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t keylen, char** errptr); +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* rocksdb_get_pinned_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_pinnableslice_destroy( + rocksdb_pinnableslice_t* v); +extern ROCKSDB_LIBRARY_API const char* rocksdb_pinnableslice_value( + const rocksdb_pinnableslice_t* t, size_t* vlen); + +extern ROCKSDB_LIBRARY_API rocksdb_memory_consumers_t* +rocksdb_memory_consumers_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_add_db( + rocksdb_memory_consumers_t* consumers, rocksdb_t* db); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_add_cache( + rocksdb_memory_consumers_t* consumers, rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_destroy( + rocksdb_memory_consumers_t* consumers); +extern ROCKSDB_LIBRARY_API rocksdb_memory_usage_t* +rocksdb_approximate_memory_usage_create(rocksdb_memory_consumers_t* consumers, + char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_approximate_memory_usage_destroy( + rocksdb_memory_usage_t* usage); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_mem_table_total( + rocksdb_memory_usage_t* memory_usage); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_mem_table_unflushed( + rocksdb_memory_usage_t* memory_usage); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_mem_table_readers_total( + rocksdb_memory_usage_t* memory_usage); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_cache_total( + rocksdb_memory_usage_t* memory_usage); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_dump_malloc_stats( + rocksdb_options_t*, unsigned char); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_memtable_whole_key_filtering(rocksdb_options_t*, + unsigned char); + +extern ROCKSDB_LIBRARY_API void rocksdb_cancel_all_background_work( + rocksdb_t* db, unsigned char wait); + +extern ROCKSDB_LIBRARY_API void rocksdb_disable_manual_compaction( + rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_enable_manual_compaction(rocksdb_t* db); + +#ifdef __cplusplus +} /* end extern "C" */ +#endif diff --git a/deps/mips/rocksdb_static/librocksdb.a b/deps/mips/rocksdb_static/librocksdb.a new file mode 100644 index 0000000000000000000000000000000000000000..6df885a0685006baaede75c70b7be0c410115439 Binary files /dev/null and b/deps/mips/rocksdb_static/librocksdb.a differ diff --git a/deps/mips/rocksdb_static/rocksdb/c.h b/deps/mips/rocksdb_static/rocksdb/c.h new file mode 100644 index 0000000000000000000000000000000000000000..1ba7fabefe92f674ecbd27a91bfbb57b0ed17d23 --- /dev/null +++ b/deps/mips/rocksdb_static/rocksdb/c.h @@ -0,0 +1,2844 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +/* Copyright (c) 2011 The LevelDB Authors. All rights reserved. + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. See the AUTHORS file for names of contributors. + + C bindings for rocksdb. May be useful as a stable ABI that can be + used by programs that keep rocksdb in a shared library, or for + a JNI api. + + Does not support: + . getters for the option types + . custom comparators that implement key shortening + . capturing post-write-snapshot + . custom iter, db, env, cache implementations using just the C bindings + + Some conventions: + + (1) We expose just opaque struct pointers and functions to clients. + This allows us to change internal representations without having to + recompile clients. + + (2) For simplicity, there is no equivalent to the Slice type. Instead, + the caller has to pass the pointer and length as separate + arguments. + + (3) Errors are represented by a null-terminated c string. NULL + means no error. All operations that can raise an error are passed + a "char** errptr" as the last argument. One of the following must + be true on entry: + *errptr == NULL + *errptr points to a malloc()ed null-terminated error message + On success, a leveldb routine leaves *errptr unchanged. + On failure, leveldb frees the old value of *errptr and + set *errptr to a malloc()ed error message. + + (4) Bools have the type unsigned char (0 == false; rest == true) + + (5) All of the pointer arguments must be non-NULL. +*/ + +#pragma once + +#ifdef _WIN32 +#ifdef ROCKSDB_DLL +#ifdef ROCKSDB_LIBRARY_EXPORTS +#define ROCKSDB_LIBRARY_API __declspec(dllexport) +#else +#define ROCKSDB_LIBRARY_API __declspec(dllimport) +#endif +#else +#define ROCKSDB_LIBRARY_API +#endif +#else +#define ROCKSDB_LIBRARY_API +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include + +/* Exported types */ + +typedef struct rocksdb_t rocksdb_t; +typedef struct rocksdb_backup_engine_t rocksdb_backup_engine_t; +typedef struct rocksdb_backup_engine_info_t rocksdb_backup_engine_info_t; +typedef struct rocksdb_backup_engine_options_t rocksdb_backup_engine_options_t; +typedef struct rocksdb_restore_options_t rocksdb_restore_options_t; +typedef struct rocksdb_memory_allocator_t rocksdb_memory_allocator_t; +typedef struct rocksdb_lru_cache_options_t rocksdb_lru_cache_options_t; +typedef struct rocksdb_hyper_clock_cache_options_t + rocksdb_hyper_clock_cache_options_t; +typedef struct rocksdb_cache_t rocksdb_cache_t; +typedef struct rocksdb_compactionfilter_t rocksdb_compactionfilter_t; +typedef struct rocksdb_compactionfiltercontext_t + rocksdb_compactionfiltercontext_t; +typedef struct rocksdb_compactionfilterfactory_t + rocksdb_compactionfilterfactory_t; +typedef struct rocksdb_comparator_t rocksdb_comparator_t; +typedef struct rocksdb_dbpath_t rocksdb_dbpath_t; +typedef struct rocksdb_env_t rocksdb_env_t; +typedef struct rocksdb_fifo_compaction_options_t + rocksdb_fifo_compaction_options_t; +typedef struct rocksdb_filelock_t rocksdb_filelock_t; +typedef struct rocksdb_filterpolicy_t rocksdb_filterpolicy_t; +typedef struct rocksdb_flushoptions_t rocksdb_flushoptions_t; +typedef struct rocksdb_iterator_t rocksdb_iterator_t; +typedef struct rocksdb_logger_t rocksdb_logger_t; +typedef struct rocksdb_mergeoperator_t rocksdb_mergeoperator_t; +typedef struct rocksdb_options_t rocksdb_options_t; +typedef struct rocksdb_compactoptions_t rocksdb_compactoptions_t; +typedef struct rocksdb_block_based_table_options_t + rocksdb_block_based_table_options_t; +typedef struct rocksdb_cuckoo_table_options_t rocksdb_cuckoo_table_options_t; +typedef struct rocksdb_randomfile_t rocksdb_randomfile_t; +typedef struct rocksdb_readoptions_t rocksdb_readoptions_t; +typedef struct rocksdb_seqfile_t rocksdb_seqfile_t; +typedef struct rocksdb_slicetransform_t rocksdb_slicetransform_t; +typedef struct rocksdb_snapshot_t rocksdb_snapshot_t; +typedef struct rocksdb_writablefile_t rocksdb_writablefile_t; +typedef struct rocksdb_writebatch_t rocksdb_writebatch_t; +typedef struct rocksdb_writebatch_wi_t rocksdb_writebatch_wi_t; +typedef struct rocksdb_writeoptions_t rocksdb_writeoptions_t; +typedef struct rocksdb_universal_compaction_options_t + rocksdb_universal_compaction_options_t; +typedef struct rocksdb_livefiles_t rocksdb_livefiles_t; +typedef struct rocksdb_column_family_handle_t rocksdb_column_family_handle_t; +typedef struct rocksdb_column_family_metadata_t + rocksdb_column_family_metadata_t; +typedef struct rocksdb_level_metadata_t rocksdb_level_metadata_t; +typedef struct rocksdb_sst_file_metadata_t rocksdb_sst_file_metadata_t; +typedef struct rocksdb_envoptions_t rocksdb_envoptions_t; +typedef struct rocksdb_ingestexternalfileoptions_t + rocksdb_ingestexternalfileoptions_t; +typedef struct rocksdb_sstfilewriter_t rocksdb_sstfilewriter_t; +typedef struct rocksdb_ratelimiter_t rocksdb_ratelimiter_t; +typedef struct rocksdb_perfcontext_t rocksdb_perfcontext_t; +typedef struct rocksdb_pinnableslice_t rocksdb_pinnableslice_t; +typedef struct rocksdb_transactiondb_options_t rocksdb_transactiondb_options_t; +typedef struct rocksdb_transactiondb_t rocksdb_transactiondb_t; +typedef struct rocksdb_transaction_options_t rocksdb_transaction_options_t; +typedef struct rocksdb_optimistictransactiondb_t + rocksdb_optimistictransactiondb_t; +typedef struct rocksdb_optimistictransaction_options_t + rocksdb_optimistictransaction_options_t; +typedef struct rocksdb_transaction_t rocksdb_transaction_t; +typedef struct rocksdb_checkpoint_t rocksdb_checkpoint_t; +typedef struct rocksdb_wal_iterator_t rocksdb_wal_iterator_t; +typedef struct rocksdb_wal_readoptions_t rocksdb_wal_readoptions_t; +typedef struct rocksdb_memory_consumers_t rocksdb_memory_consumers_t; +typedef struct rocksdb_memory_usage_t rocksdb_memory_usage_t; + +/* DB operations */ + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open( + const rocksdb_options_t* options, const char* name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_with_ttl( + const rocksdb_options_t* options, const char* name, int ttl, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_for_read_only( + const rocksdb_options_t* options, const char* name, + unsigned char error_if_wal_file_exists, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_as_secondary( + const rocksdb_options_t* options, const char* name, + const char* secondary_path, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_backup_engine_t* rocksdb_backup_engine_open( + const rocksdb_options_t* options, const char* path, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_backup_engine_t* +rocksdb_backup_engine_open_opts(const rocksdb_backup_engine_options_t* options, + rocksdb_env_t* env, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_create_new_backup( + rocksdb_backup_engine_t* be, rocksdb_t* db, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_create_new_backup_flush( + rocksdb_backup_engine_t* be, rocksdb_t* db, + unsigned char flush_before_backup, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_purge_old_backups( + rocksdb_backup_engine_t* be, uint32_t num_backups_to_keep, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_restore_options_t* +rocksdb_restore_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_restore_options_destroy( + rocksdb_restore_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_restore_options_set_keep_log_files( + rocksdb_restore_options_t* opt, int v); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_verify_backup( + rocksdb_backup_engine_t* be, uint32_t backup_id, char** errptr); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_restore_db_from_latest_backup( + rocksdb_backup_engine_t* be, const char* db_dir, const char* wal_dir, + const rocksdb_restore_options_t* restore_options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_restore_db_from_backup( + rocksdb_backup_engine_t* be, const char* db_dir, const char* wal_dir, + const rocksdb_restore_options_t* restore_options, const uint32_t backup_id, + char** errptr); + +extern ROCKSDB_LIBRARY_API const rocksdb_backup_engine_info_t* +rocksdb_backup_engine_get_backup_info(rocksdb_backup_engine_t* be); + +extern ROCKSDB_LIBRARY_API int rocksdb_backup_engine_info_count( + const rocksdb_backup_engine_info_t* info); + +extern ROCKSDB_LIBRARY_API int64_t rocksdb_backup_engine_info_timestamp( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API uint32_t rocksdb_backup_engine_info_backup_id( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API uint64_t rocksdb_backup_engine_info_size( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API uint32_t rocksdb_backup_engine_info_number_files( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_info_destroy( + const rocksdb_backup_engine_info_t* info); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_close( + rocksdb_backup_engine_t* be); + +extern ROCKSDB_LIBRARY_API void rocksdb_put_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* ts, size_t tslen, const char* val, size_t vallen, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_put_cf_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* ts, size_t tslen, const char* val, size_t vallen, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_cf_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete_cf_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_increase_full_history_ts_low( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* ts_low, size_t ts_lowlen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_full_history_ts_low( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + size_t* ts_lowlen, char** errptr); + +/* BackupEngineOptions */ + +extern ROCKSDB_LIBRARY_API rocksdb_backup_engine_options_t* +rocksdb_backup_engine_options_create(const char* backup_dir); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_set_backup_dir( + rocksdb_backup_engine_options_t* options, const char* backup_dir); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_set_env( + rocksdb_backup_engine_options_t* options, rocksdb_env_t* env); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_share_table_files( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_backup_engine_options_get_share_table_files( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_set_sync( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_backup_engine_options_get_sync( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_destroy_old_data( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_backup_engine_options_get_destroy_old_data( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_backup_log_files( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_backup_engine_options_get_backup_log_files( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_backup_rate_limit( + rocksdb_backup_engine_options_t* options, uint64_t limit); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_backup_engine_options_get_backup_rate_limit( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_restore_rate_limit( + rocksdb_backup_engine_options_t* options, uint64_t limit); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_backup_engine_options_get_restore_rate_limit( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_max_background_operations( + rocksdb_backup_engine_options_t* options, int val); + +extern ROCKSDB_LIBRARY_API int +rocksdb_backup_engine_options_get_max_background_operations( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_callback_trigger_interval_size( + rocksdb_backup_engine_options_t* options, uint64_t size); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_backup_engine_options_get_callback_trigger_interval_size( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_max_valid_backups_to_open( + rocksdb_backup_engine_options_t* options, int val); + +extern ROCKSDB_LIBRARY_API int +rocksdb_backup_engine_options_get_max_valid_backups_to_open( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_share_files_with_checksum_naming( + rocksdb_backup_engine_options_t* options, int val); + +extern ROCKSDB_LIBRARY_API int +rocksdb_backup_engine_options_get_share_files_with_checksum_naming( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_destroy( + rocksdb_backup_engine_options_t*); + +/* Checkpoint */ + +extern ROCKSDB_LIBRARY_API rocksdb_checkpoint_t* +rocksdb_checkpoint_object_create(rocksdb_t* db, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_checkpoint_create( + rocksdb_checkpoint_t* checkpoint, const char* checkpoint_dir, + uint64_t log_size_for_flush, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_checkpoint_object_destroy( + rocksdb_checkpoint_t* checkpoint); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_and_trim_history( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char* trim_ts, + size_t trim_tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_column_families( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_column_families_with_ttl( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, const int* ttls, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* +rocksdb_open_for_read_only_column_families( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, + unsigned char error_if_wal_file_exists, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_as_secondary_column_families( + const rocksdb_options_t* options, const char* name, + const char* secondary_path, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API char** rocksdb_list_column_families( + const rocksdb_options_t* options, const char* name, size_t* lencf, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_list_column_families_destroy( + char** list, size_t len); + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* +rocksdb_create_column_family(rocksdb_t* db, + const rocksdb_options_t* column_family_options, + const char* column_family_name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* +rocksdb_create_column_family_with_ttl( + rocksdb_t* db, const rocksdb_options_t* column_family_options, + const char* column_family_name, int ttl, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_drop_column_family( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_column_family_handle_destroy( + rocksdb_column_family_handle_t*); + +extern ROCKSDB_LIBRARY_API uint32_t +rocksdb_column_family_handle_get_id(rocksdb_column_family_handle_t* handle); + +extern ROCKSDB_LIBRARY_API char* rocksdb_column_family_handle_get_name( + rocksdb_column_family_handle_t* handle, size_t* name_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_close(rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_put( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_put_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_range_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* start_key, + size_t start_key_len, const char* end_key, size_t end_key_len, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_merge( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_merge_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_write( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_writebatch_t* batch, char** errptr); + +/* Returns NULL if not found. A malloc()ed array otherwise. + Stores the length of the array in *vallen. */ +extern ROCKSDB_LIBRARY_API char* rocksdb_get( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t keylen, size_t* vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t keylen, size_t* vallen, char** ts, size_t* tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_cf_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** ts, size_t* tslen, char** errptr); + +// if values_list[i] == NULL and errs[i] == NULL, +// then we got status.IsNotFound(), which we will not return. +// all errors except status status.ok() and status.IsNotFound() are returned. +// +// errs, values_list and values_list_sizes must be num_keys in length, +// allocated by the caller. +// errs is a list of strings as opposed to the conventional one error, +// where errs[i] is the status for retrieval of keys_list[i]. +// each non-NULL errs entry is a malloc()ed, null terminated string. +// each non-NULL values_list entry is a malloc()ed array, with +// the length for each stored in values_list_sizes[i]. +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get( + rocksdb_t* db, const rocksdb_readoptions_t* options, size_t num_keys, + const char* const* keys_list, const size_t* keys_list_sizes, + char** values_list, size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, size_t num_keys, + const char* const* keys_list, const size_t* keys_list_sizes, + char** values_list, size_t* values_list_sizes, char** timestamp_list, + size_t* timestamp_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_cf_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** timestamps_list, + size_t* timestamps_list_sizes, char** errs); + +// The MultiGet API that improves performance by batching operations +// in the read path for greater efficiency. Currently, only the block based +// table format with full filters are supported. Other table formats such +// as plain table, block based table with block based filters and +// partitioned indexes will still work, but will not get any performance +// benefits. +// +// Note that all the keys passed to this API are restricted to a single +// column family. +// +// Parameters - +// db - the RocksDB instance. +// options - ReadOptions +// column_family - ColumnFamilyHandle* that the keys belong to. All the keys +// passed to the API are restricted to a single column family +// num_keys - Number of keys to lookup +// keys_list - Pointer to C style array of keys with num_keys elements +// keys_list_sizes - Pointer to C style array of the size of corresponding key +// in key_list with num_keys elements. +// values - Pointer to C style array of PinnableSlices with num_keys elements +// statuses - Pointer to C style array of Status with num_keys elements +// sorted_input - If true, it means the input keys are already sorted by key +// order, so the MultiGet() API doesn't have to sort them +// again. If false, the keys will be copied and sorted +// internally by the API - the input array will not be +// modified +extern ROCKSDB_LIBRARY_API void rocksdb_batched_multi_get_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, size_t num_keys, + const char* const* keys_list, const size_t* keys_list_sizes, + rocksdb_pinnableslice_t** values, char** errs, const bool sorted_input); + +// The value is only allocated (using malloc) and returned if it is found and +// value_found isn't NULL. In that case the user is responsible for freeing it. +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_key_may_exist( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t key_len, char** value, size_t* val_len, const char* timestamp, + size_t timestamp_len, unsigned char* value_found); + +// The value is only allocated (using malloc) and returned if it is found and +// value_found isn't NULL. In that case the user is responsible for freeing it. +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_key_may_exist_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t key_len, char** value, size_t* val_len, const char* timestamp, + size_t timestamp_len, unsigned char* value_found); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* rocksdb_create_iterator( + rocksdb_t* db, const rocksdb_readoptions_t* options); + +extern ROCKSDB_LIBRARY_API rocksdb_wal_iterator_t* rocksdb_get_updates_since( + rocksdb_t* db, uint64_t seq_number, + const rocksdb_wal_readoptions_t* options, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* rocksdb_create_iterator_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API void rocksdb_create_iterators( + rocksdb_t* db, rocksdb_readoptions_t* opts, + rocksdb_column_family_handle_t** column_families, + rocksdb_iterator_t** iterators, size_t size, char** errptr); + +extern ROCKSDB_LIBRARY_API const rocksdb_snapshot_t* rocksdb_create_snapshot( + rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_release_snapshot( + rocksdb_t* db, const rocksdb_snapshot_t* snapshot); + +/* Returns NULL if property name is unknown. + Else returns a pointer to a malloc()-ed null-terminated value. */ +extern ROCKSDB_LIBRARY_API char* rocksdb_property_value(rocksdb_t* db, + const char* propname); +/* returns 0 on success, -1 otherwise */ +extern ROCKSDB_LIBRARY_API int rocksdb_property_int(rocksdb_t* db, + const char* propname, + uint64_t* out_val); + +/* returns 0 on success, -1 otherwise */ +extern ROCKSDB_LIBRARY_API int rocksdb_property_int_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* propname, uint64_t* out_val); + +extern ROCKSDB_LIBRARY_API char* rocksdb_property_value_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* propname); + +extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes( + rocksdb_t* db, int num_ranges, const char* const* range_start_key, + const size_t* range_start_key_len, const char* const* range_limit_key, + const size_t* range_limit_key_len, uint64_t* sizes, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + int num_ranges, const char* const* range_start_key, + const size_t* range_start_key_len, const char* const* range_limit_key, + const size_t* range_limit_key_len, uint64_t* sizes, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range(rocksdb_t* db, + const char* start_key, + size_t start_key_len, + const char* limit_key, + size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* limit_key, + size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_suggest_compact_range( + rocksdb_t* db, const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_suggest_compact_range_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* limit_key, + size_t limit_key_len, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range_opt( + rocksdb_t* db, rocksdb_compactoptions_t* opt, const char* start_key, + size_t start_key_len, const char* limit_key, size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range_cf_opt( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + rocksdb_compactoptions_t* opt, const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_file(rocksdb_t* db, + const char* name); + +extern ROCKSDB_LIBRARY_API const rocksdb_livefiles_t* rocksdb_livefiles( + rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush( + rocksdb_t* db, const rocksdb_flushoptions_t* options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush_cf( + rocksdb_t* db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t* column_family, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush_cfs( + rocksdb_t* db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t** column_family, int num_column_families, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush_wal(rocksdb_t* db, + unsigned char sync, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_disable_file_deletions(rocksdb_t* db, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_enable_file_deletions( + rocksdb_t* db, unsigned char force, char** errptr); + +/* Management operations */ + +extern ROCKSDB_LIBRARY_API void rocksdb_destroy_db( + const rocksdb_options_t* options, const char* name, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_repair_db( + const rocksdb_options_t* options, const char* name, char** errptr); + +/* Iterator */ + +extern ROCKSDB_LIBRARY_API void rocksdb_iter_destroy(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_iter_valid( + const rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek_to_first(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek_to_last(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek(rocksdb_iterator_t*, + const char* k, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek_for_prev(rocksdb_iterator_t*, + const char* k, + size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_next(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_prev(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API const char* rocksdb_iter_key( + const rocksdb_iterator_t*, size_t* klen); +extern ROCKSDB_LIBRARY_API const char* rocksdb_iter_value( + const rocksdb_iterator_t*, size_t* vlen); +extern ROCKSDB_LIBRARY_API const char* rocksdb_iter_timestamp( + const rocksdb_iterator_t*, size_t* tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_get_error( + const rocksdb_iterator_t*, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_wal_iter_next( + rocksdb_wal_iterator_t* iter); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_wal_iter_valid( + const rocksdb_wal_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_wal_iter_status( + const rocksdb_wal_iterator_t* iter, char** errptr); +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* rocksdb_wal_iter_get_batch( + const rocksdb_wal_iterator_t* iter, uint64_t* seq); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_get_latest_sequence_number(rocksdb_t* db); +extern ROCKSDB_LIBRARY_API void rocksdb_wal_iter_destroy( + const rocksdb_wal_iterator_t* iter); + +/* Write batch */ + +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* rocksdb_writebatch_create( + void); +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* rocksdb_writebatch_create_from( + const char* rep, size_t size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_destroy( + rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_clear(rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API int rocksdb_writebatch_count(rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put(rocksdb_writebatch_t*, + const char* key, + size_t klen, + const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put_cf( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put_cf_with_ts( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen, const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_putv( + rocksdb_writebatch_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_putv_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_merge(rocksdb_writebatch_t*, + const char* key, + size_t klen, + const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_merge_cf( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_mergev( + rocksdb_writebatch_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_mergev_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete(rocksdb_writebatch_t*, + const char* key, + size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_singledelete( + rocksdb_writebatch_t* b, const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_cf( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_cf_with_ts( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_singledelete_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_singledelete_cf_with_ts( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev( + rocksdb_writebatch_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_range( + rocksdb_writebatch_t* b, const char* start_key, size_t start_key_len, + const char* end_key, size_t end_key_len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_range_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* end_key, + size_t end_key_len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_rangev( + rocksdb_writebatch_t* b, int num_keys, const char* const* start_keys_list, + const size_t* start_keys_list_sizes, const char* const* end_keys_list, + const size_t* end_keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_rangev_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* start_keys_list, + const size_t* start_keys_list_sizes, const char* const* end_keys_list, + const size_t* end_keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put_log_data( + rocksdb_writebatch_t*, const char* blob, size_t len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_iterate( + rocksdb_writebatch_t*, void* state, + void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), + void (*deleted)(void*, const char* k, size_t klen)); +extern ROCKSDB_LIBRARY_API const char* rocksdb_writebatch_data( + rocksdb_writebatch_t*, size_t* size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_set_save_point( + rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_rollback_to_save_point( + rocksdb_writebatch_t*, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_pop_save_point( + rocksdb_writebatch_t*, char** errptr); + +/* Write batch with index */ + +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_wi_t* +rocksdb_writebatch_wi_create(size_t reserved_bytes, + unsigned char overwrite_keys); +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_wi_t* +rocksdb_writebatch_wi_create_from(const char* rep, size_t size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_destroy( + rocksdb_writebatch_wi_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_clear( + rocksdb_writebatch_wi_t*); +extern ROCKSDB_LIBRARY_API int rocksdb_writebatch_wi_count( + rocksdb_writebatch_wi_t* b); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_put( + rocksdb_writebatch_wi_t*, const char* key, size_t klen, const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_put_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_putv( + rocksdb_writebatch_wi_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_putv_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_merge( + rocksdb_writebatch_wi_t*, const char* key, size_t klen, const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_merge_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_mergev( + rocksdb_writebatch_wi_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_mergev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete( + rocksdb_writebatch_wi_t*, const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_singledelete( + rocksdb_writebatch_wi_t*, const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_singledelete_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_deletev( + rocksdb_writebatch_wi_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_deletev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes); +// DO NOT USE - rocksdb_writebatch_wi_delete_range is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_range( + rocksdb_writebatch_wi_t* b, const char* start_key, size_t start_key_len, + const char* end_key, size_t end_key_len); +// DO NOT USE - rocksdb_writebatch_wi_delete_range_cf is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_range_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* end_key, + size_t end_key_len); +// DO NOT USE - rocksdb_writebatch_wi_delete_rangev is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_rangev( + rocksdb_writebatch_wi_t* b, int num_keys, + const char* const* start_keys_list, const size_t* start_keys_list_sizes, + const char* const* end_keys_list, const size_t* end_keys_list_sizes); +// DO NOT USE - rocksdb_writebatch_wi_delete_rangev_cf is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_rangev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* start_keys_list, + const size_t* start_keys_list_sizes, const char* const* end_keys_list, + const size_t* end_keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_put_log_data( + rocksdb_writebatch_wi_t*, const char* blob, size_t len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_iterate( + rocksdb_writebatch_wi_t* b, void* state, + void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), + void (*deleted)(void*, const char* k, size_t klen)); +extern ROCKSDB_LIBRARY_API const char* rocksdb_writebatch_wi_data( + rocksdb_writebatch_wi_t* b, size_t* size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_set_save_point( + rocksdb_writebatch_wi_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_rollback_to_save_point( + rocksdb_writebatch_wi_t*, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch( + rocksdb_writebatch_wi_t* wbwi, const rocksdb_options_t* options, + const char* key, size_t keylen, size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch_cf( + rocksdb_writebatch_wi_t* wbwi, const rocksdb_options_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch_and_db( + rocksdb_writebatch_wi_t* wbwi, rocksdb_t* db, + const rocksdb_readoptions_t* options, const char* key, size_t keylen, + size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch_and_db_cf( + rocksdb_writebatch_wi_t* wbwi, rocksdb_t* db, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_write_writebatch_wi( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_writebatch_wi_t* wbwi, char** errptr); +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_writebatch_wi_create_iterator_with_base( + rocksdb_writebatch_wi_t* wbwi, rocksdb_iterator_t* base_iterator); +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_writebatch_wi_create_iterator_with_base_cf( + rocksdb_writebatch_wi_t* wbwi, rocksdb_iterator_t* base_iterator, + rocksdb_column_family_handle_t* cf); + +/* Options utils */ + +// Load the latest rocksdb options from the specified db_path. +// +// On success, num_column_families will be updated with a non-zero +// number indicating the number of column families. +// The returned db_options, column_family_names, and column_family_options +// should be released via rocksdb_load_latest_options_destroy(). +// +// On error, a non-null errptr that includes the error message will be +// returned. db_options, column_family_names, and column_family_options +// will be set to NULL. +extern ROCKSDB_LIBRARY_API void rocksdb_load_latest_options( + const char* db_path, rocksdb_env_t* env, bool ignore_unknown_options, + rocksdb_cache_t* cache, rocksdb_options_t** db_options, + size_t* num_column_families, char*** column_family_names, + rocksdb_options_t*** column_family_options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_load_latest_options_destroy( + rocksdb_options_t* db_options, char** list_column_family_names, + rocksdb_options_t** list_column_family_options, size_t len); + +/* Block based table options */ + +extern ROCKSDB_LIBRARY_API rocksdb_block_based_table_options_t* +rocksdb_block_based_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_destroy( + rocksdb_block_based_table_options_t* options); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_checksum( + rocksdb_block_based_table_options_t*, char); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_block_size( + rocksdb_block_based_table_options_t* options, size_t block_size); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_block_size_deviation( + rocksdb_block_based_table_options_t* options, int block_size_deviation); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_block_restart_interval( + rocksdb_block_based_table_options_t* options, int block_restart_interval); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_index_block_restart_interval( + rocksdb_block_based_table_options_t* options, + int index_block_restart_interval); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_metadata_block_size( + rocksdb_block_based_table_options_t* options, uint64_t metadata_block_size); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_partition_filters( + rocksdb_block_based_table_options_t* options, + unsigned char partition_filters); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_optimize_filters_for_memory( + rocksdb_block_based_table_options_t* options, + unsigned char optimize_filters_for_memory); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_use_delta_encoding( + rocksdb_block_based_table_options_t* options, + unsigned char use_delta_encoding); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_filter_policy( + rocksdb_block_based_table_options_t* options, + rocksdb_filterpolicy_t* filter_policy); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_no_block_cache( + rocksdb_block_based_table_options_t* options, unsigned char no_block_cache); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_block_cache( + rocksdb_block_based_table_options_t* options, rocksdb_cache_t* block_cache); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_whole_key_filtering( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_format_version( + rocksdb_block_based_table_options_t*, int); +enum { + rocksdb_block_based_table_index_type_binary_search = 0, + rocksdb_block_based_table_index_type_hash_search = 1, + rocksdb_block_based_table_index_type_two_level_index_search = 2, +}; +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_index_type( + rocksdb_block_based_table_options_t*, int); // uses one of the above enums +enum { + rocksdb_block_based_table_data_block_index_type_binary_search = 0, + rocksdb_block_based_table_data_block_index_type_binary_search_and_hash = 1, +}; +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_data_block_index_type( + rocksdb_block_based_table_options_t*, int); // uses one of the above enums +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_data_block_hash_ratio( + rocksdb_block_based_table_options_t* options, double v); +// rocksdb_block_based_options_set_hash_index_allow_collision() +// is removed since BlockBasedTableOptions.hash_index_allow_collision() +// is removed +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_cache_index_and_filter_blocks( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_pin_top_level_index_and_filter( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_block_based_table_factory( + rocksdb_options_t* opt, rocksdb_block_based_table_options_t* table_options); + +/* Cuckoo table options */ + +extern ROCKSDB_LIBRARY_API rocksdb_cuckoo_table_options_t* +rocksdb_cuckoo_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_destroy( + rocksdb_cuckoo_table_options_t* options); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_hash_ratio( + rocksdb_cuckoo_table_options_t* options, double v); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_max_search_depth( + rocksdb_cuckoo_table_options_t* options, uint32_t v); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_cuckoo_block_size( + rocksdb_cuckoo_table_options_t* options, uint32_t v); +extern ROCKSDB_LIBRARY_API void +rocksdb_cuckoo_options_set_identity_as_first_hash( + rocksdb_cuckoo_table_options_t* options, unsigned char v); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_use_module_hash( + rocksdb_cuckoo_table_options_t* options, unsigned char v); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_cuckoo_table_factory( + rocksdb_options_t* opt, rocksdb_cuckoo_table_options_t* table_options); + +/* Options */ +extern ROCKSDB_LIBRARY_API void rocksdb_set_options(rocksdb_t* db, int count, + const char* const keys[], + const char* const values[], + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_set_options_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, int count, + const char* const keys[], const char* const values[], char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_options_t* rocksdb_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_options_destroy(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API rocksdb_options_t* rocksdb_options_create_copy( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_increase_parallelism( + rocksdb_options_t* opt, int total_threads); +extern ROCKSDB_LIBRARY_API void rocksdb_options_optimize_for_point_lookup( + rocksdb_options_t* opt, uint64_t block_cache_size_mb); +extern ROCKSDB_LIBRARY_API void rocksdb_options_optimize_level_style_compaction( + rocksdb_options_t* opt, uint64_t memtable_memory_budget); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_optimize_universal_style_compaction( + rocksdb_options_t* opt, uint64_t memtable_memory_budget); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_allow_ingest_behind( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_allow_ingest_behind(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compaction_filter( + rocksdb_options_t*, rocksdb_compactionfilter_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compaction_filter_factory( + rocksdb_options_t*, rocksdb_compactionfilterfactory_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_compaction_readahead_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_compaction_readahead_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_comparator( + rocksdb_options_t*, rocksdb_comparator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_merge_operator( + rocksdb_options_t*, rocksdb_mergeoperator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_uint64add_merge_operator( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compression_per_level( + rocksdb_options_t* opt, const int* level_values, size_t num_levels); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_create_if_missing( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_create_if_missing( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_create_missing_column_families(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_create_missing_column_families(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_error_if_exists( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_error_if_exists( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_paranoid_checks( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_paranoid_checks( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_db_paths( + rocksdb_options_t*, const rocksdb_dbpath_t** path_values, size_t num_paths); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_env(rocksdb_options_t*, + rocksdb_env_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_info_log(rocksdb_options_t*, + rocksdb_logger_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_info_log_level( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_info_log_level( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_write_buffer_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_write_buffer_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_db_write_buffer_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_db_write_buffer_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_open_files( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_open_files( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_file_opening_threads( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_file_opening_threads( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_total_wal_size( + rocksdb_options_t* opt, uint64_t n); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_total_wal_size(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compression_options( + rocksdb_options_t*, int, int, int, int); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_zstd_max_train_bytes(rocksdb_options_t*, + int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_compression_options_zstd_max_train_bytes( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_use_zstd_dict_trainer( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_compression_options_use_zstd_dict_trainer( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_parallel_threads(rocksdb_options_t*, + int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_compression_options_parallel_threads( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_max_dict_buffer_bytes( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_compression_options_max_dict_buffer_bytes( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options(rocksdb_options_t*, int, int, + int, int, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options_zstd_max_train_bytes( + rocksdb_options_t*, int, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options_use_zstd_dict_trainer( + rocksdb_options_t*, unsigned char, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_bottommost_compression_options_use_zstd_dict_trainer( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options_max_dict_buffer_bytes( + rocksdb_options_t*, uint64_t, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_prefix_extractor( + rocksdb_options_t*, rocksdb_slicetransform_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_num_levels( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_num_levels( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_level0_file_num_compaction_trigger(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_level0_file_num_compaction_trigger(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_level0_slowdown_writes_trigger(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_level0_slowdown_writes_trigger(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_level0_stop_writes_trigger( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_level0_stop_writes_trigger( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_target_file_size_base( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_target_file_size_base(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_target_file_size_multiplier( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_target_file_size_multiplier( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_bytes_for_level_base( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_bytes_for_level_base(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_level_compaction_dynamic_level_bytes(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_level_compaction_dynamic_level_bytes(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_bytes_for_level_multiplier(rocksdb_options_t*, double); +extern ROCKSDB_LIBRARY_API double +rocksdb_options_get_max_bytes_for_level_multiplier(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_bytes_for_level_multiplier_additional( + rocksdb_options_t*, int* level_values, size_t num_levels); +extern ROCKSDB_LIBRARY_API void rocksdb_options_enable_statistics( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt, + unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_skip_stats_update_on_db_open(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open( + rocksdb_options_t* opt, unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open( + rocksdb_options_t* opt); + +/* Blob Options Settings */ +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_blob_files( + rocksdb_options_t* opt, unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_enable_blob_files( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_min_blob_size( + rocksdb_options_t* opt, uint64_t val); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_min_blob_size(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_file_size( + rocksdb_options_t* opt, uint64_t val); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_blob_file_size(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_compression_type( + rocksdb_options_t* opt, int val); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_blob_compression_type( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_blob_gc( + rocksdb_options_t* opt, unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_enable_blob_gc( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_gc_age_cutoff( + rocksdb_options_t* opt, double val); +extern ROCKSDB_LIBRARY_API double rocksdb_options_get_blob_gc_age_cutoff( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_gc_force_threshold( + rocksdb_options_t* opt, double val); +extern ROCKSDB_LIBRARY_API double rocksdb_options_get_blob_gc_force_threshold( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_blob_compaction_readahead_size(rocksdb_options_t* opt, + uint64_t val); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_blob_compaction_readahead_size(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_file_starting_level( + rocksdb_options_t* opt, int val); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_blob_file_starting_level( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_cache( + rocksdb_options_t* opt, rocksdb_cache_t* blob_cache); + +enum { + rocksdb_prepopulate_blob_disable = 0, + rocksdb_prepopulate_blob_flush_only = 1 +}; + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_prepopulate_blob_cache( + rocksdb_options_t* opt, int val); + +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_prepopulate_blob_cache( + rocksdb_options_t* opt); + +/* returns a pointer to a malloc()-ed, null terminated string */ +extern ROCKSDB_LIBRARY_API char* rocksdb_options_statistics_get_string( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_write_buffer_number( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_write_buffer_number( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_min_write_buffer_number_to_merge(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_min_write_buffer_number_to_merge(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_write_buffer_number_to_maintain(rocksdb_options_t*, + int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_max_write_buffer_number_to_maintain(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_write_buffer_size_to_maintain(rocksdb_options_t*, + int64_t); +extern ROCKSDB_LIBRARY_API int64_t +rocksdb_options_get_max_write_buffer_size_to_maintain(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_pipelined_write( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_enable_pipelined_write(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_unordered_write( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_unordered_write( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_subcompactions( + rocksdb_options_t*, uint32_t); +extern ROCKSDB_LIBRARY_API uint32_t +rocksdb_options_get_max_subcompactions(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_background_jobs( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_background_jobs( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_background_compactions( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_background_compactions( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_background_flushes( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_background_flushes( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_log_file_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_max_log_file_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_log_file_time_to_roll( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_log_file_time_to_roll(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_keep_log_file_num( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_keep_log_file_num(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_recycle_log_file_num( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_recycle_log_file_num(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_soft_pending_compaction_bytes_limit(rocksdb_options_t* opt, + size_t v); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_soft_pending_compaction_bytes_limit(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_hard_pending_compaction_bytes_limit(rocksdb_options_t* opt, + size_t v); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_hard_pending_compaction_bytes_limit(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_manifest_file_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_max_manifest_file_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_table_cache_numshardbits( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_table_cache_numshardbits( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_arena_block_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_arena_block_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_use_fsync( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_use_fsync( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_db_log_dir( + rocksdb_options_t*, const char*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_dir(rocksdb_options_t*, + const char*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_WAL_ttl_seconds( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_WAL_ttl_seconds(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_WAL_size_limit_MB( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_WAL_size_limit_MB(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_manifest_preallocation_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_manifest_preallocation_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_allow_mmap_reads( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_allow_mmap_reads( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_allow_mmap_writes( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_allow_mmap_writes( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_use_direct_reads( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_use_direct_reads( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_use_direct_io_for_flush_and_compaction(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_use_direct_io_for_flush_and_compaction(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_is_fd_close_on_exec( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_is_fd_close_on_exec(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_stats_dump_period_sec( + rocksdb_options_t*, unsigned int); +extern ROCKSDB_LIBRARY_API unsigned int +rocksdb_options_get_stats_dump_period_sec(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_stats_persist_period_sec( + rocksdb_options_t*, unsigned int); +extern ROCKSDB_LIBRARY_API unsigned int +rocksdb_options_get_stats_persist_period_sec(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_advise_random_on_open( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_advise_random_on_open(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_access_hint_on_compaction_start(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_access_hint_on_compaction_start(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_use_adaptive_mutex( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_use_adaptive_mutex( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bytes_per_sync( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_bytes_per_sync(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_bytes_per_sync( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_wal_bytes_per_sync(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_writable_file_max_buffer_size(rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_writable_file_max_buffer_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_allow_concurrent_memtable_write(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_allow_concurrent_memtable_write(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_enable_write_thread_adaptive_yield(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_enable_write_thread_adaptive_yield(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_sequential_skip_in_iterations(rocksdb_options_t*, + uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_sequential_skip_in_iterations(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_disable_auto_compactions( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_disable_auto_compactions(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_optimize_filters_for_hits( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_optimize_filters_for_hits(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_delete_obsolete_files_period_micros(rocksdb_options_t*, + uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_delete_obsolete_files_period_micros(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_prepare_for_bulk_load( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_memtable_vector_rep( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_memtable_prefix_bloom_size_ratio(rocksdb_options_t*, + double); +extern ROCKSDB_LIBRARY_API double +rocksdb_options_get_memtable_prefix_bloom_size_ratio(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_compaction_bytes( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_compaction_bytes(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_hash_skip_list_rep( + rocksdb_options_t*, size_t, int32_t, int32_t); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_hash_link_list_rep( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_plain_table_factory( + rocksdb_options_t*, uint32_t, int, double, size_t); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_min_level_to_compress( + rocksdb_options_t* opt, int level); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_memtable_huge_page_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_memtable_huge_page_size(rocksdb_options_t*); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_successive_merges( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_max_successive_merges(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bloom_locality( + rocksdb_options_t*, uint32_t); +extern ROCKSDB_LIBRARY_API uint32_t +rocksdb_options_get_bloom_locality(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_inplace_update_support( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_inplace_update_support(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_inplace_update_num_locks( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_inplace_update_num_locks(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_report_bg_io_stats( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_report_bg_io_stats( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_avoid_unnecessary_blocking_io(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_avoid_unnecessary_blocking_io(rocksdb_options_t*); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_experimental_mempurge_threshold(rocksdb_options_t*, double); +extern ROCKSDB_LIBRARY_API double +rocksdb_options_get_experimental_mempurge_threshold(rocksdb_options_t*); + +enum { + rocksdb_tolerate_corrupted_tail_records_recovery = 0, + rocksdb_absolute_consistency_recovery = 1, + rocksdb_point_in_time_recovery = 2, + rocksdb_skip_any_corrupted_records_recovery = 3 +}; +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_recovery_mode( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_wal_recovery_mode( + rocksdb_options_t*); + +enum { + rocksdb_no_compression = 0, + rocksdb_snappy_compression = 1, + rocksdb_zlib_compression = 2, + rocksdb_bz2_compression = 3, + rocksdb_lz4_compression = 4, + rocksdb_lz4hc_compression = 5, + rocksdb_xpress_compression = 6, + rocksdb_zstd_compression = 7 +}; +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compression( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_compression( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bottommost_compression( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_bottommost_compression( + rocksdb_options_t*); + +enum { + rocksdb_level_compaction = 0, + rocksdb_universal_compaction = 1, + rocksdb_fifo_compaction = 2 +}; +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compaction_style( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_compaction_style( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_universal_compaction_options( + rocksdb_options_t*, rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_fifo_compaction_options( + rocksdb_options_t* opt, rocksdb_fifo_compaction_options_t* fifo); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_ratelimiter( + rocksdb_options_t* opt, rocksdb_ratelimiter_t* limiter); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_atomic_flush( + rocksdb_options_t* opt, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_atomic_flush( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_row_cache( + rocksdb_options_t* opt, rocksdb_cache_t* cache); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_add_compact_on_deletion_collector_factory( + rocksdb_options_t*, size_t window_size, size_t num_dels_trigger); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_manual_wal_flush( + rocksdb_options_t* opt, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_manual_wal_flush( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_compression( + rocksdb_options_t* opt, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_wal_compression( + rocksdb_options_t* opt); + +/* RateLimiter */ +extern ROCKSDB_LIBRARY_API rocksdb_ratelimiter_t* rocksdb_ratelimiter_create( + int64_t rate_bytes_per_sec, int64_t refill_period_us, int32_t fairness); +extern ROCKSDB_LIBRARY_API void rocksdb_ratelimiter_destroy( + rocksdb_ratelimiter_t*); + +/* PerfContext */ +enum { + rocksdb_uninitialized = 0, + rocksdb_disable = 1, + rocksdb_enable_count = 2, + rocksdb_enable_time_except_for_mutex = 3, + rocksdb_enable_time = 4, + rocksdb_out_of_bounds = 5 +}; + +enum { + rocksdb_user_key_comparison_count = 0, + rocksdb_block_cache_hit_count, + rocksdb_block_read_count, + rocksdb_block_read_byte, + rocksdb_block_read_time, + rocksdb_block_checksum_time, + rocksdb_block_decompress_time, + rocksdb_get_read_bytes, + rocksdb_multiget_read_bytes, + rocksdb_iter_read_bytes, + rocksdb_internal_key_skipped_count, + rocksdb_internal_delete_skipped_count, + rocksdb_internal_recent_skipped_count, + rocksdb_internal_merge_count, + rocksdb_get_snapshot_time, + rocksdb_get_from_memtable_time, + rocksdb_get_from_memtable_count, + rocksdb_get_post_process_time, + rocksdb_get_from_output_files_time, + rocksdb_seek_on_memtable_time, + rocksdb_seek_on_memtable_count, + rocksdb_next_on_memtable_count, + rocksdb_prev_on_memtable_count, + rocksdb_seek_child_seek_time, + rocksdb_seek_child_seek_count, + rocksdb_seek_min_heap_time, + rocksdb_seek_max_heap_time, + rocksdb_seek_internal_seek_time, + rocksdb_find_next_user_entry_time, + rocksdb_write_wal_time, + rocksdb_write_memtable_time, + rocksdb_write_delay_time, + rocksdb_write_pre_and_post_process_time, + rocksdb_db_mutex_lock_nanos, + rocksdb_db_condition_wait_nanos, + rocksdb_merge_operator_time_nanos, + rocksdb_read_index_block_nanos, + rocksdb_read_filter_block_nanos, + rocksdb_new_table_block_iter_nanos, + rocksdb_new_table_iterator_nanos, + rocksdb_block_seek_nanos, + rocksdb_find_table_nanos, + rocksdb_bloom_memtable_hit_count, + rocksdb_bloom_memtable_miss_count, + rocksdb_bloom_sst_hit_count, + rocksdb_bloom_sst_miss_count, + rocksdb_key_lock_wait_time, + rocksdb_key_lock_wait_count, + rocksdb_env_new_sequential_file_nanos, + rocksdb_env_new_random_access_file_nanos, + rocksdb_env_new_writable_file_nanos, + rocksdb_env_reuse_writable_file_nanos, + rocksdb_env_new_random_rw_file_nanos, + rocksdb_env_new_directory_nanos, + rocksdb_env_file_exists_nanos, + rocksdb_env_get_children_nanos, + rocksdb_env_get_children_file_attributes_nanos, + rocksdb_env_delete_file_nanos, + rocksdb_env_create_dir_nanos, + rocksdb_env_create_dir_if_missing_nanos, + rocksdb_env_delete_dir_nanos, + rocksdb_env_get_file_size_nanos, + rocksdb_env_get_file_modification_time_nanos, + rocksdb_env_rename_file_nanos, + rocksdb_env_link_file_nanos, + rocksdb_env_lock_file_nanos, + rocksdb_env_unlock_file_nanos, + rocksdb_env_new_logger_nanos, + rocksdb_number_async_seek, + rocksdb_blob_cache_hit_count, + rocksdb_blob_read_count, + rocksdb_blob_read_byte, + rocksdb_blob_read_time, + rocksdb_blob_checksum_time, + rocksdb_blob_decompress_time, + rocksdb_internal_range_del_reseek_count, + rocksdb_total_metric_count = 78 +}; + +extern ROCKSDB_LIBRARY_API void rocksdb_set_perf_level(int); +extern ROCKSDB_LIBRARY_API rocksdb_perfcontext_t* rocksdb_perfcontext_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_perfcontext_reset( + rocksdb_perfcontext_t* context); +extern ROCKSDB_LIBRARY_API char* rocksdb_perfcontext_report( + rocksdb_perfcontext_t* context, unsigned char exclude_zero_counters); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_perfcontext_metric(rocksdb_perfcontext_t* context, int metric); +extern ROCKSDB_LIBRARY_API void rocksdb_perfcontext_destroy( + rocksdb_perfcontext_t* context); + +/* Compaction Filter */ + +extern ROCKSDB_LIBRARY_API rocksdb_compactionfilter_t* +rocksdb_compactionfilter_create( + void* state, void (*destructor)(void*), + unsigned char (*filter)(void*, int level, const char* key, + size_t key_length, const char* existing_value, + size_t value_length, char** new_value, + size_t* new_value_length, + unsigned char* value_changed), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_compactionfilter_set_ignore_snapshots( + rocksdb_compactionfilter_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_compactionfilter_destroy( + rocksdb_compactionfilter_t*); + +/* Compaction Filter Context */ + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactionfiltercontext_is_full_compaction( + rocksdb_compactionfiltercontext_t* context); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactionfiltercontext_is_manual_compaction( + rocksdb_compactionfiltercontext_t* context); + +/* Compaction Filter Factory */ + +extern ROCKSDB_LIBRARY_API rocksdb_compactionfilterfactory_t* +rocksdb_compactionfilterfactory_create( + void* state, void (*destructor)(void*), + rocksdb_compactionfilter_t* (*create_compaction_filter)( + void*, rocksdb_compactionfiltercontext_t* context), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_compactionfilterfactory_destroy( + rocksdb_compactionfilterfactory_t*); + +/* Comparator */ + +extern ROCKSDB_LIBRARY_API rocksdb_comparator_t* rocksdb_comparator_create( + void* state, void (*destructor)(void*), + int (*compare)(void*, const char* a, size_t alen, const char* b, + size_t blen), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_comparator_destroy( + rocksdb_comparator_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_comparator_t* +rocksdb_comparator_with_ts_create( + void* state, void (*destructor)(void*), + int (*compare)(void*, const char* a, size_t alen, const char* b, + size_t blen), + int (*compare_ts)(void*, const char* a_ts, size_t a_tslen, const char* b_ts, + size_t b_tslen), + int (*compare_without_ts)(void*, const char* a, size_t alen, + unsigned char a_has_ts, const char* b, + size_t blen, unsigned char b_has_ts), + const char* (*name)(void*), size_t timestamp_size); + +/* Filter policy */ + +extern ROCKSDB_LIBRARY_API void rocksdb_filterpolicy_destroy( + rocksdb_filterpolicy_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_bloom(double bits_per_key); +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_bloom_full(double bits_per_key); +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_ribbon(double bloom_equivalent_bits_per_key); +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_ribbon_hybrid(double bloom_equivalent_bits_per_key, + int bloom_before_level); + +/* Merge Operator */ + +extern ROCKSDB_LIBRARY_API rocksdb_mergeoperator_t* +rocksdb_mergeoperator_create( + void* state, void (*destructor)(void*), + char* (*full_merge)(void*, const char* key, size_t key_length, + const char* existing_value, + size_t existing_value_length, + const char* const* operands_list, + const size_t* operands_list_length, int num_operands, + unsigned char* success, size_t* new_value_length), + char* (*partial_merge)(void*, const char* key, size_t key_length, + const char* const* operands_list, + const size_t* operands_list_length, int num_operands, + unsigned char* success, size_t* new_value_length), + void (*delete_value)(void*, const char* value, size_t value_length), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_mergeoperator_destroy( + rocksdb_mergeoperator_t*); + +/* Read options */ + +extern ROCKSDB_LIBRARY_API rocksdb_readoptions_t* rocksdb_readoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_destroy( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_verify_checksums( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_verify_checksums(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_fill_cache( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_fill_cache( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_snapshot( + rocksdb_readoptions_t*, const rocksdb_snapshot_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iterate_upper_bound( + rocksdb_readoptions_t*, const char* key, size_t keylen); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iterate_lower_bound( + rocksdb_readoptions_t*, const char* key, size_t keylen); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_read_tier( + rocksdb_readoptions_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_readoptions_get_read_tier( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_tailing( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_tailing( + rocksdb_readoptions_t*); +// The functionality that this option controlled has been removed. +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_managed( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_readahead_size( + rocksdb_readoptions_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_readoptions_get_readahead_size(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_prefix_same_as_start( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_prefix_same_as_start(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_pin_data( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_pin_data( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_total_order_seek( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_total_order_seek(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_readoptions_set_max_skippable_internal_keys(rocksdb_readoptions_t*, + uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_readoptions_get_max_skippable_internal_keys(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_readoptions_set_background_purge_on_iterator_cleanup( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_background_purge_on_iterator_cleanup( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_ignore_range_deletions( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_ignore_range_deletions(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_deadline( + rocksdb_readoptions_t*, uint64_t microseconds); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_readoptions_get_deadline(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_io_timeout( + rocksdb_readoptions_t*, uint64_t microseconds); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_readoptions_get_io_timeout(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_async_io( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_async_io( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_timestamp( + rocksdb_readoptions_t*, const char* ts, size_t tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iter_start_ts( + rocksdb_readoptions_t*, const char* ts, size_t tslen); + +/* Write options */ + +extern ROCKSDB_LIBRARY_API rocksdb_writeoptions_t* rocksdb_writeoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_destroy( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_set_sync( + rocksdb_writeoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_sync( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_disable_WAL( + rocksdb_writeoptions_t* opt, int disable); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_disable_WAL( + rocksdb_writeoptions_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_writeoptions_set_ignore_missing_column_families(rocksdb_writeoptions_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_writeoptions_get_ignore_missing_column_families( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_set_no_slowdown( + rocksdb_writeoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_no_slowdown( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_set_low_pri( + rocksdb_writeoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_low_pri( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_writeoptions_set_memtable_insert_hint_per_batch(rocksdb_writeoptions_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_writeoptions_get_memtable_insert_hint_per_batch( + rocksdb_writeoptions_t*); + +/* Compact range options */ + +extern ROCKSDB_LIBRARY_API rocksdb_compactoptions_t* +rocksdb_compactoptions_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_destroy( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_compactoptions_set_exclusive_manual_compaction( + rocksdb_compactoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactoptions_get_exclusive_manual_compaction( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_compactoptions_set_bottommost_level_compaction( + rocksdb_compactoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactoptions_get_bottommost_level_compaction( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_set_change_level( + rocksdb_compactoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactoptions_get_change_level(rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_set_target_level( + rocksdb_compactoptions_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_compactoptions_get_target_level( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_set_full_history_ts_low( + rocksdb_compactoptions_t*, char* ts, size_t tslen); + +/* Flush options */ + +extern ROCKSDB_LIBRARY_API rocksdb_flushoptions_t* rocksdb_flushoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_flushoptions_destroy( + rocksdb_flushoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_flushoptions_set_wait( + rocksdb_flushoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_flushoptions_get_wait( + rocksdb_flushoptions_t*); + +/* Memory allocator */ + +extern ROCKSDB_LIBRARY_API rocksdb_memory_allocator_t* +rocksdb_jemalloc_nodump_allocator_create(char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_allocator_destroy( + rocksdb_memory_allocator_t*); + +/* Cache */ + +extern ROCKSDB_LIBRARY_API rocksdb_lru_cache_options_t* +rocksdb_lru_cache_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_destroy( + rocksdb_lru_cache_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_capacity( + rocksdb_lru_cache_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_num_shard_bits( + rocksdb_lru_cache_options_t*, int); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_memory_allocator( + rocksdb_lru_cache_options_t*, rocksdb_memory_allocator_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_lru( + size_t capacity); +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* +rocksdb_cache_create_lru_with_strict_capacity_limit(size_t capacity); +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_lru_opts( + rocksdb_lru_cache_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_cache_destroy(rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API void rocksdb_cache_disown_data( + rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API void rocksdb_cache_set_capacity( + rocksdb_cache_t* cache, size_t capacity); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_cache_get_capacity(rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_cache_get_usage(rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_cache_get_pinned_usage(rocksdb_cache_t* cache); + +/* HyperClockCache */ +extern ROCKSDB_LIBRARY_API rocksdb_hyper_clock_cache_options_t* +rocksdb_hyper_clock_cache_options_create(size_t capacity, + size_t estimated_entry_charge); +extern ROCKSDB_LIBRARY_API void rocksdb_hyper_clock_cache_options_destroy( + rocksdb_hyper_clock_cache_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_hyper_clock_cache_options_set_capacity( + rocksdb_hyper_clock_cache_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void +rocksdb_hyper_clock_cache_options_set_estimated_entry_charge( + rocksdb_hyper_clock_cache_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void +rocksdb_hyper_clock_cache_options_set_num_shard_bits( + rocksdb_hyper_clock_cache_options_t*, int); +extern ROCKSDB_LIBRARY_API void +rocksdb_hyper_clock_cache_options_set_memory_allocator( + rocksdb_hyper_clock_cache_options_t*, rocksdb_memory_allocator_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_hyper_clock( + size_t capacity, size_t estimated_entry_charge); +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* +rocksdb_cache_create_hyper_clock_opts(rocksdb_hyper_clock_cache_options_t*); + +/* DBPath */ + +extern ROCKSDB_LIBRARY_API rocksdb_dbpath_t* rocksdb_dbpath_create( + const char* path, uint64_t target_size); +extern ROCKSDB_LIBRARY_API void rocksdb_dbpath_destroy(rocksdb_dbpath_t*); + +/* Env */ + +extern ROCKSDB_LIBRARY_API rocksdb_env_t* rocksdb_create_default_env(void); +extern ROCKSDB_LIBRARY_API rocksdb_env_t* rocksdb_create_mem_env(void); +extern ROCKSDB_LIBRARY_API void rocksdb_env_set_background_threads( + rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int rocksdb_env_get_background_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_set_high_priority_background_threads(rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int rocksdb_env_get_high_priority_background_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_set_low_priority_background_threads( + rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int rocksdb_env_get_low_priority_background_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_set_bottom_priority_background_threads(rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int +rocksdb_env_get_bottom_priority_background_threads(rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_join_all_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_lower_thread_pool_io_priority( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_lower_high_priority_thread_pool_io_priority(rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_lower_thread_pool_cpu_priority( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_lower_high_priority_thread_pool_cpu_priority(rocksdb_env_t* env); + +extern ROCKSDB_LIBRARY_API void rocksdb_env_destroy(rocksdb_env_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_envoptions_t* rocksdb_envoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_envoptions_destroy( + rocksdb_envoptions_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_create_dir_if_missing( + rocksdb_env_t* env, const char* path, char** errptr); + +/* SstFile */ + +extern ROCKSDB_LIBRARY_API rocksdb_sstfilewriter_t* +rocksdb_sstfilewriter_create(const rocksdb_envoptions_t* env, + const rocksdb_options_t* io_options); +extern ROCKSDB_LIBRARY_API rocksdb_sstfilewriter_t* +rocksdb_sstfilewriter_create_with_comparator( + const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options, + const rocksdb_comparator_t* comparator); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_open( + rocksdb_sstfilewriter_t* writer, const char* name, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_add( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* val, size_t vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_put( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* val, size_t vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_put_with_ts( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* ts, size_t tslen, const char* val, size_t vallen, + char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_merge( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* val, size_t vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete_with_ts( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* ts, size_t tslen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete_range( + rocksdb_sstfilewriter_t* writer, const char* begin_key, size_t begin_keylen, + const char* end_key, size_t end_keylen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_finish( + rocksdb_sstfilewriter_t* writer, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_file_size( + rocksdb_sstfilewriter_t* writer, uint64_t* file_size); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_destroy( + rocksdb_sstfilewriter_t* writer); +extern ROCKSDB_LIBRARY_API rocksdb_ingestexternalfileoptions_t* +rocksdb_ingestexternalfileoptions_create(void); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_move_files( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char move_files); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_snapshot_consistency( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char snapshot_consistency); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_allow_global_seqno( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char allow_global_seqno); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_allow_blocking_flush( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char allow_blocking_flush); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_ingest_behind( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char ingest_behind); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_fail_if_not_bottommost_level( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char fail_if_not_bottommost_level); + +extern ROCKSDB_LIBRARY_API void rocksdb_ingestexternalfileoptions_destroy( + rocksdb_ingestexternalfileoptions_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_ingest_external_file( + rocksdb_t* db, const char* const* file_list, const size_t list_len, + const rocksdb_ingestexternalfileoptions_t* opt, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_ingest_external_file_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, + const char* const* file_list, const size_t list_len, + const rocksdb_ingestexternalfileoptions_t* opt, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_try_catch_up_with_primary( + rocksdb_t* db, char** errptr); + +/* SliceTransform */ + +extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t* +rocksdb_slicetransform_create( + void* state, void (*destructor)(void*), + char* (*transform)(void*, const char* key, size_t length, + size_t* dst_length), + unsigned char (*in_domain)(void*, const char* key, size_t length), + unsigned char (*in_range)(void*, const char* key, size_t length), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t* + rocksdb_slicetransform_create_fixed_prefix(size_t); +extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t* +rocksdb_slicetransform_create_noop(void); +extern ROCKSDB_LIBRARY_API void rocksdb_slicetransform_destroy( + rocksdb_slicetransform_t*); + +/* Universal Compaction options */ + +enum { + rocksdb_similar_size_compaction_stop_style = 0, + rocksdb_total_size_compaction_stop_style = 1 +}; + +extern ROCKSDB_LIBRARY_API rocksdb_universal_compaction_options_t* +rocksdb_universal_compaction_options_create(void); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_size_ratio( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_size_ratio( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_min_merge_width( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_min_merge_width( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_max_merge_width( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_max_merge_width( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_max_size_amplification_percent( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_max_size_amplification_percent( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_compression_size_percent( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_compression_size_percent( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_stop_style( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_stop_style( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_universal_compaction_options_destroy( + rocksdb_universal_compaction_options_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_fifo_compaction_options_t* +rocksdb_fifo_compaction_options_create(void); +extern ROCKSDB_LIBRARY_API void +rocksdb_fifo_compaction_options_set_allow_compaction( + rocksdb_fifo_compaction_options_t* fifo_opts, unsigned char allow_compaction); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_fifo_compaction_options_get_allow_compaction( + rocksdb_fifo_compaction_options_t* fifo_opts); +extern ROCKSDB_LIBRARY_API void +rocksdb_fifo_compaction_options_set_max_table_files_size( + rocksdb_fifo_compaction_options_t* fifo_opts, uint64_t size); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_fifo_compaction_options_get_max_table_files_size( + rocksdb_fifo_compaction_options_t* fifo_opts); +extern ROCKSDB_LIBRARY_API void rocksdb_fifo_compaction_options_destroy( + rocksdb_fifo_compaction_options_t* fifo_opts); + +extern ROCKSDB_LIBRARY_API int rocksdb_livefiles_count( + const rocksdb_livefiles_t*); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_column_family_name( + const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_name( + const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API int rocksdb_livefiles_level( + const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_livefiles_size(const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_smallestkey( + const rocksdb_livefiles_t*, int index, size_t* size); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_largestkey( + const rocksdb_livefiles_t*, int index, size_t* size); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_livefiles_entries(const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_livefiles_deletions(const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API void rocksdb_livefiles_destroy( + const rocksdb_livefiles_t*); + +/* Utility Helpers */ + +extern ROCKSDB_LIBRARY_API void rocksdb_get_options_from_string( + const rocksdb_options_t* base_options, const char* opts_str, + rocksdb_options_t* new_options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_file_in_range( + rocksdb_t* db, const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_file_in_range_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* limit_key, + size_t limit_key_len, char** errptr); + +/* MetaData */ + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_metadata_t* +rocksdb_get_column_family_metadata(rocksdb_t* db); + +/** + * Returns the rocksdb_column_family_metadata_t of the specified + * column family. + * + * Note that the caller is responsible to release the returned memory + * using rocksdb_column_family_metadata_destroy. + */ +extern ROCKSDB_LIBRARY_API rocksdb_column_family_metadata_t* +rocksdb_get_column_family_metadata_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API void rocksdb_column_family_metadata_destroy( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API uint64_t rocksdb_column_family_metadata_get_size( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API size_t rocksdb_column_family_metadata_get_file_count( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API char* rocksdb_column_family_metadata_get_name( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API size_t +rocksdb_column_family_metadata_get_level_count( + rocksdb_column_family_metadata_t* cf_meta); + +/** + * Returns the rocksdb_level_metadata_t of the ith level from the specified + * column family metadata. + * + * If the specified i is greater than or equal to the number of levels + * in the specified column family, then NULL will be returned. + * + * Note that the caller is responsible to release the returned memory + * using rocksdb_level_metadata_destroy before releasing its parent + * rocksdb_column_family_metadata_t. + */ +extern ROCKSDB_LIBRARY_API rocksdb_level_metadata_t* +rocksdb_column_family_metadata_get_level_metadata( + rocksdb_column_family_metadata_t* cf_meta, size_t i); + +/** + * Releases the specified rocksdb_level_metadata_t. + * + * Note that the specified rocksdb_level_metadata_t must be released + * before the release of its parent rocksdb_column_family_metadata_t. + */ +extern ROCKSDB_LIBRARY_API void rocksdb_level_metadata_destroy( + rocksdb_level_metadata_t* level_meta); + +extern ROCKSDB_LIBRARY_API int rocksdb_level_metadata_get_level( + rocksdb_level_metadata_t* level_meta); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_level_metadata_get_size(rocksdb_level_metadata_t* level_meta); + +extern ROCKSDB_LIBRARY_API size_t +rocksdb_level_metadata_get_file_count(rocksdb_level_metadata_t* level_meta); + +/** + * Returns the sst_file_metadata_t of the ith file from the specified level + * metadata. + * + * If the specified i is greater than or equal to the number of files + * in the specified level, then NULL will be returned. + * + * Note that the caller is responsible to release the returned memory + * using rocksdb_sst_file_metadata_destroy before releasing its + * parent rocksdb_level_metadata_t. + */ +extern ROCKSDB_LIBRARY_API rocksdb_sst_file_metadata_t* +rocksdb_level_metadata_get_sst_file_metadata( + rocksdb_level_metadata_t* level_meta, size_t i); + +/** + * Releases the specified rocksdb_sst_file_metadata_t. + * + * Note that the specified rocksdb_sst_file_metadata_t must be released + * before the release of its parent rocksdb_level_metadata_t. + */ +extern ROCKSDB_LIBRARY_API void rocksdb_sst_file_metadata_destroy( + rocksdb_sst_file_metadata_t* file_meta); + +extern ROCKSDB_LIBRARY_API char* +rocksdb_sst_file_metadata_get_relative_filename( + rocksdb_sst_file_metadata_t* file_meta); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_sst_file_metadata_get_size(rocksdb_sst_file_metadata_t* file_meta); + +/** + * Returns the smallest key of the specified sst file. + * The caller is responsible for releasing the returned memory. + * + * @param file_meta the metadata of an SST file to obtain its smallest key. + * @param len the out value which will contain the length of the returned key + * after the function call. + */ +extern ROCKSDB_LIBRARY_API char* rocksdb_sst_file_metadata_get_smallestkey( + rocksdb_sst_file_metadata_t* file_meta, size_t* len); + +/** + * Returns the smallest key of the specified sst file. + * The caller is responsible for releasing the returned memory. + * + * @param file_meta the metadata of an SST file to obtain its smallest key. + * @param len the out value which will contain the length of the returned key + * after the function call. + */ +extern ROCKSDB_LIBRARY_API char* rocksdb_sst_file_metadata_get_largestkey( + rocksdb_sst_file_metadata_t* file_meta, size_t* len); + +/* Transactions */ + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* +rocksdb_transactiondb_create_column_family( + rocksdb_transactiondb_t* txn_db, + const rocksdb_options_t* column_family_options, + const char* column_family_name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_transactiondb_t* rocksdb_transactiondb_open( + const rocksdb_options_t* options, + const rocksdb_transactiondb_options_t* txn_db_options, const char* name, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_transactiondb_t* +rocksdb_transactiondb_open_column_families( + const rocksdb_options_t* options, + const rocksdb_transactiondb_options_t* txn_db_options, const char* name, + int num_column_families, const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API const rocksdb_snapshot_t* +rocksdb_transactiondb_create_snapshot(rocksdb_transactiondb_t* txn_db); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_release_snapshot( + rocksdb_transactiondb_t* txn_db, const rocksdb_snapshot_t* snapshot); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transactiondb_property_value( + rocksdb_transactiondb_t* db, const char* propname); + +extern ROCKSDB_LIBRARY_API int rocksdb_transactiondb_property_int( + rocksdb_transactiondb_t* db, const char* propname, uint64_t* out_val); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_t* rocksdb_transaction_begin( + rocksdb_transactiondb_t* txn_db, + const rocksdb_writeoptions_t* write_options, + const rocksdb_transaction_options_t* txn_options, + rocksdb_transaction_t* old_txn); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_t** +rocksdb_transactiondb_get_prepared_transactions(rocksdb_transactiondb_t* txn_db, + size_t* cnt); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_set_name( + rocksdb_transaction_t* txn, const char* name, size_t name_len, + char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_name( + rocksdb_transaction_t* txn, size_t* name_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_prepare( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_commit( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rollback( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_set_savepoint( + rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rollback_to_savepoint( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_destroy( + rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_wi_t* +rocksdb_transaction_get_writebatch_wi(rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rebuild_from_writebatch( + rocksdb_transaction_t* txn, rocksdb_writebatch_t* writebatch, + char** errptr); + +// This rocksdb_writebatch_wi_t should be freed with rocksdb_free +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rebuild_from_writebatch_wi( + rocksdb_transaction_t* txn, rocksdb_writebatch_wi_t* wi, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_set_commit_timestamp( + rocksdb_transaction_t* txn, uint64_t commit_timestamp); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transaction_set_read_timestamp_for_validation( + rocksdb_transaction_t* txn, uint64_t read_timestamp); + +// This snapshot should be freed using rocksdb_free +extern ROCKSDB_LIBRARY_API const rocksdb_snapshot_t* +rocksdb_transaction_get_snapshot(rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const char* key, size_t klen, size_t* vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + size_t* vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned_cf(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_for_update( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const char* key, size_t klen, size_t* vlen, unsigned char exclusive, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned_for_update(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, + unsigned char exclusive, + char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_for_update_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + size_t* vlen, unsigned char exclusive, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned_for_update_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + unsigned char exclusive, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_multi_get( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_multi_get_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transactiondb_get( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + const char* key, size_t klen, size_t* vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transactiondb_get_pinned(rocksdb_transactiondb_t* txn_db, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transactiondb_get_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transactiondb_get_pinned_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_multi_get( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_multi_get_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_put( + rocksdb_transaction_t* txn, const char* key, size_t klen, const char* val, + size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_put_cf( + rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_put( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_put_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_write( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_writebatch_t* batch, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_merge( + rocksdb_transaction_t* txn, const char* key, size_t klen, const char* val, + size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_merge_cf( + rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_merge( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_merge_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_delete( + rocksdb_transaction_t* txn, const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_delete_cf( + rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_delete( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_delete_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transaction_create_iterator(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transaction_create_iterator_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transactiondb_create_iterator(rocksdb_transactiondb_t* txn_db, + const rocksdb_readoptions_t* options); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transactiondb_create_iterator_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_close( + rocksdb_transactiondb_t* txn_db); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush( + rocksdb_transactiondb_t* txn_db, const rocksdb_flushoptions_t* options, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t* column_family, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush_cfs( + rocksdb_transactiondb_t* txn_db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t** column_families, int num_column_families, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush_wal( + rocksdb_transactiondb_t* txn_db, unsigned char sync, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_checkpoint_t* +rocksdb_transactiondb_checkpoint_object_create(rocksdb_transactiondb_t* txn_db, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_optimistictransactiondb_t* +rocksdb_optimistictransactiondb_open(const rocksdb_options_t* options, + const char* name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_optimistictransactiondb_t* +rocksdb_optimistictransactiondb_open_column_families( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* +rocksdb_optimistictransactiondb_get_base_db( + rocksdb_optimistictransactiondb_t* otxn_db); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransactiondb_close_base_db( + rocksdb_t* base_db); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_t* +rocksdb_optimistictransaction_begin( + rocksdb_optimistictransactiondb_t* otxn_db, + const rocksdb_writeoptions_t* write_options, + const rocksdb_optimistictransaction_options_t* otxn_options, + rocksdb_transaction_t* old_txn); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransactiondb_write( + rocksdb_optimistictransactiondb_t* otxn_db, + const rocksdb_writeoptions_t* options, rocksdb_writebatch_t* batch, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransactiondb_close( + rocksdb_optimistictransactiondb_t* otxn_db); + +extern ROCKSDB_LIBRARY_API rocksdb_checkpoint_t* +rocksdb_optimistictransactiondb_checkpoint_object_create( + rocksdb_optimistictransactiondb_t* otxn_db, char** errptr); + +/* Transaction Options */ + +extern ROCKSDB_LIBRARY_API rocksdb_transactiondb_options_t* +rocksdb_transactiondb_options_create(void); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_options_destroy( + rocksdb_transactiondb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_options_set_max_num_locks( + rocksdb_transactiondb_options_t* opt, int64_t max_num_locks); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_options_set_num_stripes( + rocksdb_transactiondb_options_t* opt, size_t num_stripes); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transactiondb_options_set_transaction_lock_timeout( + rocksdb_transactiondb_options_t* opt, int64_t txn_lock_timeout); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transactiondb_options_set_default_lock_timeout( + rocksdb_transactiondb_options_t* opt, int64_t default_lock_timeout); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_options_t* +rocksdb_transaction_options_create(void); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_destroy( + rocksdb_transaction_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_set_snapshot( + rocksdb_transaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_deadlock_detect( + rocksdb_transaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_lock_timeout( + rocksdb_transaction_options_t* opt, int64_t lock_timeout); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_expiration( + rocksdb_transaction_options_t* opt, int64_t expiration); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transaction_options_set_deadlock_detect_depth( + rocksdb_transaction_options_t* opt, int64_t depth); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transaction_options_set_max_write_batch_size( + rocksdb_transaction_options_t* opt, size_t size); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_skip_prepare( + rocksdb_transaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API rocksdb_optimistictransaction_options_t* +rocksdb_optimistictransaction_options_create(void); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransaction_options_destroy( + rocksdb_optimistictransaction_options_t* opt); + +extern ROCKSDB_LIBRARY_API void +rocksdb_optimistictransaction_options_set_set_snapshot( + rocksdb_optimistictransaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API char* rocksdb_optimistictransactiondb_property_value( + rocksdb_optimistictransactiondb_t* db, const char* propname); + +extern ROCKSDB_LIBRARY_API int rocksdb_optimistictransactiondb_property_int( + rocksdb_optimistictransactiondb_t* db, const char* propname, + uint64_t* out_val); + +// referring to convention (3), this should be used by client +// to free memory that was malloc()ed +extern ROCKSDB_LIBRARY_API void rocksdb_free(void* ptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* rocksdb_get_pinned( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t keylen, char** errptr); +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* rocksdb_get_pinned_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_pinnableslice_destroy( + rocksdb_pinnableslice_t* v); +extern ROCKSDB_LIBRARY_API const char* rocksdb_pinnableslice_value( + const rocksdb_pinnableslice_t* t, size_t* vlen); + +extern ROCKSDB_LIBRARY_API rocksdb_memory_consumers_t* +rocksdb_memory_consumers_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_add_db( + rocksdb_memory_consumers_t* consumers, rocksdb_t* db); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_add_cache( + rocksdb_memory_consumers_t* consumers, rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_destroy( + rocksdb_memory_consumers_t* consumers); +extern ROCKSDB_LIBRARY_API rocksdb_memory_usage_t* +rocksdb_approximate_memory_usage_create(rocksdb_memory_consumers_t* consumers, + char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_approximate_memory_usage_destroy( + rocksdb_memory_usage_t* usage); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_mem_table_total( + rocksdb_memory_usage_t* memory_usage); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_mem_table_unflushed( + rocksdb_memory_usage_t* memory_usage); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_mem_table_readers_total( + rocksdb_memory_usage_t* memory_usage); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_cache_total( + rocksdb_memory_usage_t* memory_usage); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_dump_malloc_stats( + rocksdb_options_t*, unsigned char); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_memtable_whole_key_filtering(rocksdb_options_t*, + unsigned char); + +extern ROCKSDB_LIBRARY_API void rocksdb_cancel_all_background_work( + rocksdb_t* db, unsigned char wait); + +extern ROCKSDB_LIBRARY_API void rocksdb_disable_manual_compaction( + rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_enable_manual_compaction(rocksdb_t* db); + +#ifdef __cplusplus +} /* end extern "C" */ +#endif diff --git a/deps/x86/rocksdb_static/librocksdb.a b/deps/x86/rocksdb_static/librocksdb.a new file mode 100644 index 0000000000000000000000000000000000000000..0c788dd67443bf7de6c8ddc4067da8fd85cf21a9 Binary files /dev/null and b/deps/x86/rocksdb_static/librocksdb.a differ diff --git a/deps/x86/rocksdb_static/rocksdb/c.h b/deps/x86/rocksdb_static/rocksdb/c.h new file mode 100644 index 0000000000000000000000000000000000000000..1ba7fabefe92f674ecbd27a91bfbb57b0ed17d23 --- /dev/null +++ b/deps/x86/rocksdb_static/rocksdb/c.h @@ -0,0 +1,2844 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +/* Copyright (c) 2011 The LevelDB Authors. All rights reserved. + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. See the AUTHORS file for names of contributors. + + C bindings for rocksdb. May be useful as a stable ABI that can be + used by programs that keep rocksdb in a shared library, or for + a JNI api. + + Does not support: + . getters for the option types + . custom comparators that implement key shortening + . capturing post-write-snapshot + . custom iter, db, env, cache implementations using just the C bindings + + Some conventions: + + (1) We expose just opaque struct pointers and functions to clients. + This allows us to change internal representations without having to + recompile clients. + + (2) For simplicity, there is no equivalent to the Slice type. Instead, + the caller has to pass the pointer and length as separate + arguments. + + (3) Errors are represented by a null-terminated c string. NULL + means no error. All operations that can raise an error are passed + a "char** errptr" as the last argument. One of the following must + be true on entry: + *errptr == NULL + *errptr points to a malloc()ed null-terminated error message + On success, a leveldb routine leaves *errptr unchanged. + On failure, leveldb frees the old value of *errptr and + set *errptr to a malloc()ed error message. + + (4) Bools have the type unsigned char (0 == false; rest == true) + + (5) All of the pointer arguments must be non-NULL. +*/ + +#pragma once + +#ifdef _WIN32 +#ifdef ROCKSDB_DLL +#ifdef ROCKSDB_LIBRARY_EXPORTS +#define ROCKSDB_LIBRARY_API __declspec(dllexport) +#else +#define ROCKSDB_LIBRARY_API __declspec(dllimport) +#endif +#else +#define ROCKSDB_LIBRARY_API +#endif +#else +#define ROCKSDB_LIBRARY_API +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include + +/* Exported types */ + +typedef struct rocksdb_t rocksdb_t; +typedef struct rocksdb_backup_engine_t rocksdb_backup_engine_t; +typedef struct rocksdb_backup_engine_info_t rocksdb_backup_engine_info_t; +typedef struct rocksdb_backup_engine_options_t rocksdb_backup_engine_options_t; +typedef struct rocksdb_restore_options_t rocksdb_restore_options_t; +typedef struct rocksdb_memory_allocator_t rocksdb_memory_allocator_t; +typedef struct rocksdb_lru_cache_options_t rocksdb_lru_cache_options_t; +typedef struct rocksdb_hyper_clock_cache_options_t + rocksdb_hyper_clock_cache_options_t; +typedef struct rocksdb_cache_t rocksdb_cache_t; +typedef struct rocksdb_compactionfilter_t rocksdb_compactionfilter_t; +typedef struct rocksdb_compactionfiltercontext_t + rocksdb_compactionfiltercontext_t; +typedef struct rocksdb_compactionfilterfactory_t + rocksdb_compactionfilterfactory_t; +typedef struct rocksdb_comparator_t rocksdb_comparator_t; +typedef struct rocksdb_dbpath_t rocksdb_dbpath_t; +typedef struct rocksdb_env_t rocksdb_env_t; +typedef struct rocksdb_fifo_compaction_options_t + rocksdb_fifo_compaction_options_t; +typedef struct rocksdb_filelock_t rocksdb_filelock_t; +typedef struct rocksdb_filterpolicy_t rocksdb_filterpolicy_t; +typedef struct rocksdb_flushoptions_t rocksdb_flushoptions_t; +typedef struct rocksdb_iterator_t rocksdb_iterator_t; +typedef struct rocksdb_logger_t rocksdb_logger_t; +typedef struct rocksdb_mergeoperator_t rocksdb_mergeoperator_t; +typedef struct rocksdb_options_t rocksdb_options_t; +typedef struct rocksdb_compactoptions_t rocksdb_compactoptions_t; +typedef struct rocksdb_block_based_table_options_t + rocksdb_block_based_table_options_t; +typedef struct rocksdb_cuckoo_table_options_t rocksdb_cuckoo_table_options_t; +typedef struct rocksdb_randomfile_t rocksdb_randomfile_t; +typedef struct rocksdb_readoptions_t rocksdb_readoptions_t; +typedef struct rocksdb_seqfile_t rocksdb_seqfile_t; +typedef struct rocksdb_slicetransform_t rocksdb_slicetransform_t; +typedef struct rocksdb_snapshot_t rocksdb_snapshot_t; +typedef struct rocksdb_writablefile_t rocksdb_writablefile_t; +typedef struct rocksdb_writebatch_t rocksdb_writebatch_t; +typedef struct rocksdb_writebatch_wi_t rocksdb_writebatch_wi_t; +typedef struct rocksdb_writeoptions_t rocksdb_writeoptions_t; +typedef struct rocksdb_universal_compaction_options_t + rocksdb_universal_compaction_options_t; +typedef struct rocksdb_livefiles_t rocksdb_livefiles_t; +typedef struct rocksdb_column_family_handle_t rocksdb_column_family_handle_t; +typedef struct rocksdb_column_family_metadata_t + rocksdb_column_family_metadata_t; +typedef struct rocksdb_level_metadata_t rocksdb_level_metadata_t; +typedef struct rocksdb_sst_file_metadata_t rocksdb_sst_file_metadata_t; +typedef struct rocksdb_envoptions_t rocksdb_envoptions_t; +typedef struct rocksdb_ingestexternalfileoptions_t + rocksdb_ingestexternalfileoptions_t; +typedef struct rocksdb_sstfilewriter_t rocksdb_sstfilewriter_t; +typedef struct rocksdb_ratelimiter_t rocksdb_ratelimiter_t; +typedef struct rocksdb_perfcontext_t rocksdb_perfcontext_t; +typedef struct rocksdb_pinnableslice_t rocksdb_pinnableslice_t; +typedef struct rocksdb_transactiondb_options_t rocksdb_transactiondb_options_t; +typedef struct rocksdb_transactiondb_t rocksdb_transactiondb_t; +typedef struct rocksdb_transaction_options_t rocksdb_transaction_options_t; +typedef struct rocksdb_optimistictransactiondb_t + rocksdb_optimistictransactiondb_t; +typedef struct rocksdb_optimistictransaction_options_t + rocksdb_optimistictransaction_options_t; +typedef struct rocksdb_transaction_t rocksdb_transaction_t; +typedef struct rocksdb_checkpoint_t rocksdb_checkpoint_t; +typedef struct rocksdb_wal_iterator_t rocksdb_wal_iterator_t; +typedef struct rocksdb_wal_readoptions_t rocksdb_wal_readoptions_t; +typedef struct rocksdb_memory_consumers_t rocksdb_memory_consumers_t; +typedef struct rocksdb_memory_usage_t rocksdb_memory_usage_t; + +/* DB operations */ + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open( + const rocksdb_options_t* options, const char* name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_with_ttl( + const rocksdb_options_t* options, const char* name, int ttl, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_for_read_only( + const rocksdb_options_t* options, const char* name, + unsigned char error_if_wal_file_exists, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_as_secondary( + const rocksdb_options_t* options, const char* name, + const char* secondary_path, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_backup_engine_t* rocksdb_backup_engine_open( + const rocksdb_options_t* options, const char* path, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_backup_engine_t* +rocksdb_backup_engine_open_opts(const rocksdb_backup_engine_options_t* options, + rocksdb_env_t* env, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_create_new_backup( + rocksdb_backup_engine_t* be, rocksdb_t* db, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_create_new_backup_flush( + rocksdb_backup_engine_t* be, rocksdb_t* db, + unsigned char flush_before_backup, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_purge_old_backups( + rocksdb_backup_engine_t* be, uint32_t num_backups_to_keep, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_restore_options_t* +rocksdb_restore_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_restore_options_destroy( + rocksdb_restore_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_restore_options_set_keep_log_files( + rocksdb_restore_options_t* opt, int v); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_verify_backup( + rocksdb_backup_engine_t* be, uint32_t backup_id, char** errptr); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_restore_db_from_latest_backup( + rocksdb_backup_engine_t* be, const char* db_dir, const char* wal_dir, + const rocksdb_restore_options_t* restore_options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_restore_db_from_backup( + rocksdb_backup_engine_t* be, const char* db_dir, const char* wal_dir, + const rocksdb_restore_options_t* restore_options, const uint32_t backup_id, + char** errptr); + +extern ROCKSDB_LIBRARY_API const rocksdb_backup_engine_info_t* +rocksdb_backup_engine_get_backup_info(rocksdb_backup_engine_t* be); + +extern ROCKSDB_LIBRARY_API int rocksdb_backup_engine_info_count( + const rocksdb_backup_engine_info_t* info); + +extern ROCKSDB_LIBRARY_API int64_t rocksdb_backup_engine_info_timestamp( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API uint32_t rocksdb_backup_engine_info_backup_id( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API uint64_t rocksdb_backup_engine_info_size( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API uint32_t rocksdb_backup_engine_info_number_files( + const rocksdb_backup_engine_info_t* info, int index); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_info_destroy( + const rocksdb_backup_engine_info_t* info); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_close( + rocksdb_backup_engine_t* be); + +extern ROCKSDB_LIBRARY_API void rocksdb_put_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* ts, size_t tslen, const char* val, size_t vallen, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_put_cf_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* ts, size_t tslen, const char* val, size_t vallen, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_cf_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_singledelete_cf_with_ts( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* ts, size_t tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_increase_full_history_ts_low( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* ts_low, size_t ts_lowlen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_full_history_ts_low( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + size_t* ts_lowlen, char** errptr); + +/* BackupEngineOptions */ + +extern ROCKSDB_LIBRARY_API rocksdb_backup_engine_options_t* +rocksdb_backup_engine_options_create(const char* backup_dir); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_set_backup_dir( + rocksdb_backup_engine_options_t* options, const char* backup_dir); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_set_env( + rocksdb_backup_engine_options_t* options, rocksdb_env_t* env); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_share_table_files( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_backup_engine_options_get_share_table_files( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_set_sync( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_backup_engine_options_get_sync( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_destroy_old_data( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_backup_engine_options_get_destroy_old_data( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_backup_log_files( + rocksdb_backup_engine_options_t* options, unsigned char val); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_backup_engine_options_get_backup_log_files( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_backup_rate_limit( + rocksdb_backup_engine_options_t* options, uint64_t limit); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_backup_engine_options_get_backup_rate_limit( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_restore_rate_limit( + rocksdb_backup_engine_options_t* options, uint64_t limit); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_backup_engine_options_get_restore_rate_limit( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_max_background_operations( + rocksdb_backup_engine_options_t* options, int val); + +extern ROCKSDB_LIBRARY_API int +rocksdb_backup_engine_options_get_max_background_operations( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_callback_trigger_interval_size( + rocksdb_backup_engine_options_t* options, uint64_t size); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_backup_engine_options_get_callback_trigger_interval_size( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_max_valid_backups_to_open( + rocksdb_backup_engine_options_t* options, int val); + +extern ROCKSDB_LIBRARY_API int +rocksdb_backup_engine_options_get_max_valid_backups_to_open( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void +rocksdb_backup_engine_options_set_share_files_with_checksum_naming( + rocksdb_backup_engine_options_t* options, int val); + +extern ROCKSDB_LIBRARY_API int +rocksdb_backup_engine_options_get_share_files_with_checksum_naming( + rocksdb_backup_engine_options_t* options); + +extern ROCKSDB_LIBRARY_API void rocksdb_backup_engine_options_destroy( + rocksdb_backup_engine_options_t*); + +/* Checkpoint */ + +extern ROCKSDB_LIBRARY_API rocksdb_checkpoint_t* +rocksdb_checkpoint_object_create(rocksdb_t* db, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_checkpoint_create( + rocksdb_checkpoint_t* checkpoint, const char* checkpoint_dir, + uint64_t log_size_for_flush, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_checkpoint_object_destroy( + rocksdb_checkpoint_t* checkpoint); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_and_trim_history( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char* trim_ts, + size_t trim_tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_column_families( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_column_families_with_ttl( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, const int* ttls, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* +rocksdb_open_for_read_only_column_families( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, + unsigned char error_if_wal_file_exists, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_as_secondary_column_families( + const rocksdb_options_t* options, const char* name, + const char* secondary_path, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API char** rocksdb_list_column_families( + const rocksdb_options_t* options, const char* name, size_t* lencf, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_list_column_families_destroy( + char** list, size_t len); + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* +rocksdb_create_column_family(rocksdb_t* db, + const rocksdb_options_t* column_family_options, + const char* column_family_name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* +rocksdb_create_column_family_with_ttl( + rocksdb_t* db, const rocksdb_options_t* column_family_options, + const char* column_family_name, int ttl, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_drop_column_family( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_column_family_handle_destroy( + rocksdb_column_family_handle_t*); + +extern ROCKSDB_LIBRARY_API uint32_t +rocksdb_column_family_handle_get_id(rocksdb_column_family_handle_t* handle); + +extern ROCKSDB_LIBRARY_API char* rocksdb_column_family_handle_get_name( + rocksdb_column_family_handle_t* handle, size_t* name_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_close(rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_put( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_put_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_range_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* start_key, + size_t start_key_len, const char* end_key, size_t end_key_len, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_merge( + rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_merge_cf( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_write( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_writebatch_t* batch, char** errptr); + +/* Returns NULL if not found. A malloc()ed array otherwise. + Stores the length of the array in *vallen. */ +extern ROCKSDB_LIBRARY_API char* rocksdb_get( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t keylen, size_t* vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t keylen, size_t* vallen, char** ts, size_t* tslen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_get_cf_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** ts, size_t* tslen, char** errptr); + +// if values_list[i] == NULL and errs[i] == NULL, +// then we got status.IsNotFound(), which we will not return. +// all errors except status status.ok() and status.IsNotFound() are returned. +// +// errs, values_list and values_list_sizes must be num_keys in length, +// allocated by the caller. +// errs is a list of strings as opposed to the conventional one error, +// where errs[i] is the status for retrieval of keys_list[i]. +// each non-NULL errs entry is a malloc()ed, null terminated string. +// each non-NULL values_list entry is a malloc()ed array, with +// the length for each stored in values_list_sizes[i]. +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get( + rocksdb_t* db, const rocksdb_readoptions_t* options, size_t num_keys, + const char* const* keys_list, const size_t* keys_list_sizes, + char** values_list, size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, size_t num_keys, + const char* const* keys_list, const size_t* keys_list_sizes, + char** values_list, size_t* values_list_sizes, char** timestamp_list, + size_t* timestamp_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_cf_with_ts( + rocksdb_t* db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** timestamps_list, + size_t* timestamps_list_sizes, char** errs); + +// The MultiGet API that improves performance by batching operations +// in the read path for greater efficiency. Currently, only the block based +// table format with full filters are supported. Other table formats such +// as plain table, block based table with block based filters and +// partitioned indexes will still work, but will not get any performance +// benefits. +// +// Note that all the keys passed to this API are restricted to a single +// column family. +// +// Parameters - +// db - the RocksDB instance. +// options - ReadOptions +// column_family - ColumnFamilyHandle* that the keys belong to. All the keys +// passed to the API are restricted to a single column family +// num_keys - Number of keys to lookup +// keys_list - Pointer to C style array of keys with num_keys elements +// keys_list_sizes - Pointer to C style array of the size of corresponding key +// in key_list with num_keys elements. +// values - Pointer to C style array of PinnableSlices with num_keys elements +// statuses - Pointer to C style array of Status with num_keys elements +// sorted_input - If true, it means the input keys are already sorted by key +// order, so the MultiGet() API doesn't have to sort them +// again. If false, the keys will be copied and sorted +// internally by the API - the input array will not be +// modified +extern ROCKSDB_LIBRARY_API void rocksdb_batched_multi_get_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, size_t num_keys, + const char* const* keys_list, const size_t* keys_list_sizes, + rocksdb_pinnableslice_t** values, char** errs, const bool sorted_input); + +// The value is only allocated (using malloc) and returned if it is found and +// value_found isn't NULL. In that case the user is responsible for freeing it. +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_key_may_exist( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t key_len, char** value, size_t* val_len, const char* timestamp, + size_t timestamp_len, unsigned char* value_found); + +// The value is only allocated (using malloc) and returned if it is found and +// value_found isn't NULL. In that case the user is responsible for freeing it. +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_key_may_exist_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t key_len, char** value, size_t* val_len, const char* timestamp, + size_t timestamp_len, unsigned char* value_found); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* rocksdb_create_iterator( + rocksdb_t* db, const rocksdb_readoptions_t* options); + +extern ROCKSDB_LIBRARY_API rocksdb_wal_iterator_t* rocksdb_get_updates_since( + rocksdb_t* db, uint64_t seq_number, + const rocksdb_wal_readoptions_t* options, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* rocksdb_create_iterator_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API void rocksdb_create_iterators( + rocksdb_t* db, rocksdb_readoptions_t* opts, + rocksdb_column_family_handle_t** column_families, + rocksdb_iterator_t** iterators, size_t size, char** errptr); + +extern ROCKSDB_LIBRARY_API const rocksdb_snapshot_t* rocksdb_create_snapshot( + rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_release_snapshot( + rocksdb_t* db, const rocksdb_snapshot_t* snapshot); + +/* Returns NULL if property name is unknown. + Else returns a pointer to a malloc()-ed null-terminated value. */ +extern ROCKSDB_LIBRARY_API char* rocksdb_property_value(rocksdb_t* db, + const char* propname); +/* returns 0 on success, -1 otherwise */ +extern ROCKSDB_LIBRARY_API int rocksdb_property_int(rocksdb_t* db, + const char* propname, + uint64_t* out_val); + +/* returns 0 on success, -1 otherwise */ +extern ROCKSDB_LIBRARY_API int rocksdb_property_int_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* propname, uint64_t* out_val); + +extern ROCKSDB_LIBRARY_API char* rocksdb_property_value_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* propname); + +extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes( + rocksdb_t* db, int num_ranges, const char* const* range_start_key, + const size_t* range_start_key_len, const char* const* range_limit_key, + const size_t* range_limit_key_len, uint64_t* sizes, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + int num_ranges, const char* const* range_start_key, + const size_t* range_start_key_len, const char* const* range_limit_key, + const size_t* range_limit_key_len, uint64_t* sizes, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range(rocksdb_t* db, + const char* start_key, + size_t start_key_len, + const char* limit_key, + size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* limit_key, + size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_suggest_compact_range( + rocksdb_t* db, const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_suggest_compact_range_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* limit_key, + size_t limit_key_len, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range_opt( + rocksdb_t* db, rocksdb_compactoptions_t* opt, const char* start_key, + size_t start_key_len, const char* limit_key, size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_compact_range_cf_opt( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + rocksdb_compactoptions_t* opt, const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_file(rocksdb_t* db, + const char* name); + +extern ROCKSDB_LIBRARY_API const rocksdb_livefiles_t* rocksdb_livefiles( + rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush( + rocksdb_t* db, const rocksdb_flushoptions_t* options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush_cf( + rocksdb_t* db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t* column_family, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush_cfs( + rocksdb_t* db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t** column_family, int num_column_families, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_flush_wal(rocksdb_t* db, + unsigned char sync, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_disable_file_deletions(rocksdb_t* db, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_enable_file_deletions( + rocksdb_t* db, unsigned char force, char** errptr); + +/* Management operations */ + +extern ROCKSDB_LIBRARY_API void rocksdb_destroy_db( + const rocksdb_options_t* options, const char* name, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_repair_db( + const rocksdb_options_t* options, const char* name, char** errptr); + +/* Iterator */ + +extern ROCKSDB_LIBRARY_API void rocksdb_iter_destroy(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_iter_valid( + const rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek_to_first(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek_to_last(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek(rocksdb_iterator_t*, + const char* k, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_seek_for_prev(rocksdb_iterator_t*, + const char* k, + size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_next(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_prev(rocksdb_iterator_t*); +extern ROCKSDB_LIBRARY_API const char* rocksdb_iter_key( + const rocksdb_iterator_t*, size_t* klen); +extern ROCKSDB_LIBRARY_API const char* rocksdb_iter_value( + const rocksdb_iterator_t*, size_t* vlen); +extern ROCKSDB_LIBRARY_API const char* rocksdb_iter_timestamp( + const rocksdb_iterator_t*, size_t* tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_iter_get_error( + const rocksdb_iterator_t*, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_wal_iter_next( + rocksdb_wal_iterator_t* iter); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_wal_iter_valid( + const rocksdb_wal_iterator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_wal_iter_status( + const rocksdb_wal_iterator_t* iter, char** errptr); +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* rocksdb_wal_iter_get_batch( + const rocksdb_wal_iterator_t* iter, uint64_t* seq); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_get_latest_sequence_number(rocksdb_t* db); +extern ROCKSDB_LIBRARY_API void rocksdb_wal_iter_destroy( + const rocksdb_wal_iterator_t* iter); + +/* Write batch */ + +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* rocksdb_writebatch_create( + void); +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_t* rocksdb_writebatch_create_from( + const char* rep, size_t size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_destroy( + rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_clear(rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API int rocksdb_writebatch_count(rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put(rocksdb_writebatch_t*, + const char* key, + size_t klen, + const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put_cf( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put_cf_with_ts( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen, const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_putv( + rocksdb_writebatch_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_putv_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_merge(rocksdb_writebatch_t*, + const char* key, + size_t klen, + const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_merge_cf( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_mergev( + rocksdb_writebatch_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_mergev_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete(rocksdb_writebatch_t*, + const char* key, + size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_singledelete( + rocksdb_writebatch_t* b, const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_cf( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_cf_with_ts( + rocksdb_writebatch_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_singledelete_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_singledelete_cf_with_ts( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* ts, size_t tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev( + rocksdb_writebatch_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_deletev_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_range( + rocksdb_writebatch_t* b, const char* start_key, size_t start_key_len, + const char* end_key, size_t end_key_len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_range_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* end_key, + size_t end_key_len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_rangev( + rocksdb_writebatch_t* b, int num_keys, const char* const* start_keys_list, + const size_t* start_keys_list_sizes, const char* const* end_keys_list, + const size_t* end_keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_delete_rangev_cf( + rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* start_keys_list, + const size_t* start_keys_list_sizes, const char* const* end_keys_list, + const size_t* end_keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_put_log_data( + rocksdb_writebatch_t*, const char* blob, size_t len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_iterate( + rocksdb_writebatch_t*, void* state, + void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), + void (*deleted)(void*, const char* k, size_t klen)); +extern ROCKSDB_LIBRARY_API const char* rocksdb_writebatch_data( + rocksdb_writebatch_t*, size_t* size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_set_save_point( + rocksdb_writebatch_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_rollback_to_save_point( + rocksdb_writebatch_t*, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_pop_save_point( + rocksdb_writebatch_t*, char** errptr); + +/* Write batch with index */ + +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_wi_t* +rocksdb_writebatch_wi_create(size_t reserved_bytes, + unsigned char overwrite_keys); +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_wi_t* +rocksdb_writebatch_wi_create_from(const char* rep, size_t size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_destroy( + rocksdb_writebatch_wi_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_clear( + rocksdb_writebatch_wi_t*); +extern ROCKSDB_LIBRARY_API int rocksdb_writebatch_wi_count( + rocksdb_writebatch_wi_t* b); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_put( + rocksdb_writebatch_wi_t*, const char* key, size_t klen, const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_put_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_putv( + rocksdb_writebatch_wi_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_putv_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_merge( + rocksdb_writebatch_wi_t*, const char* key, size_t klen, const char* val, + size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_merge_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_mergev( + rocksdb_writebatch_wi_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, int num_values, + const char* const* values_list, const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_mergev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes, + int num_values, const char* const* values_list, + const size_t* values_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete( + rocksdb_writebatch_wi_t*, const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_singledelete( + rocksdb_writebatch_wi_t*, const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_singledelete_cf( + rocksdb_writebatch_wi_t*, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_deletev( + rocksdb_writebatch_wi_t* b, int num_keys, const char* const* keys_list, + const size_t* keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_deletev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* keys_list, const size_t* keys_list_sizes); +// DO NOT USE - rocksdb_writebatch_wi_delete_range is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_range( + rocksdb_writebatch_wi_t* b, const char* start_key, size_t start_key_len, + const char* end_key, size_t end_key_len); +// DO NOT USE - rocksdb_writebatch_wi_delete_range_cf is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_range_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* end_key, + size_t end_key_len); +// DO NOT USE - rocksdb_writebatch_wi_delete_rangev is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_rangev( + rocksdb_writebatch_wi_t* b, int num_keys, + const char* const* start_keys_list, const size_t* start_keys_list_sizes, + const char* const* end_keys_list, const size_t* end_keys_list_sizes); +// DO NOT USE - rocksdb_writebatch_wi_delete_rangev_cf is not yet supported +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_delete_rangev_cf( + rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family, + int num_keys, const char* const* start_keys_list, + const size_t* start_keys_list_sizes, const char* const* end_keys_list, + const size_t* end_keys_list_sizes); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_put_log_data( + rocksdb_writebatch_wi_t*, const char* blob, size_t len); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_iterate( + rocksdb_writebatch_wi_t* b, void* state, + void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), + void (*deleted)(void*, const char* k, size_t klen)); +extern ROCKSDB_LIBRARY_API const char* rocksdb_writebatch_wi_data( + rocksdb_writebatch_wi_t* b, size_t* size); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_set_save_point( + rocksdb_writebatch_wi_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writebatch_wi_rollback_to_save_point( + rocksdb_writebatch_wi_t*, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch( + rocksdb_writebatch_wi_t* wbwi, const rocksdb_options_t* options, + const char* key, size_t keylen, size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch_cf( + rocksdb_writebatch_wi_t* wbwi, const rocksdb_options_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch_and_db( + rocksdb_writebatch_wi_t* wbwi, rocksdb_t* db, + const rocksdb_readoptions_t* options, const char* key, size_t keylen, + size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API char* rocksdb_writebatch_wi_get_from_batch_and_db_cf( + rocksdb_writebatch_wi_t* wbwi, rocksdb_t* db, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_write_writebatch_wi( + rocksdb_t* db, const rocksdb_writeoptions_t* options, + rocksdb_writebatch_wi_t* wbwi, char** errptr); +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_writebatch_wi_create_iterator_with_base( + rocksdb_writebatch_wi_t* wbwi, rocksdb_iterator_t* base_iterator); +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_writebatch_wi_create_iterator_with_base_cf( + rocksdb_writebatch_wi_t* wbwi, rocksdb_iterator_t* base_iterator, + rocksdb_column_family_handle_t* cf); + +/* Options utils */ + +// Load the latest rocksdb options from the specified db_path. +// +// On success, num_column_families will be updated with a non-zero +// number indicating the number of column families. +// The returned db_options, column_family_names, and column_family_options +// should be released via rocksdb_load_latest_options_destroy(). +// +// On error, a non-null errptr that includes the error message will be +// returned. db_options, column_family_names, and column_family_options +// will be set to NULL. +extern ROCKSDB_LIBRARY_API void rocksdb_load_latest_options( + const char* db_path, rocksdb_env_t* env, bool ignore_unknown_options, + rocksdb_cache_t* cache, rocksdb_options_t** db_options, + size_t* num_column_families, char*** column_family_names, + rocksdb_options_t*** column_family_options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_load_latest_options_destroy( + rocksdb_options_t* db_options, char** list_column_family_names, + rocksdb_options_t** list_column_family_options, size_t len); + +/* Block based table options */ + +extern ROCKSDB_LIBRARY_API rocksdb_block_based_table_options_t* +rocksdb_block_based_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_destroy( + rocksdb_block_based_table_options_t* options); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_checksum( + rocksdb_block_based_table_options_t*, char); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_block_size( + rocksdb_block_based_table_options_t* options, size_t block_size); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_block_size_deviation( + rocksdb_block_based_table_options_t* options, int block_size_deviation); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_block_restart_interval( + rocksdb_block_based_table_options_t* options, int block_restart_interval); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_index_block_restart_interval( + rocksdb_block_based_table_options_t* options, + int index_block_restart_interval); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_metadata_block_size( + rocksdb_block_based_table_options_t* options, uint64_t metadata_block_size); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_partition_filters( + rocksdb_block_based_table_options_t* options, + unsigned char partition_filters); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_optimize_filters_for_memory( + rocksdb_block_based_table_options_t* options, + unsigned char optimize_filters_for_memory); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_use_delta_encoding( + rocksdb_block_based_table_options_t* options, + unsigned char use_delta_encoding); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_filter_policy( + rocksdb_block_based_table_options_t* options, + rocksdb_filterpolicy_t* filter_policy); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_no_block_cache( + rocksdb_block_based_table_options_t* options, unsigned char no_block_cache); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_block_cache( + rocksdb_block_based_table_options_t* options, rocksdb_cache_t* block_cache); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_whole_key_filtering( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_format_version( + rocksdb_block_based_table_options_t*, int); +enum { + rocksdb_block_based_table_index_type_binary_search = 0, + rocksdb_block_based_table_index_type_hash_search = 1, + rocksdb_block_based_table_index_type_two_level_index_search = 2, +}; +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_index_type( + rocksdb_block_based_table_options_t*, int); // uses one of the above enums +enum { + rocksdb_block_based_table_data_block_index_type_binary_search = 0, + rocksdb_block_based_table_data_block_index_type_binary_search_and_hash = 1, +}; +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_data_block_index_type( + rocksdb_block_based_table_options_t*, int); // uses one of the above enums +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_data_block_hash_ratio( + rocksdb_block_based_table_options_t* options, double v); +// rocksdb_block_based_options_set_hash_index_allow_collision() +// is removed since BlockBasedTableOptions.hash_index_allow_collision() +// is removed +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_cache_index_and_filter_blocks( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_pin_top_level_index_and_filter( + rocksdb_block_based_table_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_block_based_table_factory( + rocksdb_options_t* opt, rocksdb_block_based_table_options_t* table_options); + +/* Cuckoo table options */ + +extern ROCKSDB_LIBRARY_API rocksdb_cuckoo_table_options_t* +rocksdb_cuckoo_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_destroy( + rocksdb_cuckoo_table_options_t* options); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_hash_ratio( + rocksdb_cuckoo_table_options_t* options, double v); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_max_search_depth( + rocksdb_cuckoo_table_options_t* options, uint32_t v); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_cuckoo_block_size( + rocksdb_cuckoo_table_options_t* options, uint32_t v); +extern ROCKSDB_LIBRARY_API void +rocksdb_cuckoo_options_set_identity_as_first_hash( + rocksdb_cuckoo_table_options_t* options, unsigned char v); +extern ROCKSDB_LIBRARY_API void rocksdb_cuckoo_options_set_use_module_hash( + rocksdb_cuckoo_table_options_t* options, unsigned char v); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_cuckoo_table_factory( + rocksdb_options_t* opt, rocksdb_cuckoo_table_options_t* table_options); + +/* Options */ +extern ROCKSDB_LIBRARY_API void rocksdb_set_options(rocksdb_t* db, int count, + const char* const keys[], + const char* const values[], + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_set_options_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, int count, + const char* const keys[], const char* const values[], char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_options_t* rocksdb_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_options_destroy(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API rocksdb_options_t* rocksdb_options_create_copy( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_increase_parallelism( + rocksdb_options_t* opt, int total_threads); +extern ROCKSDB_LIBRARY_API void rocksdb_options_optimize_for_point_lookup( + rocksdb_options_t* opt, uint64_t block_cache_size_mb); +extern ROCKSDB_LIBRARY_API void rocksdb_options_optimize_level_style_compaction( + rocksdb_options_t* opt, uint64_t memtable_memory_budget); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_optimize_universal_style_compaction( + rocksdb_options_t* opt, uint64_t memtable_memory_budget); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_allow_ingest_behind( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_allow_ingest_behind(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compaction_filter( + rocksdb_options_t*, rocksdb_compactionfilter_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compaction_filter_factory( + rocksdb_options_t*, rocksdb_compactionfilterfactory_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_compaction_readahead_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_compaction_readahead_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_comparator( + rocksdb_options_t*, rocksdb_comparator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_merge_operator( + rocksdb_options_t*, rocksdb_mergeoperator_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_uint64add_merge_operator( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compression_per_level( + rocksdb_options_t* opt, const int* level_values, size_t num_levels); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_create_if_missing( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_create_if_missing( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_create_missing_column_families(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_create_missing_column_families(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_error_if_exists( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_error_if_exists( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_paranoid_checks( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_paranoid_checks( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_db_paths( + rocksdb_options_t*, const rocksdb_dbpath_t** path_values, size_t num_paths); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_env(rocksdb_options_t*, + rocksdb_env_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_info_log(rocksdb_options_t*, + rocksdb_logger_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_info_log_level( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_info_log_level( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_write_buffer_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_write_buffer_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_db_write_buffer_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_db_write_buffer_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_open_files( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_open_files( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_file_opening_threads( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_file_opening_threads( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_total_wal_size( + rocksdb_options_t* opt, uint64_t n); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_total_wal_size(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compression_options( + rocksdb_options_t*, int, int, int, int); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_zstd_max_train_bytes(rocksdb_options_t*, + int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_compression_options_zstd_max_train_bytes( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_use_zstd_dict_trainer( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_compression_options_use_zstd_dict_trainer( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_parallel_threads(rocksdb_options_t*, + int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_compression_options_parallel_threads( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_max_dict_buffer_bytes( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_compression_options_max_dict_buffer_bytes( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options(rocksdb_options_t*, int, int, + int, int, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options_zstd_max_train_bytes( + rocksdb_options_t*, int, unsigned char); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options_use_zstd_dict_trainer( + rocksdb_options_t*, unsigned char, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_bottommost_compression_options_use_zstd_dict_trainer( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options_max_dict_buffer_bytes( + rocksdb_options_t*, uint64_t, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_prefix_extractor( + rocksdb_options_t*, rocksdb_slicetransform_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_num_levels( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_num_levels( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_level0_file_num_compaction_trigger(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_level0_file_num_compaction_trigger(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_level0_slowdown_writes_trigger(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_level0_slowdown_writes_trigger(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_level0_stop_writes_trigger( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_level0_stop_writes_trigger( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_target_file_size_base( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_target_file_size_base(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_target_file_size_multiplier( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_target_file_size_multiplier( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_bytes_for_level_base( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_bytes_for_level_base(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_level_compaction_dynamic_level_bytes(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_level_compaction_dynamic_level_bytes(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_bytes_for_level_multiplier(rocksdb_options_t*, double); +extern ROCKSDB_LIBRARY_API double +rocksdb_options_get_max_bytes_for_level_multiplier(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_bytes_for_level_multiplier_additional( + rocksdb_options_t*, int* level_values, size_t num_levels); +extern ROCKSDB_LIBRARY_API void rocksdb_options_enable_statistics( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt, + unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_skip_stats_update_on_db_open(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open( + rocksdb_options_t* opt, unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open( + rocksdb_options_t* opt); + +/* Blob Options Settings */ +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_blob_files( + rocksdb_options_t* opt, unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_enable_blob_files( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_min_blob_size( + rocksdb_options_t* opt, uint64_t val); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_min_blob_size(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_file_size( + rocksdb_options_t* opt, uint64_t val); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_blob_file_size(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_compression_type( + rocksdb_options_t* opt, int val); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_blob_compression_type( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_blob_gc( + rocksdb_options_t* opt, unsigned char val); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_enable_blob_gc( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_gc_age_cutoff( + rocksdb_options_t* opt, double val); +extern ROCKSDB_LIBRARY_API double rocksdb_options_get_blob_gc_age_cutoff( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_gc_force_threshold( + rocksdb_options_t* opt, double val); +extern ROCKSDB_LIBRARY_API double rocksdb_options_get_blob_gc_force_threshold( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_blob_compaction_readahead_size(rocksdb_options_t* opt, + uint64_t val); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_blob_compaction_readahead_size(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_file_starting_level( + rocksdb_options_t* opt, int val); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_blob_file_starting_level( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_cache( + rocksdb_options_t* opt, rocksdb_cache_t* blob_cache); + +enum { + rocksdb_prepopulate_blob_disable = 0, + rocksdb_prepopulate_blob_flush_only = 1 +}; + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_prepopulate_blob_cache( + rocksdb_options_t* opt, int val); + +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_prepopulate_blob_cache( + rocksdb_options_t* opt); + +/* returns a pointer to a malloc()-ed, null terminated string */ +extern ROCKSDB_LIBRARY_API char* rocksdb_options_statistics_get_string( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_write_buffer_number( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_write_buffer_number( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_min_write_buffer_number_to_merge(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_min_write_buffer_number_to_merge(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_write_buffer_number_to_maintain(rocksdb_options_t*, + int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_max_write_buffer_number_to_maintain(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_write_buffer_size_to_maintain(rocksdb_options_t*, + int64_t); +extern ROCKSDB_LIBRARY_API int64_t +rocksdb_options_get_max_write_buffer_size_to_maintain(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_pipelined_write( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_enable_pipelined_write(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_unordered_write( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_unordered_write( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_subcompactions( + rocksdb_options_t*, uint32_t); +extern ROCKSDB_LIBRARY_API uint32_t +rocksdb_options_get_max_subcompactions(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_background_jobs( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_background_jobs( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_background_compactions( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_background_compactions( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_background_flushes( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_max_background_flushes( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_log_file_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_max_log_file_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_log_file_time_to_roll( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_log_file_time_to_roll(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_keep_log_file_num( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_keep_log_file_num(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_recycle_log_file_num( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_recycle_log_file_num(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_soft_pending_compaction_bytes_limit(rocksdb_options_t* opt, + size_t v); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_soft_pending_compaction_bytes_limit(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_hard_pending_compaction_bytes_limit(rocksdb_options_t* opt, + size_t v); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_hard_pending_compaction_bytes_limit(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_manifest_file_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_max_manifest_file_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_table_cache_numshardbits( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_table_cache_numshardbits( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_arena_block_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_arena_block_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_use_fsync( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_use_fsync( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_db_log_dir( + rocksdb_options_t*, const char*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_dir(rocksdb_options_t*, + const char*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_WAL_ttl_seconds( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_WAL_ttl_seconds(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_WAL_size_limit_MB( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_WAL_size_limit_MB(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_manifest_preallocation_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_manifest_preallocation_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_allow_mmap_reads( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_allow_mmap_reads( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_allow_mmap_writes( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_allow_mmap_writes( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_use_direct_reads( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_use_direct_reads( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_use_direct_io_for_flush_and_compaction(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_use_direct_io_for_flush_and_compaction(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_is_fd_close_on_exec( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_is_fd_close_on_exec(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_stats_dump_period_sec( + rocksdb_options_t*, unsigned int); +extern ROCKSDB_LIBRARY_API unsigned int +rocksdb_options_get_stats_dump_period_sec(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_stats_persist_period_sec( + rocksdb_options_t*, unsigned int); +extern ROCKSDB_LIBRARY_API unsigned int +rocksdb_options_get_stats_persist_period_sec(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_advise_random_on_open( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_advise_random_on_open(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_access_hint_on_compaction_start(rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_options_get_access_hint_on_compaction_start(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_use_adaptive_mutex( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_use_adaptive_mutex( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bytes_per_sync( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_bytes_per_sync(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_bytes_per_sync( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_wal_bytes_per_sync(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_writable_file_max_buffer_size(rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_writable_file_max_buffer_size(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_allow_concurrent_memtable_write(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_allow_concurrent_memtable_write(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_enable_write_thread_adaptive_yield(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_enable_write_thread_adaptive_yield(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_max_sequential_skip_in_iterations(rocksdb_options_t*, + uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_sequential_skip_in_iterations(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_disable_auto_compactions( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_disable_auto_compactions(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_optimize_filters_for_hits( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_optimize_filters_for_hits(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_delete_obsolete_files_period_micros(rocksdb_options_t*, + uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_delete_obsolete_files_period_micros(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_prepare_for_bulk_load( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_memtable_vector_rep( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_memtable_prefix_bloom_size_ratio(rocksdb_options_t*, + double); +extern ROCKSDB_LIBRARY_API double +rocksdb_options_get_memtable_prefix_bloom_size_ratio(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_compaction_bytes( + rocksdb_options_t*, uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_max_compaction_bytes(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_hash_skip_list_rep( + rocksdb_options_t*, size_t, int32_t, int32_t); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_hash_link_list_rep( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_plain_table_factory( + rocksdb_options_t*, uint32_t, int, double, size_t); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_min_level_to_compress( + rocksdb_options_t* opt, int level); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_memtable_huge_page_size( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_memtable_huge_page_size(rocksdb_options_t*); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_successive_merges( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_max_successive_merges(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bloom_locality( + rocksdb_options_t*, uint32_t); +extern ROCKSDB_LIBRARY_API uint32_t +rocksdb_options_get_bloom_locality(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_inplace_update_support( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_inplace_update_support(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_inplace_update_num_locks( + rocksdb_options_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_options_get_inplace_update_num_locks(rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_report_bg_io_stats( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_report_bg_io_stats( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_avoid_unnecessary_blocking_io(rocksdb_options_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_avoid_unnecessary_blocking_io(rocksdb_options_t*); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_experimental_mempurge_threshold(rocksdb_options_t*, double); +extern ROCKSDB_LIBRARY_API double +rocksdb_options_get_experimental_mempurge_threshold(rocksdb_options_t*); + +enum { + rocksdb_tolerate_corrupted_tail_records_recovery = 0, + rocksdb_absolute_consistency_recovery = 1, + rocksdb_point_in_time_recovery = 2, + rocksdb_skip_any_corrupted_records_recovery = 3 +}; +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_recovery_mode( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_wal_recovery_mode( + rocksdb_options_t*); + +enum { + rocksdb_no_compression = 0, + rocksdb_snappy_compression = 1, + rocksdb_zlib_compression = 2, + rocksdb_bz2_compression = 3, + rocksdb_lz4_compression = 4, + rocksdb_lz4hc_compression = 5, + rocksdb_xpress_compression = 6, + rocksdb_zstd_compression = 7 +}; +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compression( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_compression( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bottommost_compression( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_bottommost_compression( + rocksdb_options_t*); + +enum { + rocksdb_level_compaction = 0, + rocksdb_universal_compaction = 1, + rocksdb_fifo_compaction = 2 +}; +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compaction_style( + rocksdb_options_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_compaction_style( + rocksdb_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_universal_compaction_options( + rocksdb_options_t*, rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_fifo_compaction_options( + rocksdb_options_t* opt, rocksdb_fifo_compaction_options_t* fifo); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_ratelimiter( + rocksdb_options_t* opt, rocksdb_ratelimiter_t* limiter); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_atomic_flush( + rocksdb_options_t* opt, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_atomic_flush( + rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_row_cache( + rocksdb_options_t* opt, rocksdb_cache_t* cache); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_add_compact_on_deletion_collector_factory( + rocksdb_options_t*, size_t window_size, size_t num_dels_trigger); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_manual_wal_flush( + rocksdb_options_t* opt, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_manual_wal_flush( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_wal_compression( + rocksdb_options_t* opt, int); +extern ROCKSDB_LIBRARY_API int rocksdb_options_get_wal_compression( + rocksdb_options_t* opt); + +/* RateLimiter */ +extern ROCKSDB_LIBRARY_API rocksdb_ratelimiter_t* rocksdb_ratelimiter_create( + int64_t rate_bytes_per_sec, int64_t refill_period_us, int32_t fairness); +extern ROCKSDB_LIBRARY_API void rocksdb_ratelimiter_destroy( + rocksdb_ratelimiter_t*); + +/* PerfContext */ +enum { + rocksdb_uninitialized = 0, + rocksdb_disable = 1, + rocksdb_enable_count = 2, + rocksdb_enable_time_except_for_mutex = 3, + rocksdb_enable_time = 4, + rocksdb_out_of_bounds = 5 +}; + +enum { + rocksdb_user_key_comparison_count = 0, + rocksdb_block_cache_hit_count, + rocksdb_block_read_count, + rocksdb_block_read_byte, + rocksdb_block_read_time, + rocksdb_block_checksum_time, + rocksdb_block_decompress_time, + rocksdb_get_read_bytes, + rocksdb_multiget_read_bytes, + rocksdb_iter_read_bytes, + rocksdb_internal_key_skipped_count, + rocksdb_internal_delete_skipped_count, + rocksdb_internal_recent_skipped_count, + rocksdb_internal_merge_count, + rocksdb_get_snapshot_time, + rocksdb_get_from_memtable_time, + rocksdb_get_from_memtable_count, + rocksdb_get_post_process_time, + rocksdb_get_from_output_files_time, + rocksdb_seek_on_memtable_time, + rocksdb_seek_on_memtable_count, + rocksdb_next_on_memtable_count, + rocksdb_prev_on_memtable_count, + rocksdb_seek_child_seek_time, + rocksdb_seek_child_seek_count, + rocksdb_seek_min_heap_time, + rocksdb_seek_max_heap_time, + rocksdb_seek_internal_seek_time, + rocksdb_find_next_user_entry_time, + rocksdb_write_wal_time, + rocksdb_write_memtable_time, + rocksdb_write_delay_time, + rocksdb_write_pre_and_post_process_time, + rocksdb_db_mutex_lock_nanos, + rocksdb_db_condition_wait_nanos, + rocksdb_merge_operator_time_nanos, + rocksdb_read_index_block_nanos, + rocksdb_read_filter_block_nanos, + rocksdb_new_table_block_iter_nanos, + rocksdb_new_table_iterator_nanos, + rocksdb_block_seek_nanos, + rocksdb_find_table_nanos, + rocksdb_bloom_memtable_hit_count, + rocksdb_bloom_memtable_miss_count, + rocksdb_bloom_sst_hit_count, + rocksdb_bloom_sst_miss_count, + rocksdb_key_lock_wait_time, + rocksdb_key_lock_wait_count, + rocksdb_env_new_sequential_file_nanos, + rocksdb_env_new_random_access_file_nanos, + rocksdb_env_new_writable_file_nanos, + rocksdb_env_reuse_writable_file_nanos, + rocksdb_env_new_random_rw_file_nanos, + rocksdb_env_new_directory_nanos, + rocksdb_env_file_exists_nanos, + rocksdb_env_get_children_nanos, + rocksdb_env_get_children_file_attributes_nanos, + rocksdb_env_delete_file_nanos, + rocksdb_env_create_dir_nanos, + rocksdb_env_create_dir_if_missing_nanos, + rocksdb_env_delete_dir_nanos, + rocksdb_env_get_file_size_nanos, + rocksdb_env_get_file_modification_time_nanos, + rocksdb_env_rename_file_nanos, + rocksdb_env_link_file_nanos, + rocksdb_env_lock_file_nanos, + rocksdb_env_unlock_file_nanos, + rocksdb_env_new_logger_nanos, + rocksdb_number_async_seek, + rocksdb_blob_cache_hit_count, + rocksdb_blob_read_count, + rocksdb_blob_read_byte, + rocksdb_blob_read_time, + rocksdb_blob_checksum_time, + rocksdb_blob_decompress_time, + rocksdb_internal_range_del_reseek_count, + rocksdb_total_metric_count = 78 +}; + +extern ROCKSDB_LIBRARY_API void rocksdb_set_perf_level(int); +extern ROCKSDB_LIBRARY_API rocksdb_perfcontext_t* rocksdb_perfcontext_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_perfcontext_reset( + rocksdb_perfcontext_t* context); +extern ROCKSDB_LIBRARY_API char* rocksdb_perfcontext_report( + rocksdb_perfcontext_t* context, unsigned char exclude_zero_counters); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_perfcontext_metric(rocksdb_perfcontext_t* context, int metric); +extern ROCKSDB_LIBRARY_API void rocksdb_perfcontext_destroy( + rocksdb_perfcontext_t* context); + +/* Compaction Filter */ + +extern ROCKSDB_LIBRARY_API rocksdb_compactionfilter_t* +rocksdb_compactionfilter_create( + void* state, void (*destructor)(void*), + unsigned char (*filter)(void*, int level, const char* key, + size_t key_length, const char* existing_value, + size_t value_length, char** new_value, + size_t* new_value_length, + unsigned char* value_changed), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_compactionfilter_set_ignore_snapshots( + rocksdb_compactionfilter_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_compactionfilter_destroy( + rocksdb_compactionfilter_t*); + +/* Compaction Filter Context */ + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactionfiltercontext_is_full_compaction( + rocksdb_compactionfiltercontext_t* context); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactionfiltercontext_is_manual_compaction( + rocksdb_compactionfiltercontext_t* context); + +/* Compaction Filter Factory */ + +extern ROCKSDB_LIBRARY_API rocksdb_compactionfilterfactory_t* +rocksdb_compactionfilterfactory_create( + void* state, void (*destructor)(void*), + rocksdb_compactionfilter_t* (*create_compaction_filter)( + void*, rocksdb_compactionfiltercontext_t* context), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_compactionfilterfactory_destroy( + rocksdb_compactionfilterfactory_t*); + +/* Comparator */ + +extern ROCKSDB_LIBRARY_API rocksdb_comparator_t* rocksdb_comparator_create( + void* state, void (*destructor)(void*), + int (*compare)(void*, const char* a, size_t alen, const char* b, + size_t blen), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_comparator_destroy( + rocksdb_comparator_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_comparator_t* +rocksdb_comparator_with_ts_create( + void* state, void (*destructor)(void*), + int (*compare)(void*, const char* a, size_t alen, const char* b, + size_t blen), + int (*compare_ts)(void*, const char* a_ts, size_t a_tslen, const char* b_ts, + size_t b_tslen), + int (*compare_without_ts)(void*, const char* a, size_t alen, + unsigned char a_has_ts, const char* b, + size_t blen, unsigned char b_has_ts), + const char* (*name)(void*), size_t timestamp_size); + +/* Filter policy */ + +extern ROCKSDB_LIBRARY_API void rocksdb_filterpolicy_destroy( + rocksdb_filterpolicy_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_bloom(double bits_per_key); +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_bloom_full(double bits_per_key); +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_ribbon(double bloom_equivalent_bits_per_key); +extern ROCKSDB_LIBRARY_API rocksdb_filterpolicy_t* +rocksdb_filterpolicy_create_ribbon_hybrid(double bloom_equivalent_bits_per_key, + int bloom_before_level); + +/* Merge Operator */ + +extern ROCKSDB_LIBRARY_API rocksdb_mergeoperator_t* +rocksdb_mergeoperator_create( + void* state, void (*destructor)(void*), + char* (*full_merge)(void*, const char* key, size_t key_length, + const char* existing_value, + size_t existing_value_length, + const char* const* operands_list, + const size_t* operands_list_length, int num_operands, + unsigned char* success, size_t* new_value_length), + char* (*partial_merge)(void*, const char* key, size_t key_length, + const char* const* operands_list, + const size_t* operands_list_length, int num_operands, + unsigned char* success, size_t* new_value_length), + void (*delete_value)(void*, const char* value, size_t value_length), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API void rocksdb_mergeoperator_destroy( + rocksdb_mergeoperator_t*); + +/* Read options */ + +extern ROCKSDB_LIBRARY_API rocksdb_readoptions_t* rocksdb_readoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_destroy( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_verify_checksums( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_verify_checksums(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_fill_cache( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_fill_cache( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_snapshot( + rocksdb_readoptions_t*, const rocksdb_snapshot_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iterate_upper_bound( + rocksdb_readoptions_t*, const char* key, size_t keylen); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iterate_lower_bound( + rocksdb_readoptions_t*, const char* key, size_t keylen); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_read_tier( + rocksdb_readoptions_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_readoptions_get_read_tier( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_tailing( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_tailing( + rocksdb_readoptions_t*); +// The functionality that this option controlled has been removed. +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_managed( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_readahead_size( + rocksdb_readoptions_t*, size_t); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_readoptions_get_readahead_size(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_prefix_same_as_start( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_prefix_same_as_start(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_pin_data( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_pin_data( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_total_order_seek( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_total_order_seek(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_readoptions_set_max_skippable_internal_keys(rocksdb_readoptions_t*, + uint64_t); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_readoptions_get_max_skippable_internal_keys(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_readoptions_set_background_purge_on_iterator_cleanup( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_background_purge_on_iterator_cleanup( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_ignore_range_deletions( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_readoptions_get_ignore_range_deletions(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_deadline( + rocksdb_readoptions_t*, uint64_t microseconds); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_readoptions_get_deadline(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_io_timeout( + rocksdb_readoptions_t*, uint64_t microseconds); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_readoptions_get_io_timeout(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_async_io( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_async_io( + rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_timestamp( + rocksdb_readoptions_t*, const char* ts, size_t tslen); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iter_start_ts( + rocksdb_readoptions_t*, const char* ts, size_t tslen); + +/* Write options */ + +extern ROCKSDB_LIBRARY_API rocksdb_writeoptions_t* rocksdb_writeoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_destroy( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_set_sync( + rocksdb_writeoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_sync( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_disable_WAL( + rocksdb_writeoptions_t* opt, int disable); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_disable_WAL( + rocksdb_writeoptions_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_writeoptions_set_ignore_missing_column_families(rocksdb_writeoptions_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_writeoptions_get_ignore_missing_column_families( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_set_no_slowdown( + rocksdb_writeoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_no_slowdown( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_writeoptions_set_low_pri( + rocksdb_writeoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_writeoptions_get_low_pri( + rocksdb_writeoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_writeoptions_set_memtable_insert_hint_per_batch(rocksdb_writeoptions_t*, + unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_writeoptions_get_memtable_insert_hint_per_batch( + rocksdb_writeoptions_t*); + +/* Compact range options */ + +extern ROCKSDB_LIBRARY_API rocksdb_compactoptions_t* +rocksdb_compactoptions_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_destroy( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_compactoptions_set_exclusive_manual_compaction( + rocksdb_compactoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactoptions_get_exclusive_manual_compaction( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_compactoptions_set_bottommost_level_compaction( + rocksdb_compactoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactoptions_get_bottommost_level_compaction( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_set_change_level( + rocksdb_compactoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_compactoptions_get_change_level(rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_set_target_level( + rocksdb_compactoptions_t*, int); +extern ROCKSDB_LIBRARY_API int rocksdb_compactoptions_get_target_level( + rocksdb_compactoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_compactoptions_set_full_history_ts_low( + rocksdb_compactoptions_t*, char* ts, size_t tslen); + +/* Flush options */ + +extern ROCKSDB_LIBRARY_API rocksdb_flushoptions_t* rocksdb_flushoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_flushoptions_destroy( + rocksdb_flushoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_flushoptions_set_wait( + rocksdb_flushoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_flushoptions_get_wait( + rocksdb_flushoptions_t*); + +/* Memory allocator */ + +extern ROCKSDB_LIBRARY_API rocksdb_memory_allocator_t* +rocksdb_jemalloc_nodump_allocator_create(char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_allocator_destroy( + rocksdb_memory_allocator_t*); + +/* Cache */ + +extern ROCKSDB_LIBRARY_API rocksdb_lru_cache_options_t* +rocksdb_lru_cache_options_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_destroy( + rocksdb_lru_cache_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_capacity( + rocksdb_lru_cache_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_num_shard_bits( + rocksdb_lru_cache_options_t*, int); +extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_memory_allocator( + rocksdb_lru_cache_options_t*, rocksdb_memory_allocator_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_lru( + size_t capacity); +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* +rocksdb_cache_create_lru_with_strict_capacity_limit(size_t capacity); +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_lru_opts( + rocksdb_lru_cache_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_cache_destroy(rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API void rocksdb_cache_disown_data( + rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API void rocksdb_cache_set_capacity( + rocksdb_cache_t* cache, size_t capacity); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_cache_get_capacity(rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_cache_get_usage(rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_cache_get_pinned_usage(rocksdb_cache_t* cache); + +/* HyperClockCache */ +extern ROCKSDB_LIBRARY_API rocksdb_hyper_clock_cache_options_t* +rocksdb_hyper_clock_cache_options_create(size_t capacity, + size_t estimated_entry_charge); +extern ROCKSDB_LIBRARY_API void rocksdb_hyper_clock_cache_options_destroy( + rocksdb_hyper_clock_cache_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_hyper_clock_cache_options_set_capacity( + rocksdb_hyper_clock_cache_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void +rocksdb_hyper_clock_cache_options_set_estimated_entry_charge( + rocksdb_hyper_clock_cache_options_t*, size_t); +extern ROCKSDB_LIBRARY_API void +rocksdb_hyper_clock_cache_options_set_num_shard_bits( + rocksdb_hyper_clock_cache_options_t*, int); +extern ROCKSDB_LIBRARY_API void +rocksdb_hyper_clock_cache_options_set_memory_allocator( + rocksdb_hyper_clock_cache_options_t*, rocksdb_memory_allocator_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_hyper_clock( + size_t capacity, size_t estimated_entry_charge); +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* +rocksdb_cache_create_hyper_clock_opts(rocksdb_hyper_clock_cache_options_t*); + +/* DBPath */ + +extern ROCKSDB_LIBRARY_API rocksdb_dbpath_t* rocksdb_dbpath_create( + const char* path, uint64_t target_size); +extern ROCKSDB_LIBRARY_API void rocksdb_dbpath_destroy(rocksdb_dbpath_t*); + +/* Env */ + +extern ROCKSDB_LIBRARY_API rocksdb_env_t* rocksdb_create_default_env(void); +extern ROCKSDB_LIBRARY_API rocksdb_env_t* rocksdb_create_mem_env(void); +extern ROCKSDB_LIBRARY_API void rocksdb_env_set_background_threads( + rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int rocksdb_env_get_background_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_set_high_priority_background_threads(rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int rocksdb_env_get_high_priority_background_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_set_low_priority_background_threads( + rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int rocksdb_env_get_low_priority_background_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_set_bottom_priority_background_threads(rocksdb_env_t* env, int n); +extern ROCKSDB_LIBRARY_API int +rocksdb_env_get_bottom_priority_background_threads(rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_join_all_threads( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_lower_thread_pool_io_priority( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_lower_high_priority_thread_pool_io_priority(rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void rocksdb_env_lower_thread_pool_cpu_priority( + rocksdb_env_t* env); +extern ROCKSDB_LIBRARY_API void +rocksdb_env_lower_high_priority_thread_pool_cpu_priority(rocksdb_env_t* env); + +extern ROCKSDB_LIBRARY_API void rocksdb_env_destroy(rocksdb_env_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_envoptions_t* rocksdb_envoptions_create( + void); +extern ROCKSDB_LIBRARY_API void rocksdb_envoptions_destroy( + rocksdb_envoptions_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_create_dir_if_missing( + rocksdb_env_t* env, const char* path, char** errptr); + +/* SstFile */ + +extern ROCKSDB_LIBRARY_API rocksdb_sstfilewriter_t* +rocksdb_sstfilewriter_create(const rocksdb_envoptions_t* env, + const rocksdb_options_t* io_options); +extern ROCKSDB_LIBRARY_API rocksdb_sstfilewriter_t* +rocksdb_sstfilewriter_create_with_comparator( + const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options, + const rocksdb_comparator_t* comparator); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_open( + rocksdb_sstfilewriter_t* writer, const char* name, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_add( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* val, size_t vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_put( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* val, size_t vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_put_with_ts( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* ts, size_t tslen, const char* val, size_t vallen, + char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_merge( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* val, size_t vallen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete_with_ts( + rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen, + const char* ts, size_t tslen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete_range( + rocksdb_sstfilewriter_t* writer, const char* begin_key, size_t begin_keylen, + const char* end_key, size_t end_keylen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_finish( + rocksdb_sstfilewriter_t* writer, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_file_size( + rocksdb_sstfilewriter_t* writer, uint64_t* file_size); +extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_destroy( + rocksdb_sstfilewriter_t* writer); +extern ROCKSDB_LIBRARY_API rocksdb_ingestexternalfileoptions_t* +rocksdb_ingestexternalfileoptions_create(void); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_move_files( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char move_files); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_snapshot_consistency( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char snapshot_consistency); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_allow_global_seqno( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char allow_global_seqno); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_allow_blocking_flush( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char allow_blocking_flush); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_ingest_behind( + rocksdb_ingestexternalfileoptions_t* opt, unsigned char ingest_behind); +extern ROCKSDB_LIBRARY_API void +rocksdb_ingestexternalfileoptions_set_fail_if_not_bottommost_level( + rocksdb_ingestexternalfileoptions_t* opt, + unsigned char fail_if_not_bottommost_level); + +extern ROCKSDB_LIBRARY_API void rocksdb_ingestexternalfileoptions_destroy( + rocksdb_ingestexternalfileoptions_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_ingest_external_file( + rocksdb_t* db, const char* const* file_list, const size_t list_len, + const rocksdb_ingestexternalfileoptions_t* opt, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_ingest_external_file_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* handle, + const char* const* file_list, const size_t list_len, + const rocksdb_ingestexternalfileoptions_t* opt, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_try_catch_up_with_primary( + rocksdb_t* db, char** errptr); + +/* SliceTransform */ + +extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t* +rocksdb_slicetransform_create( + void* state, void (*destructor)(void*), + char* (*transform)(void*, const char* key, size_t length, + size_t* dst_length), + unsigned char (*in_domain)(void*, const char* key, size_t length), + unsigned char (*in_range)(void*, const char* key, size_t length), + const char* (*name)(void*)); +extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t* + rocksdb_slicetransform_create_fixed_prefix(size_t); +extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t* +rocksdb_slicetransform_create_noop(void); +extern ROCKSDB_LIBRARY_API void rocksdb_slicetransform_destroy( + rocksdb_slicetransform_t*); + +/* Universal Compaction options */ + +enum { + rocksdb_similar_size_compaction_stop_style = 0, + rocksdb_total_size_compaction_stop_style = 1 +}; + +extern ROCKSDB_LIBRARY_API rocksdb_universal_compaction_options_t* +rocksdb_universal_compaction_options_create(void); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_size_ratio( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_size_ratio( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_min_merge_width( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_min_merge_width( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_max_merge_width( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_max_merge_width( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_max_size_amplification_percent( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_max_size_amplification_percent( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_compression_size_percent( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_compression_size_percent( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void +rocksdb_universal_compaction_options_set_stop_style( + rocksdb_universal_compaction_options_t*, int); +extern ROCKSDB_LIBRARY_API int +rocksdb_universal_compaction_options_get_stop_style( + rocksdb_universal_compaction_options_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_universal_compaction_options_destroy( + rocksdb_universal_compaction_options_t*); + +extern ROCKSDB_LIBRARY_API rocksdb_fifo_compaction_options_t* +rocksdb_fifo_compaction_options_create(void); +extern ROCKSDB_LIBRARY_API void +rocksdb_fifo_compaction_options_set_allow_compaction( + rocksdb_fifo_compaction_options_t* fifo_opts, unsigned char allow_compaction); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_fifo_compaction_options_get_allow_compaction( + rocksdb_fifo_compaction_options_t* fifo_opts); +extern ROCKSDB_LIBRARY_API void +rocksdb_fifo_compaction_options_set_max_table_files_size( + rocksdb_fifo_compaction_options_t* fifo_opts, uint64_t size); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_fifo_compaction_options_get_max_table_files_size( + rocksdb_fifo_compaction_options_t* fifo_opts); +extern ROCKSDB_LIBRARY_API void rocksdb_fifo_compaction_options_destroy( + rocksdb_fifo_compaction_options_t* fifo_opts); + +extern ROCKSDB_LIBRARY_API int rocksdb_livefiles_count( + const rocksdb_livefiles_t*); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_column_family_name( + const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_name( + const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API int rocksdb_livefiles_level( + const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API size_t +rocksdb_livefiles_size(const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_smallestkey( + const rocksdb_livefiles_t*, int index, size_t* size); +extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_largestkey( + const rocksdb_livefiles_t*, int index, size_t* size); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_livefiles_entries(const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_livefiles_deletions(const rocksdb_livefiles_t*, int index); +extern ROCKSDB_LIBRARY_API void rocksdb_livefiles_destroy( + const rocksdb_livefiles_t*); + +/* Utility Helpers */ + +extern ROCKSDB_LIBRARY_API void rocksdb_get_options_from_string( + const rocksdb_options_t* base_options, const char* opts_str, + rocksdb_options_t* new_options, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_file_in_range( + rocksdb_t* db, const char* start_key, size_t start_key_len, + const char* limit_key, size_t limit_key_len, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_delete_file_in_range_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family, + const char* start_key, size_t start_key_len, const char* limit_key, + size_t limit_key_len, char** errptr); + +/* MetaData */ + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_metadata_t* +rocksdb_get_column_family_metadata(rocksdb_t* db); + +/** + * Returns the rocksdb_column_family_metadata_t of the specified + * column family. + * + * Note that the caller is responsible to release the returned memory + * using rocksdb_column_family_metadata_destroy. + */ +extern ROCKSDB_LIBRARY_API rocksdb_column_family_metadata_t* +rocksdb_get_column_family_metadata_cf( + rocksdb_t* db, rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API void rocksdb_column_family_metadata_destroy( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API uint64_t rocksdb_column_family_metadata_get_size( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API size_t rocksdb_column_family_metadata_get_file_count( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API char* rocksdb_column_family_metadata_get_name( + rocksdb_column_family_metadata_t* cf_meta); + +extern ROCKSDB_LIBRARY_API size_t +rocksdb_column_family_metadata_get_level_count( + rocksdb_column_family_metadata_t* cf_meta); + +/** + * Returns the rocksdb_level_metadata_t of the ith level from the specified + * column family metadata. + * + * If the specified i is greater than or equal to the number of levels + * in the specified column family, then NULL will be returned. + * + * Note that the caller is responsible to release the returned memory + * using rocksdb_level_metadata_destroy before releasing its parent + * rocksdb_column_family_metadata_t. + */ +extern ROCKSDB_LIBRARY_API rocksdb_level_metadata_t* +rocksdb_column_family_metadata_get_level_metadata( + rocksdb_column_family_metadata_t* cf_meta, size_t i); + +/** + * Releases the specified rocksdb_level_metadata_t. + * + * Note that the specified rocksdb_level_metadata_t must be released + * before the release of its parent rocksdb_column_family_metadata_t. + */ +extern ROCKSDB_LIBRARY_API void rocksdb_level_metadata_destroy( + rocksdb_level_metadata_t* level_meta); + +extern ROCKSDB_LIBRARY_API int rocksdb_level_metadata_get_level( + rocksdb_level_metadata_t* level_meta); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_level_metadata_get_size(rocksdb_level_metadata_t* level_meta); + +extern ROCKSDB_LIBRARY_API size_t +rocksdb_level_metadata_get_file_count(rocksdb_level_metadata_t* level_meta); + +/** + * Returns the sst_file_metadata_t of the ith file from the specified level + * metadata. + * + * If the specified i is greater than or equal to the number of files + * in the specified level, then NULL will be returned. + * + * Note that the caller is responsible to release the returned memory + * using rocksdb_sst_file_metadata_destroy before releasing its + * parent rocksdb_level_metadata_t. + */ +extern ROCKSDB_LIBRARY_API rocksdb_sst_file_metadata_t* +rocksdb_level_metadata_get_sst_file_metadata( + rocksdb_level_metadata_t* level_meta, size_t i); + +/** + * Releases the specified rocksdb_sst_file_metadata_t. + * + * Note that the specified rocksdb_sst_file_metadata_t must be released + * before the release of its parent rocksdb_level_metadata_t. + */ +extern ROCKSDB_LIBRARY_API void rocksdb_sst_file_metadata_destroy( + rocksdb_sst_file_metadata_t* file_meta); + +extern ROCKSDB_LIBRARY_API char* +rocksdb_sst_file_metadata_get_relative_filename( + rocksdb_sst_file_metadata_t* file_meta); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_sst_file_metadata_get_size(rocksdb_sst_file_metadata_t* file_meta); + +/** + * Returns the smallest key of the specified sst file. + * The caller is responsible for releasing the returned memory. + * + * @param file_meta the metadata of an SST file to obtain its smallest key. + * @param len the out value which will contain the length of the returned key + * after the function call. + */ +extern ROCKSDB_LIBRARY_API char* rocksdb_sst_file_metadata_get_smallestkey( + rocksdb_sst_file_metadata_t* file_meta, size_t* len); + +/** + * Returns the smallest key of the specified sst file. + * The caller is responsible for releasing the returned memory. + * + * @param file_meta the metadata of an SST file to obtain its smallest key. + * @param len the out value which will contain the length of the returned key + * after the function call. + */ +extern ROCKSDB_LIBRARY_API char* rocksdb_sst_file_metadata_get_largestkey( + rocksdb_sst_file_metadata_t* file_meta, size_t* len); + +/* Transactions */ + +extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t* +rocksdb_transactiondb_create_column_family( + rocksdb_transactiondb_t* txn_db, + const rocksdb_options_t* column_family_options, + const char* column_family_name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_transactiondb_t* rocksdb_transactiondb_open( + const rocksdb_options_t* options, + const rocksdb_transactiondb_options_t* txn_db_options, const char* name, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_transactiondb_t* +rocksdb_transactiondb_open_column_families( + const rocksdb_options_t* options, + const rocksdb_transactiondb_options_t* txn_db_options, const char* name, + int num_column_families, const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API const rocksdb_snapshot_t* +rocksdb_transactiondb_create_snapshot(rocksdb_transactiondb_t* txn_db); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_release_snapshot( + rocksdb_transactiondb_t* txn_db, const rocksdb_snapshot_t* snapshot); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transactiondb_property_value( + rocksdb_transactiondb_t* db, const char* propname); + +extern ROCKSDB_LIBRARY_API int rocksdb_transactiondb_property_int( + rocksdb_transactiondb_t* db, const char* propname, uint64_t* out_val); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_t* rocksdb_transaction_begin( + rocksdb_transactiondb_t* txn_db, + const rocksdb_writeoptions_t* write_options, + const rocksdb_transaction_options_t* txn_options, + rocksdb_transaction_t* old_txn); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_t** +rocksdb_transactiondb_get_prepared_transactions(rocksdb_transactiondb_t* txn_db, + size_t* cnt); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_set_name( + rocksdb_transaction_t* txn, const char* name, size_t name_len, + char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_name( + rocksdb_transaction_t* txn, size_t* name_len); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_prepare( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_commit( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rollback( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_set_savepoint( + rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rollback_to_savepoint( + rocksdb_transaction_t* txn, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_destroy( + rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API rocksdb_writebatch_wi_t* +rocksdb_transaction_get_writebatch_wi(rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rebuild_from_writebatch( + rocksdb_transaction_t* txn, rocksdb_writebatch_t* writebatch, + char** errptr); + +// This rocksdb_writebatch_wi_t should be freed with rocksdb_free +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_rebuild_from_writebatch_wi( + rocksdb_transaction_t* txn, rocksdb_writebatch_wi_t* wi, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_set_commit_timestamp( + rocksdb_transaction_t* txn, uint64_t commit_timestamp); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transaction_set_read_timestamp_for_validation( + rocksdb_transaction_t* txn, uint64_t read_timestamp); + +// This snapshot should be freed using rocksdb_free +extern ROCKSDB_LIBRARY_API const rocksdb_snapshot_t* +rocksdb_transaction_get_snapshot(rocksdb_transaction_t* txn); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const char* key, size_t klen, size_t* vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + size_t* vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned_cf(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_for_update( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const char* key, size_t klen, size_t* vlen, unsigned char exclusive, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned_for_update(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, + unsigned char exclusive, + char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_for_update_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + size_t* vlen, unsigned char exclusive, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transaction_get_pinned_for_update_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + unsigned char exclusive, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_multi_get( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_multi_get_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transactiondb_get( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + const char* key, size_t klen, size_t* vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transactiondb_get_pinned(rocksdb_transactiondb_t* txn_db, + const rocksdb_readoptions_t* options, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API char* rocksdb_transactiondb_get_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, size_t* vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* +rocksdb_transactiondb_get_pinned_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_multi_get( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_multi_get_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + const rocksdb_column_family_handle_t* const* column_families, + size_t num_keys, const char* const* keys_list, + const size_t* keys_list_sizes, char** values_list, + size_t* values_list_sizes, char** errs); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_put( + rocksdb_transaction_t* txn, const char* key, size_t klen, const char* val, + size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_put_cf( + rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_put( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_put_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, const char* val, size_t vallen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_write( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_writebatch_t* batch, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_merge( + rocksdb_transaction_t* txn, const char* key, size_t klen, const char* val, + size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_merge_cf( + rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_merge( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + const char* key, size_t klen, const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_merge_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, size_t klen, + const char* val, size_t vlen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_delete( + rocksdb_transaction_t* txn, const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_delete_cf( + rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_delete( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + const char* key, size_t klen, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_delete_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transaction_create_iterator(rocksdb_transaction_t* txn, + const rocksdb_readoptions_t* options); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transaction_create_iterator_cf( + rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transactiondb_create_iterator(rocksdb_transactiondb_t* txn_db, + const rocksdb_readoptions_t* options); + +extern ROCKSDB_LIBRARY_API rocksdb_iterator_t* +rocksdb_transactiondb_create_iterator_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_close( + rocksdb_transactiondb_t* txn_db); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush( + rocksdb_transactiondb_t* txn_db, const rocksdb_flushoptions_t* options, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush_cf( + rocksdb_transactiondb_t* txn_db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t* column_family, char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush_cfs( + rocksdb_transactiondb_t* txn_db, const rocksdb_flushoptions_t* options, + rocksdb_column_family_handle_t** column_families, int num_column_families, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_flush_wal( + rocksdb_transactiondb_t* txn_db, unsigned char sync, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_checkpoint_t* +rocksdb_transactiondb_checkpoint_object_create(rocksdb_transactiondb_t* txn_db, + char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_optimistictransactiondb_t* +rocksdb_optimistictransactiondb_open(const rocksdb_options_t* options, + const char* name, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_optimistictransactiondb_t* +rocksdb_optimistictransactiondb_open_column_families( + const rocksdb_options_t* options, const char* name, int num_column_families, + const char* const* column_family_names, + const rocksdb_options_t* const* column_family_options, + rocksdb_column_family_handle_t** column_family_handles, char** errptr); + +extern ROCKSDB_LIBRARY_API rocksdb_t* +rocksdb_optimistictransactiondb_get_base_db( + rocksdb_optimistictransactiondb_t* otxn_db); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransactiondb_close_base_db( + rocksdb_t* base_db); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_t* +rocksdb_optimistictransaction_begin( + rocksdb_optimistictransactiondb_t* otxn_db, + const rocksdb_writeoptions_t* write_options, + const rocksdb_optimistictransaction_options_t* otxn_options, + rocksdb_transaction_t* old_txn); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransactiondb_write( + rocksdb_optimistictransactiondb_t* otxn_db, + const rocksdb_writeoptions_t* options, rocksdb_writebatch_t* batch, + char** errptr); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransactiondb_close( + rocksdb_optimistictransactiondb_t* otxn_db); + +extern ROCKSDB_LIBRARY_API rocksdb_checkpoint_t* +rocksdb_optimistictransactiondb_checkpoint_object_create( + rocksdb_optimistictransactiondb_t* otxn_db, char** errptr); + +/* Transaction Options */ + +extern ROCKSDB_LIBRARY_API rocksdb_transactiondb_options_t* +rocksdb_transactiondb_options_create(void); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_options_destroy( + rocksdb_transactiondb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_options_set_max_num_locks( + rocksdb_transactiondb_options_t* opt, int64_t max_num_locks); + +extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_options_set_num_stripes( + rocksdb_transactiondb_options_t* opt, size_t num_stripes); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transactiondb_options_set_transaction_lock_timeout( + rocksdb_transactiondb_options_t* opt, int64_t txn_lock_timeout); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transactiondb_options_set_default_lock_timeout( + rocksdb_transactiondb_options_t* opt, int64_t default_lock_timeout); + +extern ROCKSDB_LIBRARY_API rocksdb_transaction_options_t* +rocksdb_transaction_options_create(void); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_destroy( + rocksdb_transaction_options_t* opt); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_set_snapshot( + rocksdb_transaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_deadlock_detect( + rocksdb_transaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_lock_timeout( + rocksdb_transaction_options_t* opt, int64_t lock_timeout); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_expiration( + rocksdb_transaction_options_t* opt, int64_t expiration); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transaction_options_set_deadlock_detect_depth( + rocksdb_transaction_options_t* opt, int64_t depth); + +extern ROCKSDB_LIBRARY_API void +rocksdb_transaction_options_set_max_write_batch_size( + rocksdb_transaction_options_t* opt, size_t size); + +extern ROCKSDB_LIBRARY_API void rocksdb_transaction_options_set_skip_prepare( + rocksdb_transaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API rocksdb_optimistictransaction_options_t* +rocksdb_optimistictransaction_options_create(void); + +extern ROCKSDB_LIBRARY_API void rocksdb_optimistictransaction_options_destroy( + rocksdb_optimistictransaction_options_t* opt); + +extern ROCKSDB_LIBRARY_API void +rocksdb_optimistictransaction_options_set_set_snapshot( + rocksdb_optimistictransaction_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API char* rocksdb_optimistictransactiondb_property_value( + rocksdb_optimistictransactiondb_t* db, const char* propname); + +extern ROCKSDB_LIBRARY_API int rocksdb_optimistictransactiondb_property_int( + rocksdb_optimistictransactiondb_t* db, const char* propname, + uint64_t* out_val); + +// referring to convention (3), this should be used by client +// to free memory that was malloc()ed +extern ROCKSDB_LIBRARY_API void rocksdb_free(void* ptr); + +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* rocksdb_get_pinned( + rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key, + size_t keylen, char** errptr); +extern ROCKSDB_LIBRARY_API rocksdb_pinnableslice_t* rocksdb_get_pinned_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, const char* key, + size_t keylen, char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_pinnableslice_destroy( + rocksdb_pinnableslice_t* v); +extern ROCKSDB_LIBRARY_API const char* rocksdb_pinnableslice_value( + const rocksdb_pinnableslice_t* t, size_t* vlen); + +extern ROCKSDB_LIBRARY_API rocksdb_memory_consumers_t* +rocksdb_memory_consumers_create(void); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_add_db( + rocksdb_memory_consumers_t* consumers, rocksdb_t* db); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_add_cache( + rocksdb_memory_consumers_t* consumers, rocksdb_cache_t* cache); +extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_destroy( + rocksdb_memory_consumers_t* consumers); +extern ROCKSDB_LIBRARY_API rocksdb_memory_usage_t* +rocksdb_approximate_memory_usage_create(rocksdb_memory_consumers_t* consumers, + char** errptr); +extern ROCKSDB_LIBRARY_API void rocksdb_approximate_memory_usage_destroy( + rocksdb_memory_usage_t* usage); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_mem_table_total( + rocksdb_memory_usage_t* memory_usage); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_mem_table_unflushed( + rocksdb_memory_usage_t* memory_usage); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_mem_table_readers_total( + rocksdb_memory_usage_t* memory_usage); +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_approximate_memory_usage_get_cache_total( + rocksdb_memory_usage_t* memory_usage); + +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_dump_malloc_stats( + rocksdb_options_t*, unsigned char); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_memtable_whole_key_filtering(rocksdb_options_t*, + unsigned char); + +extern ROCKSDB_LIBRARY_API void rocksdb_cancel_all_background_work( + rocksdb_t* db, unsigned char wait); + +extern ROCKSDB_LIBRARY_API void rocksdb_disable_manual_compaction( + rocksdb_t* db); + +extern ROCKSDB_LIBRARY_API void rocksdb_enable_manual_compaction(rocksdb_t* db); + +#ifdef __cplusplus +} /* end extern "C" */ +#endif diff --git a/docs/assets/TDengine-logo-trans-small.png b/docs/assets/TDengine-logo-trans-small.png new file mode 100644 index 0000000000000000000000000000000000000000..1cf890facd14839529e76d2ff24f532235aa58ef Binary files /dev/null and b/docs/assets/TDengine-logo-trans-small.png differ diff --git a/docs/assets/TDengine-logo-trans.png b/docs/assets/TDengine-logo-trans.png new file mode 100644 index 0000000000000000000000000000000000000000..85f55ad3b98f1deb472145788311951b2b89af9c Binary files /dev/null and b/docs/assets/TDengine-logo-trans.png differ diff --git a/docs/doxgen/CMakeLists.txt b/docs/doxgen/CMakeLists.txt index da76b2c3fd9d8e2abbead43080984af48d02ed4c..9100137bb8fee1ce4f2b58d3c91c161446124b7d 100644 --- a/docs/doxgen/CMakeLists.txt +++ b/docs/doxgen/CMakeLists.txt @@ -4,7 +4,7 @@ if(${BUILD_DOCS}) find_package(Doxygen) if (DOXYGEN_FOUND) # Build the doc - set(DOXYGEN_IN ${TD_SOURCE_DIR}/docs/Doxyfile.in) + set(DOXYGEN_IN ${TD_SOURCE_DIR}/docs/doxgen/Doxyfile.in) set(DOXYGEN_OUT ${CMAKE_BINARY_DIR}/Doxyfile) configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY) diff --git a/docs/en/05-get-started/01-docker.md b/docs/en/05-get-started/01-docker.md index 2049e1615f33134f43aeb668308f19a4f0059bdd..723194a325b4e2819842921432b1d29db165f046 100644 --- a/docs/en/05-get-started/01-docker.md +++ b/docs/en/05-get-started/01-docker.md @@ -32,6 +32,20 @@ docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043 Note that TDengine Server 3.0 uses TCP port 6030. Port 6041 is used by taosAdapter for the REST API service. Ports 6043 through 6049 are used by taosAdapter for other connectors. You can open these ports as needed. +If you need to persist data to a specific directory on your local machine, please run the following command: +```shell +docker run -d -v ~/data/taos/dnode/data:/var/lib/taos \ + -v ~/data/taos/dnode/log:/var/log/taos \ + -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine +``` +:::note + +- /var/lib/taos: TDengine's default data file directory. The location can be changed via [configuration file]. Also you can modify ~/data/taos/dnode/data to your any local empty data directory +- /var/log/taos: TDengine's default log file directory. The location can be changed via [configure file]. you can modify ~/data/taos/dnode/log to your any local empty log directory + +::: + + Run the following command to ensure that your container is running: ```shell @@ -113,4 +127,4 @@ In the query above you are selecting the first timestamp (ts) in the interval, a ## Additional Information -For more information about deploying TDengine in a Docker environment, see [Using TDengine in Docker](../../reference/docker). +For more information about deploying TDengine in a Docker environment, see [Deploying TDengine with Docker](../../deployment/docker). diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index b47855103c6eb4d00405bae60a688e4a0a6fc50c..3e3c04682faede39a46545e88f1ab682845a0730 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -18,7 +18,20 @@ The full package of TDengine includes the TDengine Server (`taosd`), TDengine Cl The standard server installation package includes `taos`, `taosd`, `taosAdapter`, `taosBenchmark`, and sample code. You can also download the Lite package that includes only `taosd` and the C/C++ connector. -The TDengine Community Edition is released as Deb and RPM packages. The Deb package can be installed on Debian, Ubuntu, and derivative systems. The RPM package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the Deb or RPM package, download and install taosTools separately. TDengine can also be installed on x64 Windows and x64/m1 macOS. +TDengine OSS is released as Deb and RPM packages. The Deb package can be installed on Debian, Ubuntu, and derivative systems. The RPM package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the Deb or RPM package, download and install taosTools separately. TDengine can also be installed on x64 Windows and x64/m1 macOS. + +## Operating environment requirements +In the Linux system, the minimum requirements for the operating environment are as follows: + +linux core version - 3.10.0-1160.83.1.el7.x86_64; + +glibc version - 2.17; + +If compiling and installing through clone source code, it is also necessary to meet the following requirements: + +cmake version - 3.26.4 or above; + +gcc version - 9.3.1 or above; ## Installation @@ -188,7 +201,7 @@ You can use the TDengine CLI to monitor your TDengine deployment and execute ad -After the installation is complete, please run `sc start taosd` or run `C:\TDengine\taosd.exe` with administrator privilege to start TDengine Server. +After the installation is complete, please run `sc start taosd` or run `C:\TDengine\taosd.exe` with administrator privilege to start TDengine Server. Please run `sc start taosadapter` or run `C:\TDengine\taosadapter.exe` with administrator privilege to start taosAdapter to provide http/REST service. ## Command Line Interface (CLI) diff --git a/docs/en/05-get-started/index.md b/docs/en/05-get-started/index.md index 66573a89cd6e181192132cc0b304f415fa25b89c..cc3b4826dd26c9a2395d22ecf85ccf23da86d6d6 100644 --- a/docs/en/05-get-started/index.md +++ b/docs/en/05-get-started/index.md @@ -21,17 +21,6 @@ import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; ``` -## Study TDengine Knowledge Map - -The TDengine Knowledge Map covers the various knowledge points of TDengine, revealing the invocation relationships and data flow between various conceptual entities. Learning and understanding the TDengine Knowledge Map will help you quickly master the TDengine knowledge system. - -
-
- -
Diagram 1. TDengine Knowledge Map
-
-
- ## Join TDengine Community diff --git a/docs/en/07-develop/01-connect/index.md b/docs/en/07-develop/01-connect/index.md index 76c26bae23a0f546ccd6fbd14f66a5942909f4c2..49c0b11a36c61ae43cc613663387aee36cec2fab 100644 --- a/docs/en/07-develop/01-connect/index.md +++ b/docs/en/07-develop/01-connect/index.md @@ -83,7 +83,7 @@ If `maven` is used to manage the projects, what needs to be done is only adding com.taosdata.jdbc taos-jdbcdriver - 3.0.0 + 3.2.4 ``` diff --git a/docs/en/07-develop/03-insert-data/01-sql-writing.mdx b/docs/en/07-develop/03-insert-data/01-sql-writing.mdx index 3731882fb23677588e72ba5e9d39049af2dfd97d..4d1b67e451ecf50697156c2a838f83b31262b0b9 100644 --- a/docs/en/07-develop/03-insert-data/01-sql-writing.mdx +++ b/docs/en/07-develop/03-insert-data/01-sql-writing.mdx @@ -33,7 +33,7 @@ The below SQL statement is used to insert one row into table "d1001". INSERT INTO d1001 VALUES (ts1, 10.3, 219, 0.31); ``` -`ts1` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detial, refer to [TDengine SQL insert timestamp section](/taos-sql/insert). +`ts1` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detail, refer to [TDengine SQL insert timestamp section](/taos-sql/insert). ### Insert Multiple Rows @@ -43,7 +43,7 @@ Multiple rows can be inserted in a single SQL statement. The example below inser INSERT INTO d1001 VALUES (ts2, 10.2, 220, 0.23) (ts2, 10.3, 218, 0.25); ``` -`ts1` and `ts2` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detial, refer to [TDengine SQL insert timestamp section](/taos-sql/insert). +`ts1` and `ts2` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detail, refer to [TDengine SQL insert timestamp section](/taos-sql/insert). ### Insert into Multiple Tables @@ -53,7 +53,7 @@ Data can be inserted into multiple tables in the same SQL statement. The example INSERT INTO d1001 VALUES (ts1, 10.3, 219, 0.31) (ts2, 12.6, 218, 0.33) d1002 VALUES (ts3, 12.3, 221, 0.31); ``` -`ts1`, `ts2` and `ts3` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detial, refer to [TDengine SQL insert timestamp section](/taos-sql/insert). +`ts1`, `ts2` and `ts3` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detail, refer to [TDengine SQL insert timestamp section](/taos-sql/insert). For more details about `INSERT` please refer to [INSERT](/taos-sql/insert). diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index a4eb41bd7e2a461dca6d3a9fa9bac39f589ec66c..ccf39ef5818477d7a236e73358691dc859e80447 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -81,10 +81,6 @@ Set subscription() throws SQLException; ConsumerRecords poll(Duration timeout) throws SQLException; -void commitAsync(); - -void commitAsync(OffsetCommitCallback callback); - void commitSync() throws SQLException; void close() throws SQLException; @@ -105,6 +101,12 @@ class Consumer: def poll(self, timeout: float = 1.0): pass + def assignment(self): + pass + + def poll(self, timeout: float = 1.0): + pass + def close(self): pass @@ -238,6 +240,8 @@ The following SQL statement creates a topic in TDengine: CREATE TOPIC topic_name AS SELECT ts, c1, c2, c3 FROM tmqdb.stb WHERE c1 > 1; ``` +- There is an upper limit to the number of topics created, controlled by the parameter tmqMaxTopicNum, with a default of 20 + Multiple subscription types are supported. #### Subscribe to a Column @@ -259,14 +263,15 @@ You can subscribe to a topic through a SELECT statement. Statements that specify Syntax: ```sql -CREATE TOPIC topic_name AS STABLE stb_name +CREATE TOPIC topic_name [with meta] AS STABLE stb_name [where_condition] ``` Creating a topic in this manner differs from a `SELECT * from stbName` statement as follows: - The table schema can be modified. - Unstructured data is returned. The format of the data returned changes based on the supertable schema. -- A different table schema may exist for every data block to be processed. +- The 'with meta' parameter is optional. When selected, statements such as creating super tables and sub tables will be returned, mainly used for Taosx to perform super table migration +- The 'where_condition' parameter is optional and will be used to filter and subscribe to sub tables that meet the criteria. Where conditions cannot have ordinary columns, only tags or tbnames. Functions can be used in where conditions to filter tags, but cannot be aggregate functions because sub table tag values cannot be aggregated. It can also be a constant expression, such as 2>1 (subscribing to all child tables), Or false (subscribe to 0 sub tables) - The data returned does not include tags. ### Subscribe to a Database @@ -274,10 +279,12 @@ Creating a topic in this manner differs from a `SELECT * from stbName` statement Syntax: ```sql -CREATE TOPIC topic_name [WITH META] AS DATABASE db_name; +CREATE TOPIC topic_name [with meta] AS DATABASE db_name; ``` -This SQL statement creates a subscription to all tables in the database. You can add the `WITH META` parameter to include schema changes in the subscription, including creating and deleting supertables; adding, deleting, and modifying columns; and creating, deleting, and modifying the tags of subtables. Consumers can determine the message type from the API. Note that this differs from Kafka. +This SQL statement creates a subscription to all tables in the database. + +- The 'with meta' parameter is optional. When selected, it will return statements for creating all super tables and sub tables in the database, mainly used for Taosx database migration ## Create a Consumer @@ -285,13 +292,13 @@ You configure the following parameters when creating a consumer: | Parameter | Type | Description | Remarks | | :----------------------------: | :-----: | -------------------------------------------------------- | ------------------------------------------- | -| `td.connect.ip` | string | Used in establishing a connection; same as `taos_connect` | Only valid for establishing native connection | -| `td.connect.user` | string | Used in establishing a connection; same as `taos_connect` | Only valid for establishing native connection | -| `td.connect.pass` | string | Used in establishing a connection; same as `taos_connect` | Only valid for establishing native connection | -| `td.connect.port` | string | Used in establishing a connection; same as `taos_connect` | Only valid for establishing native connection | -| `group.id` | string | Consumer group ID; consumers with the same ID are in the same group | **Required**. Maximum length: 192. | +| `td.connect.ip` | string | IP address of the server side | | +| `td.connect.user` | string | User Name | | +| `td.connect.pass` | string | Password | | +| `td.connect.port` | string | Port of the server side | | +| `group.id` | string | Consumer group ID; consumers with the same ID are in the same group | **Required**. Maximum length: 192. Each topic can create up to 100 consumer groups. | | `client.id` | string | Client ID | Maximum length: 192. | -| `auto.offset.reset` | enum | Initial offset for the consumer group | Specify `earliest`, `latest`, or `none`(default) | +| `auto.offset.reset` | enum | Initial offset for the consumer group | `earliest`: subscribe from the earliest data, this is the default behavior; `latest`: subscribe from the latest data; or `none`: can't subscribe without committed offset| | `enable.auto.commit` | boolean | Commit automatically; true: user application doesn't need to explicitly commit; false: user application need to handle commit by itself | Default value is true | | `auto.commit.interval.ms` | integer | Interval for automatic commits, in milliseconds | | `msg.with.table.name` | boolean | Specify whether to deserialize table names from messages | default value: false @@ -325,6 +332,7 @@ Java programs use the following parameters: | Parameter | Type | Description | Remarks | | ----------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------- | +| `td.connect.type` | string | connection type: "jni" means native connection, "ws" means websocket connection, the default is "jni" | | `bootstrap.servers` | string |Connection address, such as `localhost:6030` | | `value.deserializer` | string | Value deserializer; to use this method, implement the `com.taosdata.jdbc.tmq.Deserializer` interface or inherit the `com.taosdata.jdbc.tmq.ReferenceDeserializer` type | | `value.deserializer.encoding` | string | Specify the encoding for string deserialization | | @@ -399,22 +407,6 @@ from taos.tmq import Consumer consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) ``` -Python programs use the following parameters: - -| Parameter | Type | Description | Remarks | -|:---------:|:----:|:-----------:|:-------:| -| `td.connect.ip` | string | Used in establishing a connection|| -| `td.connect.user` | string | Used in establishing a connection|| -| `td.connect.pass` | string | Used in establishing a connection|| -| `td.connect.port` | string | Used in establishing a connection|| -| `group.id` | string | Consumer group ID; consumers with the same ID are in the same group | **Required**. Maximum length: 192 | -| `client.id` | string | Client ID | Maximum length: 192 | -| `msg.with.table.name` | string | Specify whether to deserialize table names from messages | pecify `true` or `false` | -| `enable.auto.commit` | string | Commit automatically | pecify `true` or `false` | -| `auto.commit.interval.ms` | string | Interval for automatic commits, in milliseconds | | -| `auto.offset.reset` | string | Initial offset for the consumer group | Specify `earliest`, `latest`, or `none`(default) | -| `enable.heartbeat.background` | string | Backend heartbeat; if enabled, the consumer does not go offline even if it has not polled for a long time | Specify `true` or `false` | - diff --git a/docs/en/07-develop/08-cache.md b/docs/en/07-develop/08-cache.md index 6a6ca3e5947b6a3233f90e5e0a01417e6be3b19d..9ef5d2c4f0c7697a1bdbb45c84ff28d3531417d8 100644 --- a/docs/en/07-develop/08-cache.md +++ b/docs/en/07-develop/08-cache.md @@ -10,10 +10,10 @@ TDengine uses various kinds of caching techniques to efficiently write and query TDengine uses an insert-driven cache management policy, known as first in, first out (FIFO). This policy differs from read-driven "least recently used (LRU)" cache management. A FIFO policy stores the latest data in cache and flushes the oldest data from cache to disk when the cache usage reaches a threshold. In IoT use cases, the most recent data or the current state is most important. The cache policy in TDengine, like much of the design and architecture of TDengine, is based on the nature of IoT data. -When you create a database, you can configure the size of the write cache on each vnode. The **vgroups** parameter determines the number of vgroups that process data in the database, and the **buffer** parameter determines the size of the write cache for each vnode. +When you create a database, you can configure the size of the write cache on each vnode. The **vgroups** parameter determines the number of vgroups that process data in the database, and the **buffer** parameter determines the size of the write cache for each vnode. The unit of buffer is MB. ```sql -create database db0 vgroups 100 buffer 16MB +create database db0 vgroups 100 buffer 16 ``` In theory, larger cache sizes are always better. However, at a certain point, it becomes impossible to improve performance by increasing cache size. In most scenarios, you can retain the default cache settings. @@ -28,10 +28,10 @@ When you create a database, you can configure whether the latest data from every ## Metadata Cache -To improve query and write performance, each vnode caches the metadata that it receives. When you create a database, you can configure the size of the metadata cache through the *pages* and *pagesize* parameters. +To improve query and write performance, each vnode caches the metadata that it receives. When you create a database, you can configure the size of the metadata cache through the *pages* and *pagesize* parameters. The unit of pagesize is kb. ```sql -create database db0 pages 128 pagesize 16kb +create database db0 pages 128 pagesize 16 ``` The preceding SQL statement creates 128 pages on each vnode in the `db0` database. Each page has a 16 KB metadata cache. diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index f107512e9c14459453d6424e6f05770bf510bdde..7526aba43bb1b47dffdc93648351df3186ddd384 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -10,14 +10,14 @@ User-defined functions can be scalar functions or aggregate functions. Scalar fu TDengine supports user-defined functions written in C or Python. This document describes the usage of user-defined functions. -## Implement a UDF in C +## Implement a UDF in C When you create a user-defined function, you must implement standard interface functions: - For scalar functions, implement the `scalarfn` interface function. - For aggregate functions, implement the `aggfn_start`, `aggfn`, and `aggfn_finish` interface functions. - To initialize your function, implement the `udf_init` function. To terminate your function, implement the `udf_destroy` function. -There are strict naming conventions for these interface functions. The names of the start, finish, init, and destroy interfaces must be _start, _finish, _init, and _destroy, respectively. Replace `scalarfn`, `aggfn`, and `udf` with the name of your user-defined function. +There are strict naming conventions for these interface functions. The names of the start, finish, init, and destroy interfaces must be `_start`, `_finish`, `_init`, and `_destroy`, respectively. Replace `scalarfn`, `aggfn`, and `udf` with the name of your user-defined function. ### Implementing a Scalar Function in C The implementation of a scalar function is described as follows: @@ -111,13 +111,13 @@ Interface functions return a value that indicates whether the operation was succ For information about the parameters for interface functions, see Data Model #### Scalar Interface - `int32_t scalarfn(SUdfDataBlock* inputDataBlock, SUdfColumn *resultColumn)` - + `int32_t scalarfn(SUdfDataBlock* inputDataBlock, SUdfColumn *resultColumn)` + Replace `scalarfn` with the name of your function. This function performs scalar calculations on data blocks. You can configure a value through the parameters in the `resultColumn` structure. The parameters in the function are defined as follows: - inputDataBlock: The data block to input. - - resultColumn: The column to output. The column to output. + - resultColumn: The column to output. The column to output. #### Aggregate Interface @@ -197,7 +197,7 @@ The data structure is described as follows: - The SUdfDataBlock block includes the number of rows (numOfRows) and the number of columns (numCols). udfCols[i] (0 <= i <= numCols-1) indicates that each column is of type SUdfColumn. - SUdfColumn includes the definition of the data type of the column (colMeta) and the data in the column (colData). - The member definitions of SUdfColumnMeta are the same as the data type definitions in `taos.h`. -- The data in SUdfColumnData can become longer. varLenCol indicates variable-length data, and fixLenCol indicates fixed-length data. +- The data in SUdfColumnData can become longer. varLenCol indicates variable-length data, and fixLenCol indicates fixed-length data. - SUdfInterBuf defines the intermediate structure `buffer` and the number of results in the buffer `numOfResult`. Additional functions are defined in `taosudf.h` to make it easier to work with these structures. @@ -270,29 +270,95 @@ select max_vol(vol1,vol2,vol3,deviceid) from battery; ## Implement a UDF in Python +### Prepare Environment + +1. Prepare Python Environment + +Please follow standard procedure of python environment preparation. + +2. Install Python package `taospyudf` + +```shell +pip3 install taospyudf +``` + +During this process, some C++ code needs to be compiled. So it's required to have `cmake` and `gcc` on your system. The compiled `libtaospyudf.so` will be automatically copied to `/usr/local/lib` path. If you are not root user, please use `sudo`. After installation is done, please check using the command below. + +```shell +root@slave11 ~/udf $ ls -l /usr/local/lib/libtaos* +-rw-r--r-- 1 root root 671344 May 24 22:54 /usr/local/lib/libtaospyudf.so +``` + +Then execute the command below. + +```shell +ldconfig +``` + +3. If you want to utilize some 3rd party python packages in your Python UDF, please set configuration parameter `UdfdLdLibPath` to the value of `PYTHONPATH` before starting `taosd`. + +4. Launch `taosd` service + +Please refer to [Get Started](../../get-started) + +### Interface definition + +#### Introduction to Interface + Implement the specified interface functions when implementing a UDF in Python. - implement `process` function for the scalar UDF. - implement `start`, `reduce`, `finish` for the aggregate UDF. - implement `init` for initialization and `destroy` for termination. -### Implement a Scalar UDF in Python +#### Scalar UDF Interface The implementation of a scalar UDF is described as follows: +```Python +def process(input: datablock) -> tuple[output_type]: +``` + +Description: this function processes datablock, which is the input; you can use datablock.data(row, col) to access the python object at location(row,col); the output is a tuple object consisted of objects of type outputtype + +#### Aggregate UDF Interface + +The implementation of an aggregate function is described as follows: + +```Python +def start() -> bytes: +def reduce(inputs: datablock, buf: bytes) -> bytes +def finish(buf: bytes) -> output_type: +``` + +Description: first the start() is invoked to generate the initial result `buffer`; then the input data is divided into multiple row blocks, and reduce() is invoked for each block `inputs` and current intermediate result `buf`; finally finish() is invoked to generate the final result from intermediate `buf`, the final result can only contains 0 or 1 data. + +#### Initialization and Cleanup Interface + +```python +def init() +def destroy() +``` + +Description: init() does the work of initialization before processing any data; destroy() does the work of cleanup after the data is processed. + +### Python UDF Template + +#### Scalar Template + ```Python def init(): # initialization def destroy(): # destroy def process(input: datablock) -> tuple[output_type]: - # process input datablock, + # process input datablock, # datablock.data(row, col) is to access the python object in location(row,col) - # return tuple object consisted of object of type outputtype + # return tuple object consisted of object of type outputtype ``` -### Implement an Aggregate UDF in Python +Note：process() must be implemented, init() and destroy() must be defined too but they can do nothing. -The implementation of an aggregate function is described as follows: +#### Aggregate Template ```Python def init(): @@ -303,41 +369,15 @@ def start() -> bytes: #return serialize(init_state) def reduce(inputs: datablock, buf: bytes) -> bytes # deserialize buf to state - # reduce the inputs and state into new_state. - # use inputs.data(i,j) to access python ojbect of location(i,j) + # reduce the inputs and state into new_state. + # use inputs.data(i,j) to access python object of location(i,j) # serialize new_state into new_state_bytes - return new_state_bytes + return new_state_bytes def finish(buf: bytes) -> output_type: - #return obj of type outputtype + #return obj of type outputtype ``` -### Python UDF Interface Definition - -#### Scalar interface -```Python -def process(input: datablock) -> tuple[output_type]: -``` -- `input` is a data block two-dimension matrix-like object, of which method `data(row, col)` returns the Python object located at location (`row`, `col`) -- return a Python tuple object, of which each item is a Python object of type `output_type` - -#### Aggregate Interface -```Python -def start() -> bytes: -def reduce(input: datablock, buf: bytes) -> bytes -def finish(buf: bytes) -> output_type: -``` - -- first `start()` is called to return the initial result in type `bytes` -- then the input data are divided into multiple data blocks and for each block `input`, `reduce` is called with the data block `input` and the current result `buf` bytes and generates a new intermediate result buffer. -- finally, the `finish` function is called on the intermediate result `buf` and outputs 0 or 1 data of type `output_type` - - -#### Initialization and Cleanup Interface -```Python -def init() -def destroy() -``` -Implement `init` for initialization and `destroy` for termination. +Note: aggregate UDF requires init(), destroy(), start(), reduce() and finish() to be implemented. start() generates the initial result in buffer, then the input data is divided into multiple row data blocks, reduce() is invoked for each data block `inputs` and intermediate `buf`, finally finish() is invoked to generate final result from the intermediate result `buf`. ### Data Mapping between TDengine SQL and Python UDF @@ -353,15 +393,463 @@ The following table describes the mapping between TDengine SQL data type and Pyt |TIMESTAMP | int | |JSON and other types | Not Supported | -### Installing Python UDF -1. Install Python package `taospyudf` that executes Python UDF -```bash -sudo pip install taospyudf -ldconfig +### Development Guide + +In this section we will demonstrate 5 examples of developing UDF in Python language. In this guide, you will learn the development skills from easy case to hard case, the examples include: +1. A scalar function which accepts only one integer as input and outputs ln(n^2 + 1)。 +2. A scalar function which accepts n integers, like（x1, x2, ..., xn）and output the sum of the product of each input and its sequence number, i.e. x1 + 2 * x2 + ... + n * xn。 +3. A scalar function which accepts a timestamp and output the next closest Sunday of the timestamp. In this case, we will demonstrate how to use 3rd party library `moment`. +4. An aggregate function which calculates the difference between the maximum and the minimum of a specific column, i.e. same functionality of built-in spread(). + +In the guide, some debugging skills of using Python UDF will be explained too. + +We assume you are using Linux system and already have TDengine 3.0.4.0+ and Python 3.7+. + +Note:**You can't use print() function to output log inside a UDF, you have to write the log to a specific file or use logging module of Python.** + +#### Sample 1: Simplest UDF + +This scalar UDF accepts an integer as input and output ln(n^2 + 1). + +Firstly, please compose a Python source code file in your system and save it, e.g. `/root/udf/myfun.py`, the code is like below. + +```python +from math import log + +def init(): + pass + +def destroy(): + pass + +def process(block): + rows, _ = block.shape() + return [log(block.data(i, 0) ** 2 + 1) for i in range(rows)] +``` + +This program consists of 3 functions, init() and destroy() do nothing, but they have to be defined even though there is nothing to do in them because they are critical parts of a python UDF. The most important function is process(), which accepts a data block and the data block object has two methods: +1. shape() returns the number of rows and the number of columns of the data block +2. data(i, j) returns the value at (i,j) in the block + +The output of the process() function of a scalar UDF returns exactly same number of data as the number of input rows. We will ignore the number of columns because we just want to compute on the first column. + +Then, we create the UDF using the SQL command below. + +```sql +create function myfun as '/root/udf/myfun.py' outputtype double language 'Python' +``` + +Here is the output example, it may change a little depending on your version being used. + +```shell + taos> create function myfun as '/root/udf/myfun.py' outputtype double language 'Python'; +Create OK, 0 row(s) affected (0.005202s) +``` + +Then, we used the `show` command to prove the creation of the UDF is successful. + +```text +taos> show functions; + name | +================================= + myfun | +Query OK, 1 row(s) in set (0.005767s) +``` + +Next, we can try to test the function. Before executing the UDF, we need to prepare some data using the command below in TDengine CLI. + +```sql +create database test; +create table t(ts timestamp, v1 int, v2 int, v3 int); +insert into t values('2023-05-01 12:13:14', 1, 2, 3); +insert into t values('2023-05-03 08:09:10', 2, 3, 4); +insert into t values('2023-05-10 07:06:05', 3, 4, 5); +``` + +Execute the UDF to test it: + +```sql +taos> select myfun(v1, v2) from t; + +DB error: udf function execution failure (0.011088s) +``` + +Unfortunately, the UDF execution failed. We need to check the log `udfd` daemon to find out why. + +```shell +tail -10 /var/log/taos/udfd.log +``` + +Below is the output. + +```text +05/24 22:46:28.733545 01665799 UDF ERROR can not load library libtaospyudf.so. error: operation not permitted +05/24 22:46:28.733561 01665799 UDF ERROR can not load python plugin. lib path libtaospyudf.so +``` + +From the error message we can find out that `libtaospyudf.so` was not loaded successfully. Please refer to the [Prepare Environment] section. + +After correcting environment issues, execute the UDF: + +```sql +taos> select myfun(v1) from t; + myfun(v1) | +============================ + 0.693147181 | + 1.609437912 | + 2.302585093 | +``` + +Now, we have finished the first PDF in Python, and learned some basic debugging skills. + +#### Sample 2: Abnormal Processing + +The `myfun` UDF example in sample 1 has passed, but it has two drawbacks. + +1. It the program accepts only one column of data as input, but it doesn't throw exception if you passes multiple columns. + +```sql +taos> select myfun(v1, v2) from t; + myfun(v1, v2) | +============================ + 0.693147181 | + 1.609437912 | + 2.302585093 | +``` + +2. `null` value is not processed. We expect the program to throw exception and terminate if `null` is passed as input. + +So, we try to optimize the process() function as below. + +```python +def process(block): + rows, cols = block.shape() + if cols > 1: + raise Exception(f"require 1 parameter but given {cols}") + return [ None if block.data(i, 0) is None else log(block.data(i, 0) ** 2 + 1) for i in range(rows)] +``` + +The update the UDF with command below. + +```sql +create or replace function myfun as '/root/udf/myfun.py' outputtype double language 'Python'; +``` + +At this time, if we pass two arguments to `myfun`, the execution would fail. + +```sql +taos> select myfun(v1, v2) from t; + +DB error: udf function execution failure (0.014643s) +``` + +However, the exception is not shown to end user, but displayed in the log file `/var/log/taos/taospyudf.log` + +```text +2023-05-24 23:21:06.790 ERROR [1666188] [doPyUdfScalarProc@507] call pyUdfScalar proc function. context 0x7faade26d180. error: Exception: require 1 parameter but given 2 + +At: + /var/lib/taos//.udf/myfun_3_1884e1281d9.py(12): process + +``` + +Now, we have learned how to update a UDF and check the log of a UDF. + +Note: Prior to TDengine 3.0.5.0 (excluding), updating a UDF requires to restart `taosd` service. After 3.0.5.0, restarting is not required. + +#### Sample 3: UDF with n arguments + +A UDF which accepts n integers, likee (x1, x2, ..., xn) and output the sum of the product of each value and its sequence number: 1 * x1 + 2 * x2 + ... + n * xn. If there is `null` in the input, then the result is `null`. The difference from sample 1 is that it can accept any number of columns as input and process each column. Assume the program is written in /root/udf/nsum.py: + +```python +def init(): + pass + + +def destroy(): + pass + + +def process(block): + rows, cols = block.shape() + result = [] + for i in range(rows): + total = 0 + for j in range(cols): + v = block.data(i, j) + if v is None: + total = None + break + total += (j + 1) * block.data(i, j) + result.append(total) + return result +``` + +Crate and test the UDF: + +```sql +create function nsum as '/root/udf/nsum.py' outputtype double language 'Python'; +``` + +```sql +taos> insert into t values('2023-05-25 09:09:15', 6, null, 8); +Insert OK, 1 row(s) affected (0.003675s) + +taos> select ts, v1, v2, v3, nsum(v1, v2, v3) from t; + ts | v1 | v2 | v3 | nsum(v1, v2, v3) | +================================================================================================ + 2023-05-01 12:13:14.000 | 1 | 2 | 3 | 14.000000000 | + 2023-05-03 08:09:10.000 | 2 | 3 | 4 | 20.000000000 | + 2023-05-10 07:06:05.000 | 3 | 4 | 5 | 26.000000000 | + 2023-05-25 09:09:15.000 | 6 | NULL | 8 | NULL | +Query OK, 4 row(s) in set (0.010653s) +``` + +#### Sample 4: Utilize 3rd party package + +A UDF which accepts a timestamp and output the next closed Sunday. This sample requires to use third party package `moment`, you need to install it firstly. + +```shell +pip3 install moment +``` + +Then compose the Python code in /root/udf/nextsunday.py + +```python +import moment + + +def init(): + pass + + +def destroy(): + pass + + +def process(block): + rows, cols = block.shape() + if cols > 1: + raise Exception("require only 1 parameter") + if not type(block.data(0, 0)) is int: + raise Exception("type error") + return [moment.unix(block.data(i, 0)).replace(weekday=7).format('YYYY-MM-DD') + for i in range(rows)] +``` + +UDF framework will map the TDengine timestamp to Python int type, so this function only accepts an integer representing millisecond. process() firstly validates the parameters, then use `moment` to replace the time, format the result and output. + +Create and test the UDF. + +```sql +create function nextsunday as '/root/udf/nextsunday.py' outputtype binary(10) language 'Python'; +``` + +If your `taosd` is started using `systemd`, you may encounter the error below. Next we will show how to debug. + +```sql +taos> select ts, nextsunday(ts) from t; + +DB error: udf function execution failure (1.123615s) +``` + +```shell + tail -20 taospyudf.log +2023-05-25 11:42:34.541 ERROR [1679419] [PyUdf::PyUdf@217] py udf load module failure. error ModuleNotFoundError: No module named 'moment' +``` + +This is because `moment` doesn't exist in the default library search path of python UDF, please check the log file `taosdpyudf.log`. + +```shell +grep 'sys path' taospyudf.log | tail -1 +``` + +```text +2023-05-25 10:58:48.554 INFO [1679419] [doPyOpen@592] python sys path: ['', '/lib/python38.zip', '/lib/python3.8', '/lib/python3.8/lib-dynload', '/lib/python3/dist-packages', '/var/lib/taos//.udf'] +``` + +You may find that the default library search path is `/lib/python3/dist-packages` (just for example, it may be different in your system), but `moment` is installed to `/usr/local/lib/python3.8/dist-packages` (for example, it may be different in your system). Then we change the library search path of python UDF. + +Check `sys.path`, which must include the packages you install with pip3 command previously, as shown below: + +```python +>>> import sys +>>> ":".join(sys.path) +'/usr/lib/python3.8:/usr/lib/python3.8/lib-dynload:/usr/local/lib/python3.8/dist-packages:/usr/lib/python3/dist-packages' +``` + +Copy the output and edit /var/taos/taos.cfg to add below configuration parameter. + +```shell +UdfdLdLibPath /usr/lib/python3.8:/usr/lib/python3.8/lib-dynload:/usr/local/lib/python3.8/dist-packages:/usr/lib/python3/dist-packages +``` + +Save it, then restart `taosd`, using `systemctl restart taosd`, and test again, it will succeed this time. + +Note: If your cluster consists of multiple `taosd` instances, you have to repeat same process for each of them. + +```sql +taos> select ts, nextsunday(ts) from t; + ts | nextsunday(ts) | +=========================================== + 2023-05-01 12:13:14.000 | 2023-05-07 | + 2023-05-03 08:09:10.000 | 2023-05-07 | + 2023-05-10 07:06:05.000 | 2023-05-14 | + 2023-05-25 09:09:15.000 | 2023-05-28 | +Query OK, 4 row(s) in set (1.011474s) ``` -2. If PYTHONPATH is needed to find Python packages when the Python UDF executes, include the PYTHONPATH contents into the udfdLdLibPath variable of the taos.cfg configuration file - -### Python UDF Sample Code + +#### Sample 5: Aggregate Function + +An aggregate function which calculates the difference of the maximum and the minimum in a column. An aggregate funnction takes multiple rows as input and output only one data. The execution process of an aggregate UDF is like map-reduce, the framework divides the input into multiple parts, each mapper processes one block and the reducer aggregates the result of the mappers. The reduce() of Python UDF has the functionality of both map() and reduce(). The reduce() takes two arguments: the data to be processed; and the result of other tasks executing reduce(). For example, assume the code is in `/root/udf/myspread.py`. + +```python +import io +import math +import pickle + +LOG_FILE: io.TextIOBase = None + + +def init(): + global LOG_FILE + LOG_FILE = open("/var/log/taos/spread.log", "wt") + log("init function myspead success") + + +def log(o): + LOG_FILE.write(str(o) + '\n') + + +def destroy(): + log("close log file: spread.log") + LOG_FILE.close() + + +def start(): + return pickle.dumps((-math.inf, math.inf)) + + +def reduce(block, buf): + max_number, min_number = pickle.loads(buf) + log(f"initial max_number={max_number}, min_number={min_number}") + rows, _ = block.shape() + for i in range(rows): + v = block.data(i, 0) + if v > max_number: + log(f"max_number={v}") + max_number = v + if v < min_number: + log(f"min_number={v}") + min_number = v + return pickle.dumps((max_number, min_number)) + + +def finish(buf): + max_number, min_number = pickle.loads(buf) + return max_number - min_number +``` + +In this example, we implemented an aggregate function, and added some logging. +1. init() opens a file for logging +2. log() is the function for logging, it converts the input object to string and output with an end of line +3. destroy() closes the log file \ +4. start() returns the initial buffer for storing the intermediate result +5. reduce() processes each data block and aggregates the result +6. finish() converts the final buffer() to final result\ + +Create the UDF. + +```sql +create or replace aggregate function myspread as '/root/udf/myspread.py' outputtype double bufsize 128 language 'Python'; +``` + +This SQL command has two important different points from the command creating scalar UDF. +1. keyword `aggregate` is used +2. keyword `bufsize` is used to specify the memory size for storing the intermediate result. In this example, the result is 32 bytes, but we specified 128 bytes for `bufsize`. You can use the `python` CLI to print actual size. + +```python +>>> len(pickle.dumps((12345.6789, 23456789.9877))) +32 +``` + +Test this function, you can see the result is same as built-in spread() function. \ + +```sql +taos> select myspread(v1) from t; + myspread(v1) | +============================ + 5.000000000 | +Query OK, 1 row(s) in set (0.013486s) + +taos> select spread(v1) from t; + spread(v1) | +============================ + 5.000000000 | +Query OK, 1 row(s) in set (0.005501s) +``` + +At last, check the log file, we can see that the reduce() function is executed 3 times, max value is updated 3 times and min value is updated only one time. + +```shell +root@slave11 /var/log/taos $ cat spread.log +init function myspead success +initial max_number=-inf, min_number=inf +max_number=1 +min_number=1 +initial max_number=1, min_number=1 +max_number=2 +max_number=3 +initial max_number=3, min_number=1 +max_number=6 +close log file: spread.log +``` + +### SQL Commands + +1. Create Scalar UDF + +```sql +CREATE FUNCTION function_name AS library_path OUTPUTTYPE output_type LANGUAGE 'Python'; +``` + +2. Create Aggregate UDF + +```sql +CREATE AGGREGATE FUNCTION function_name library_path OUTPUTTYPE output_type LANGUAGE 'Python'; +``` + +3. Update Scalar UDF + +```sql +CREATE OR REPLACE FUNCTION function_name AS OUTPUTTYPE int LANGUAGE 'Python'; +``` + +4. Update Aggregate UDF + +```sql +CREATE OR REPLACE AGGREGATE FUNCTION function_name AS OUTPUTTYPE BUFSIZE buf_size int LANGUAGE 'Python'; +``` + +Note: If keyword `AGGREGATE` used, the UDF will be treated as aggregate UDF despite what it was before; Similarly, if there is no keyword `aggregate`, the UDF will be treated as scalar function despite what it was before. + +5. Show the UDF + +The version of a UDF is increased by one every time it's updated. + +```sql +select * from ins_functions \G; +``` + +6. Show and Drop existing UDF + +```sql +SHOW functions; +DROP FUNCTION function_name; +``` + +### More Python UDF Samples + #### Scalar Function [pybitand](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pybitand.py) The `pybitand` function implements bitwise addition for multiple columns. If there is only one column, the column is returned. The `pybitand` function ignores null values. @@ -377,7 +865,7 @@ The `pybitand` function implements bitwise addition for multiple columns. If the #### Aggregate Function [pyl2norm](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pyl2norm.py) -The `pyl2norm` function finds the second-order norm for all data in the input column. This squares the values, takes a cumulative sum, and finds the square root. +The `pyl2norm` function finds the second-order norm for all data in the input columns. This squares the values, takes a cumulative sum, and finds the square root.
pyl2norm.py @@ -387,5 +875,16 @@ The `pyl2norm` function finds the second-order norm for all data in the input co
+#### Aggregate Function [pycumsum](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pycumsum.py) + +The `pycumsum` function finds the cumulative sum for all data in the input columns. +
+pycumsum.py + +```c +{{#include tests/script/sh/pycumsum.py}} +``` + +
## Manage and Use UDF You need to add UDF to TDengine before using it in SQL queries. For more information about how to manage UDF and how to invoke UDF, please see [Manage and Use UDF](../12-taos-sql/26-udf.md). diff --git a/docs/en/10-deployment/01-deploy.md b/docs/en/10-deployment/01-deploy.md index da00e21a7ec7c7a34c0a3ae60b2928bd3b036cc4..ec9162c8593a6506d26b29ff7b2680d134ea1c52 100644 --- a/docs/en/10-deployment/01-deploy.md +++ b/docs/en/10-deployment/01-deploy.md @@ -62,12 +62,13 @@ serverPort 6030 For all the dnodes in a TDengine cluster, the below parameters must be configured exactly the same, any node whose configuration is different from dnodes already in the cluster can't join the cluster. -| **#** | **Parameter** | **Definition** | -| ----- | ------------------ | ------------------------------------------- | -| 1 | statusInterval | The interval by which dnode reports its status to mnode | -| 2 | timezone | Timezone | -| 3 | locale | System region and encoding | -| 4 | charset | Character set | +| **#** | **Parameter** | **Definition** | +| ----- | ---------------- | ----------------------------------------------------------------------------- | +| 1 | statusInterval | The interval by which dnode reports its status to mnode | +| 2 | timezone | Timezone | +| 3 | locale | System region and encoding | +| 4 | charset | Character set | +| 5 | ttlChangeOnWrite | Whether the ttl expiration time changes with the table modification operation | ## Start Cluster @@ -97,7 +98,7 @@ Then, on the first dnode i.e. h1.tdengine.com in our example, use TDengine CLI ` CREATE DNODE "h2.taos.com:6030"; ```` -This adds the end point of the new dnode (from Step 4) into the end point list of the cluster. In the command "fqdn:port" should be quoted using double quotes. Change `"h2.taos.com:6030"` to the end point of your new dnode. +This adds the end point of the new dnode (from Step 4) into the end point list of the cluster. In the command "fqdn:port" should be quoted using double quotes. Change `"h2.taos.com:6030"` to the end point of your new dnode. Then on the first dnode h1.tdengine.com, execute `show dnodes` in `taos` diff --git a/docs/en/14-reference/11-docker/index.md b/docs/en/10-deployment/02-docker.md similarity index 90% rename from docs/en/14-reference/11-docker/index.md rename to docs/en/10-deployment/02-docker.md index 5a48f2e4b1e58132c728dadfba2afbe39ae8bef3..63153f3033ba1e5ff6764829c312d227220b47a6 100644 --- a/docs/en/14-reference/11-docker/index.md +++ b/docs/en/10-deployment/02-docker.md @@ -1,5 +1,6 @@ --- title: Deploying TDengine with Docker +sidebar_label: Docker description: This chapter describes how to start and access TDengine in a Docker container. --- @@ -10,8 +11,17 @@ This chapter describes how to start the TDengine service in a container and acce The TDengine image starts with the HTTP service activated by default, using the following command: ```shell -docker run -d --name tdengine -p 6041:6041 tdengine/tdengine +docker run -d --name tdengine \ +-v ~/data/taos/dnode/data:/var/lib/taos \ +-v ~/data/taos/dnode/log:/var/log/taos \ +-p 6041:6041 tdengine/tdengine ``` +:::note + +* /var/lib/taos: TDengine's default data file directory. The location can be changed via [configuration file]. And also you can modify ~/data/taos/dnode/data to your any other local emtpy data directory +* /var/log/taos: TDengine's default log file directory. The location can be changed via [configure file]. And also you can modify ~/data/taos/dnode/log to your any other local empty log directory + +::: The above command starts a container named "tdengine" and maps the HTTP service port 6041 to the host port 6041. You can verify that the HTTP service provided in this container is available using the following command. @@ -283,39 +293,38 @@ services: environment: TAOS_FQDN: "td-1" TAOS_FIRST_EP: "td-1" + ports: + - 6041:6041 + - 6030:6030 volumes: - - taosdata-td1:/var/lib/taos/ - - taoslog-td1:/var/log/taos/ + # /var/lib/taos: TDengine's default data file directory. The location can be changed via [configuration file]. you can modify ~/data/taos/dnode1/data to your own data directory + - ~/data/taos/dnode1/data:/var/lib/taos + # /var/log/taos: TDengine's default log file directory. The location can be changed via [configure file]. you can modify ~/data/taos/dnode1/log to your own log directory + - ~/data/taos/dnode1/log:/var/log/taos td-2: image: tdengine/tdengine:$VERSION environment: TAOS_FQDN: "td-2" TAOS_FIRST_EP: "td-1" volumes: - - taosdata-td2:/var/lib/taos/ - - taoslog-td2:/var/log/taos/ + - ~/data/taos/dnode2/data:/var/lib/taos + - ~/data/taos/dnode2/log:/var/log/taos td-3: image: tdengine/tdengine:$VERSION environment: TAOS_FQDN: "td-3" TAOS_FIRST_EP: "td-1" volumes: - - taosdata-td3:/var/lib/taos/ - - taoslog-td3:/var/log/taos/ -volumes: - taosdata-td1: - taoslog-td1: - taosdata-td2: - taoslog-td2: - taosdata-td3: - taoslog-td3: + - ~/data/taos/dnode3/data:/var/lib/taos + - ~/data/taos/dnode3/log:/var/log/taos ``` :::note - The `VERSION` environment variable is used to set the tdengine image tag - `TAOS_FIRST_EP` must be set on the newly created instance so that it can join the TDengine cluster; if there is a high availability requirement, `TAOS_SECOND_EP` needs to be used at the same time - ::: + +::: 2. Start the cluster @@ -382,24 +391,22 @@ networks: services: td-1: image: tdengine/tdengine:$VERSION - networks: - - inter environment: TAOS_FQDN: "td-1" TAOS_FIRST_EP: "td-1" volumes: - - taosdata-td1:/var/lib/taos/ - - taoslog-td1:/var/log/taos/ + # /var/lib/taos: TDengine's default data file directory. The location can be changed via [configuration file]. you can modify ~/data/taos/dnode1/data to your own data directory + - ~/data/taos/dnode1/data:/var/lib/taos + # /var/log/taos: TDengine's default log file directory. The location can be changed via [configure file]. you can modify ~/data/taos/dnode1/log to your own log directory + - ~/data/taos/dnode1/log:/var/log/taos td-2: image: tdengine/tdengine:$VERSION - networks: - - inter environment: TAOS_FQDN: "td-2" TAOS_FIRST_EP: "td-1" volumes: - - taosdata-td2:/var/lib/taos/ - - taoslog-td2:/var/log/taos/ + - ~/data/taos/dnode2/data:/var/lib/taos + - ~/data/taos/dnode2/log:/var/log/taos adapter: image: tdengine/tdengine:$VERSION entrypoint: "taosadapter" @@ -431,11 +438,6 @@ services: >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf; nginx -g 'daemon off;'", ] -volumes: - taosdata-td1: - taoslog-td1: - taosdata-td2: - taoslog-td2: ``` ## Deploy with docker swarm diff --git a/docs/en/10-deployment/03-k8s.md b/docs/en/10-deployment/03-k8s.md index 49e61caafce6414a10c4bc1937ce13394a7da9eb..10c0341598cd051ff84950958b9c1f9589ff9319 100644 --- a/docs/en/10-deployment/03-k8s.md +++ b/docs/en/10-deployment/03-k8s.md @@ -4,23 +4,31 @@ sidebar_label: Kubernetes description: This document describes how to deploy TDengine on Kubernetes. --- -TDengine is a cloud-native time-series database that can be deployed on Kubernetes. This document gives a step-by-step description of how you can use YAML files to create a TDengine cluster and introduces common operations for TDengine in a Kubernetes environment. +## Overview + +As a time series database for Cloud Native architecture design, TDengine supports Kubernetes deployment. Firstly we introduce how to use YAML files to create a highly available TDengine cluster from scratch step by step for production usage, and highlight the common operations of TDengine in Kubernetes environment. + +To meet [high availability ](https://docs.taosdata.com/tdinternal/high-availability/)requirements, clusters need to meet the following requirements: + +- 3 or more dnodes: multiple vnodes in the same vgroup of TDengine are not allowed to be distributed in one dnode at the same time, so if you create a database with 3 replicas, the number of dnodes is greater than or equal to 3 +- 3 mnodes: mnode is responsible for the management of the entire TDengine cluster. The default number of mnode in TDengine cluster is only one. If the dnode where the mnode located is dropped, the entire cluster is unavailable. +- Database 3 replicas: The TDengine replica configuration is the database level, so 3 replicas for the database must need three dnodes in the cluster. If any one dnode is offline, does not affect the normal usage of the whole cluster. **If the number of offline** **dnodes** **is 2, then the cluster is not available,** **because** ** the cluster can not complete the election based on RAFT** **.** (Enterprise version: in the disaster recovery scenario, any node data file is damaged, can be restored by pulling up the dnode again) ## Prerequisites Before deploying TDengine on Kubernetes, perform the following: -* Current steps are compatible with Kubernetes v1.5 and later version. -* Install and configure minikube, kubectl, and helm. -* Install and deploy Kubernetes and ensure that it can be accessed and used normally. Update any container registries or other services as necessary. +- This article applies Kubernetes 1.19 and above +- This article uses the **kubectl** tool to install and deploy, please install the corresponding software in advance +- Kubernetes have been installed and deployed and can access or update the necessary container repositories or other services You can download the configuration files in this document from [GitHub](https://github.com/taosdata/TDengine-Operator/tree/3.0/src/tdengine). ## Configure the service -Create a service configuration file named `taosd-service.yaml`. Record the value of `metadata.name` (in this example, `taos`) for use in the next step. Add the ports required by TDengine: +Create a service configuration file named `taosd-service.yaml`. Record the value of `metadata.name` (in this example, `taos`) for use in the next step. And then add the ports required by TDengine and record the value of the selector label "app" (in this example, `tdengine`) for use in the next step: -```yaml +```YAML --- apiVersion: v1 kind: Service @@ -31,10 +39,10 @@ metadata: spec: ports: - name: tcp6030 - - protocol: "TCP" + protocol: "TCP" port: 6030 - name: tcp6041 - - protocol: "TCP" + protocol: "TCP" port: 6041 selector: app: "tdengine" @@ -42,10 +50,11 @@ spec: ## Configure the service as StatefulSet -Configure the TDengine service as a StatefulSet. -Create the `tdengine.yaml` file and set `replicas` to 3. In this example, the region is set to Asia/Shanghai and 10 GB of standard storage are allocated per node. You can change the configuration based on your environment and business requirements. +According to Kubernetes instructions for various deployments, we will use StatefulSet as the deployment resource type of TDengine. Create the file `tdengine.yaml `, where replicas defines the number of cluster nodes as 3. The node time zone is China (Asia/Shanghai), and each node is allocated 5G standard storage (refer to the [Storage Classes ](https://kubernetes.io/docs/concepts/storage/storage-classes/)configuration storage class). You can also modify accordingly according to the actual situation. + +Please pay special attention to the startupProbe configuration. If dnode's Pod drops for a period of time and then restart, the newly launched dnode Pod will be temporarily unavailable. The reason is the startupProbe configuration is too small, Kubernetes will know that the Pod is in an abnormal state and try to restart it, then the dnode's Pod will restart frequently and never return to the normal status. Refer to [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) -```yaml +```YAML --- apiVersion: apps/v1 kind: StatefulSet @@ -69,14 +78,14 @@ spec: spec: containers: - name: "tdengine" - image: "tdengine/tdengine:3.0.0.0" + image: "tdengine/tdengine:3.0.7.1" imagePullPolicy: "IfNotPresent" ports: - name: tcp6030 - - protocol: "TCP" + protocol: "TCP" containerPort: 6030 - name: tcp6041 - - protocol: "TCP" + protocol: "TCP" containerPort: 6041 env: # POD_NAME for FQDN config @@ -102,12 +111,18 @@ spec: # Must set if you want a cluster. - name: TAOS_FIRST_EP value: "$(STS_NAME)-0.$(SERVICE_NAME).$(STS_NAMESPACE).svc.cluster.local:$(TAOS_SERVER_PORT)" - # TAOS_FQDN should always be set in k8s env. + # TAOS_FQND should always be set in k8s env. - name: TAOS_FQDN value: "$(POD_NAME).$(SERVICE_NAME).$(STS_NAMESPACE).svc.cluster.local" volumeMounts: - name: taosdata mountPath: /var/lib/taos + startupProbe: + exec: + command: + - taos-check + failureThreshold: 360 + periodSeconds: 10 readinessProbe: exec: command: @@ -129,266 +144,401 @@ spec: storageClassName: "standard" resources: requests: - storage: "10Gi" + storage: "5Gi" ``` ## Use kubectl to deploy TDengine -Run the following commands: +First create the corresponding namespace, and then execute the following command in sequence : -```bash -kubectl apply -f taosd-service.yaml -kubectl apply -f tdengine.yaml +```Bash +kubectl apply -f taosd-service.yaml -n tdengine-test +kubectl apply -f tdengine.yaml -n tdengine-test ``` -The preceding configuration generates a TDengine cluster with three nodes in which dnodes are automatically configured. You can run the `show dnodes` command to query the nodes in the cluster: +The above configuration will generate a three-node TDengine cluster, dnode is automatically configured, you can use the **show dnodes** command to view the nodes of the current cluster: -```bash -kubectl exec -i -t tdengine-0 -- taos -s "show dnodes" -kubectl exec -i -t tdengine-1 -- taos -s "show dnodes" -kubectl exec -i -t tdengine-2 -- taos -s "show dnodes" +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show dnodes" +kubectl exec -it tdengine-2 -n tdengine-test -- taos -s "show dnodes" ``` The output is as follows: -``` +```Bash taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | -Query OK, 3 rows in database (0.003655s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 0 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-19 17:54:18.469 | | | | + 2 | tdengine-1.ta... | 0 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-19 17:54:38.698 | | | | + 3 | tdengine-2.ta... | 0 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-19 17:55:02.039 | | | | +Query OK, 3 row(s) in set (0.001853s) +``` + +View the current mnode + +```Bash +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G" +taos> show mnodes\G +*************************** 1.row *************************** + id: 1 + endpoint: tdengine-0.taosd.tdengine-test.svc.cluster.local:6030 + role: leader + status: ready +create_time: 2023-07-19 17:54:18.559 +reboot_time: 2023-07-19 17:54:19.520 +Query OK, 1 row(s) in set (0.001282s) +``` + +## Create mnode + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "create mnode on dnode 2" +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "create mnode on dnode 3" +``` + +View mnode + +```Bash +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G" + +taos> show mnodes\G +*************************** 1.row *************************** + id: 1 + endpoint: tdengine-0.taosd.tdengine-test.svc.cluster.local:6030 + role: leader + status: ready +create_time: 2023-07-19 17:54:18.559 +reboot_time: 2023-07-20 09:19:36.060 +*************************** 2.row *************************** + id: 2 + endpoint: tdengine-1.taosd.tdengine-test.svc.cluster.local:6030 + role: follower + status: ready +create_time: 2023-07-20 09:22:05.600 +reboot_time: 2023-07-20 09:22:12.838 +*************************** 3.row *************************** + id: 3 + endpoint: tdengine-2.taosd.tdengine-test.svc.cluster.local:6030 + role: follower + status: ready +create_time: 2023-07-20 09:22:20.042 +reboot_time: 2023-07-20 09:22:23.271 +Query OK, 3 row(s) in set (0.003108s) ``` ## Enable port forwarding -The kubectl port forwarding feature allows applications to access the TDengine cluster running on Kubernetes. +Kubectl port forwarding enables applications to access TDengine clusters running in Kubernetes environments. -``` -kubectl port-forward tdengine-0 6041:6041 & +```bash +kubectl port-forward -n tdengine-test tdengine-0 6041:6041 & ``` -Use curl to verify that the TDengine REST API is working on port 6041: +Use **curl** to verify that the TDengine REST API is working on port 6041: -``` -$ curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql -Handling connection for 6041 -{"code":0,"column_meta":[["name","VARCHAR",64],["create_time","TIMESTAMP",8],["vgroups","SMALLINT",2],["ntables","BIGINT",8],["replica","TINYINT",1],["strict","VARCHAR",4],["duration","VARCHAR",10],["keep","VARCHAR",32],["buffer","INT",4],["pagesize","INT",4],["pages","INT",4],["minrows","INT",4],["maxrows","INT",4],["comp","TINYINT",1],["precision","VARCHAR",2],["status","VARCHAR",10],["retention","VARCHAR",60],["single_stable","BOOL",1],["cachemodel","VARCHAR",11],["cachesize","INT",4],["wal_level","TINYINT",1],["wal_fsync_period","INT",4],["wal_retention_period","INT",4],["wal_retention_size","BIGINT",8],["wal_roll_period","INT",4],["wal_segment_size","BIGINT",8]],"data":[["information_schema",null,null,16,null,null,null,null,null,null,null,null,null,null,null,"ready",null,null,null,null,null,null,null,null,null,null],["performance_schema",null,null,10,null,null,null,null,null,null,null,null,null,null,null,"ready",null,null,null,null,null,null,null,null,null,null]],"rows":2} +```bash +curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql +{"code":0,"column_meta":[["name","VARCHAR",64]],"data":[["information_schema"],["performance_schema"],["test"],["test1"]],"rows":4} ``` -## Enable the dashboard for visualization +## Test cluster - The minikube dashboard command enables visualized cluster management. +### Data preparation -``` -$ minikube dashboard -* Verifying dashboard health ... -* Launching proxy ... -* Verifying proxy health ... -* Opening http://127.0.0.1:46617/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/ in your default browser... -http://127.0.0.1:46617/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/ -``` +#### taosBenchmark -In some public clouds, minikube cannot be remotely accessed if it is bound to 127.0.0.1. In this case, use the kubectl proxy command to map the port to 0.0.0.0. Then, you can access the dashboard by using a web browser to open the dashboard URL above on the public IP address and port of the virtual machine. +Create a 3 replicas database with taosBenchmark, write 100 million data at the same time, and view the data at the same time +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taosBenchmark -I stmt -d test -n 10000 -t 10000 -a 3 + +# query data +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "select count(*) from test.meters;" + +taos> select count(*) from test.meters; + count(*) | +======================== + 100000000 | +Query OK, 1 row(s) in set (0.103537s) ``` -$ kubectl proxy --accept-hosts='^.*$' --address='0.0.0.0' -``` + +View vnode distribution by showing dnodes + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" + +taos> show dnodes + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 8 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-19 17:54:18.469 | | | | + 2 | tdengine-1.ta... | 8 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-19 17:54:38.698 | | | | + 3 | tdengine-2.ta... | 8 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-19 17:55:02.039 | | | | +Query OK, 3 row(s) in set (0.001357s) +``` + +View xnode distribution by showing vgroup + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show test.vgroups" + +taos> show test.vgroups + vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode | v4_status | cacheload | cacheelements | tsma | +============================================================================================================================================================================================== + 2 | test | 1267 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | + 3 | test | 1215 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 4 | test | 1215 | 1 | leader | 2 | follower | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 5 | test | 1307 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 6 | test | 1245 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | + 7 | test | 1275 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 8 | test | 1231 | 1 | leader | 2 | follower | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 9 | test | 1245 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | +Query OK, 8 row(s) in set (0.001488s) +``` + +#### Manually created + +Common a three-copy test1, and create a table, write 2 pieces of data + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- \ + taos -s \ + "create database if not exists test1 replica 3; + use test1; + create table if not exists t1(ts timestamp, n int); + insert into t1 values(now, 1)(now+1s, 2);" +``` + +View xnode distribution by showing test1.vgroup + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show test1.vgroups" + +taos> show test1.vgroups + vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode | v4_status | cacheload | cacheelements | tsma | +============================================================================================================================================================================================== + 10 | test1 | 1 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | + 11 | test1 | 0 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | +Query OK, 2 row(s) in set (0.001489s) +``` + +### Test fault tolerance + +The dnode where the mnode leader is located is disconnected, dnode1 + +```Bash +kubectl get pod -l app=tdengine -n tdengine-test -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 0/1 ErrImagePull 2 (2s ago) 20m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (6m48s ago) 20m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 21m 10.244.1.223 node85 +``` + +At this time, the cluster mnode has a re-election, and the monde on dnode1 becomes the leader. + +```Bash +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G" +Welcome to the TDengine Command Line Interface, Client Version:3.0.7.1.202307190706 +Copyright (c) 2022 by TDengine, all rights reserved. + +taos> show mnodes\G +*************************** 1.row *************************** + id: 1 + endpoint: tdengine-0.taosd.tdengine-test.svc.cluster.local:6030 + role: offline + status: offline +create_time: 2023-07-19 17:54:18.559 +reboot_time: 1970-01-01 08:00:00.000 +*************************** 2.row *************************** + id: 2 + endpoint: tdengine-1.taosd.tdengine-test.svc.cluster.local:6030 + role: leader + status: ready +create_time: 2023-07-20 09:22:05.600 +reboot_time: 2023-07-20 09:32:00.227 +*************************** 3.row *************************** + id: 3 + endpoint: tdengine-2.taosd.tdengine-test.svc.cluster.local:6030 + role: follower + status: ready +create_time: 2023-07-20 09:22:20.042 +reboot_time: 2023-07-20 09:32:00.026 +Query OK, 3 row(s) in set (0.001513s) +``` + +Cluster can read and write normally + +```Bash +# insert +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "insert into test1.t1 values(now, 1)(now+1s, 2);" + +taos> insert into test1.t1 values(now, 1)(now+1s, 2); +Insert OK, 2 row(s) affected (0.002098s) + +# select +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "select *from test1.t1" + +taos> select *from test1.t1 + ts | n | +======================================== + 2023-07-19 18:04:58.104 | 1 | + 2023-07-19 18:04:59.104 | 2 | + 2023-07-19 18:06:00.303 | 1 | + 2023-07-19 18:06:01.303 | 2 | +Query OK, 4 row(s) in set (0.001994s) +``` + +Similarly, as for the non-leader mnode dropped, read and write can of course be normal, here will not do too much display . ## Scaling Out Your Cluster -TDengine clusters can scale automatically: +TDengine cluster supports automatic expansion: -```bash +```Bash kubectl scale statefulsets tdengine --replicas=4 ``` -The preceding command increases the number of replicas to 4. After running this command, query the pod status: +The parameter `--replica = 4 `in the above command line indicates that you want to expand the TDengine cluster to 4 nodes. After execution, first check the status of the Pod: -```bash -kubectl get pods -l app=tdengine +```Bash +kubectl get pod -l app=tdengine -n tdengine-test -o wide ``` The output is as follows: -``` -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 161m -tdengine-1 1/1 Running 0 161m -tdengine-2 1/1 Running 0 32m -tdengine-3 1/1 Running 0 32m +```Plain +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 1/1 Running 4 (6h26m ago) 6h53m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (6h39m ago) 6h53m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 5h16m 10.244.1.224 node85 +tdengine-3 1/1 Running 0 3m24s 10.244.2.76 node86 ``` -The status of all pods is Running. Once the pod status changes to Ready, you can check the dnode status: +At this time, the state of the POD is still Running, and the dnode state in the TDengine cluster can only be seen after the Pod status is `ready `: -```bash -kubectl exec -i -t tdengine-3 -- taos -s "show dnodes" +```Bash +kubectl exec -it tdengine-3 -n tdengine-test -- taos -s "show dnodes" ``` -The following output shows that the TDengine cluster has been expanded to 4 replicas: +The dnode list of the expanded four-node TDengine cluster: -``` +```Plain taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | - 4 | tdengine-3.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:33:16.039 | | -Query OK, 4 rows in database (0.008377s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | + 4 | tdengine-3.ta... | 0 | 16 | ready | 2023-07-20 16:01:44.007 | 2023-07-20 16:01:44.889 | | | | +Query OK, 4 row(s) in set (0.003628s) ``` ## Scaling In Your Cluster -When you scale in a TDengine cluster, your data is migrated to different nodes. You must run the drop dnodes command in TDengine to remove dnodes before scaling in your Kubernetes environment. - -Note: In a Kubernetes StatefulSet service, the newest pods are always removed first. For this reason, when you scale in your TDengine cluster, ensure that you drop the newest dnodes. +Since the TDengine cluster will migrate data between nodes during volume expansion and contraction, using the **kubectl** command to reduce the volume requires first using the "drop dnodes" command ( **If there are 3 replicas of db in the cluster, the number of dnodes after reduction must also be greater than or equal to 3, otherwise the drop dnode operation will be aborted** ), the node deletion is completed before Kubernetes cluster reduction. -``` -$ kubectl exec -i -t tdengine-0 -- taos -s "drop dnode 4" -``` +Note: Since Kubernetes Pods in the Statefulset can only be removed in reverse order of creation, the TDengine drop dnode also needs to be removed in reverse order of creation, otherwise the Pod will be in an error state. -```bash -$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "drop dnode 4" +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | -Query OK, 3 rows in database (0.004861s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | +Query OK, 3 row(s) in set (0.003324s) ``` -Verify that the dnode have been successfully removed by running the `kubectl exec -i -t tdengine-0 -- taos -s "show dnodes"` command. Then run the following command to remove the pod: +After confirming that the removal is successful (use kubectl exec -i -t tdengine-0 --taos -s "show dnodes" to view and confirm the dnode list), use the kubectl command to remove the Pod: -``` -kubectl scale statefulsets tdengine --replicas=3 +```Plain +kubectl scale statefulsets tdengine --replicas=3 -n tdengine-test ``` -The newest pod in the deployment is removed. Run the `kubectl get pods -l app=tdengine` command to query the pod status: +The last Pod will be deleted. Use the command kubectl get pods -l app = tdengine to check the Pod status: -``` -$ kubectl get pods -l app=tdengine -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 4m7s -tdengine-1 1/1 Running 0 3m55s -tdengine-2 1/1 Running 0 2m28s +```Plain +kubectl get pod -l app=tdengine -n tdengine-test -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 1/1 Running 4 (6h55m ago) 7h22m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (7h9m ago) 7h23m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 5h45m 10.244.1.224 node85 ``` -After the pod has been removed, manually delete the PersistentVolumeClaim (PVC). Otherwise, future scale-outs will attempt to use existing data. +After the Pod is deleted, the PVC needs to be deleted manually, otherwise the previous data will continue to be used for the next expansion, resulting in the inability to join the cluster normally. -```bash -$ kubectl delete pvc taosdata-tdengine-3 +```Bash +kubectl delete pvc aosdata-tdengine-3 -n tdengine-test ``` -Your cluster has now been safely scaled in, and you can scale it out again as necessary. +The cluster state at this time is safe and can be scaled up again if needed. -```bash -$ kubectl scale statefulsets tdengine --replicas=4 +```Bash +kubectl scale statefulsets tdengine --replicas=4 -n tdengine-test statefulset.apps/tdengine scaled -it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl get pods -l app=tdengine -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 35m -tdengine-1 1/1 Running 0 34m -tdengine-2 1/1 Running 0 12m -tdengine-3 0/1 ContainerCreating 0 4s -it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl get pods -l app=tdengine -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 35m -tdengine-1 1/1 Running 0 34m -tdengine-2 1/1 Running 0 12m -tdengine-3 0/1 Running 0 7s -it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" + +kubectl get pod -l app=tdengine -n tdengine-test -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 1/1 Running 4 (6h59m ago) 7h27m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (7h13m ago) 7h27m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 5h49m 10.244.1.224 node85 +tdengine-3 1/1 Running 0 20s 10.244.2.77 node86 + +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" taos> show dnodes -id | endpoint | vnodes | support_vnodes | status | create_time | offline reason | -====================================================================================================================================== -1 | tdengine-0.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 17:38:49.012 | | -2 | tdengine-1.taosd.default.sv... | 1 | 4 | ready | 2022-07-25 17:39:01.517 | | -5 | tdengine-2.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 18:01:36.479 | | -6 | tdengine-3.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 18:13:54.411 | | -Query OK, 4 row(s) in set (0.001348s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | + 5 | tdengine-3.ta... | 0 | 16 | ready | 2023-07-20 16:31:34.092 | 2023-07-20 16:38:17.419 | | | | +Query OK, 4 row(s) in set (0.003881s) ``` ## Remove a TDengine Cluster -To fully remove a TDengine cluster, you must delete its statefulset, svc, configmap, and pvc entries: +> **When deleting the PVC, you need to pay attention to the pv persistentVolumeReclaimPolicy policy. It is recommended to change to Delete, so that the PV will be automatically cleaned up when the PVC is deleted, and the underlying CSI storage resources will be cleaned up at the same time. If the policy of deleting the PVC to automatically clean up the PV is not configured, and then after deleting the pvc, when manually cleaning up the PV, the CSI storage resources corresponding to the PV may not be released.** -```bash -kubectl delete statefulset -l app=tdengine -kubectl delete svc -l app=tdengine -kubectl delete pvc -l app=tdengine -kubectl delete configmap taoscfg +Complete removal of TDengine cluster, need to clean up statefulset, svc, configmap, pvc respectively. +```Bash +kubectl delete statefulset -l app=tdengine -n tdengine-test +kubectl delete svc -l app=tdengine -n tdengine-test +kubectl delete pvc -l app=tdengine -n tdengine-test +kubectl delete configmap taoscfg -n tdengine-test ``` ## Troubleshooting ### Error 1 -If you remove a pod without first running `drop dnode`, some TDengine nodes will go offline. +No "drop dnode" is directly reduced. Since the TDengine has not deleted the node, the reduced pod causes some nodes in the TDengine cluster to be offline. -``` -$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" +```Plain +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" taos> show dnodes -id | endpoint | vnodes | support_vnodes | status | create_time | offline reason | -====================================================================================================================================== -1 | tdengine-0.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 17:38:49.012 | | -2 | tdengine-1.taosd.default.sv... | 1 | 4 | ready | 2022-07-25 17:39:01.517 | | -5 | tdengine-2.taosd.default.sv... | 0 | 4 | offline | 2022-07-25 18:01:36.479 | status msg timeout | -6 | tdengine-3.taosd.default.sv... | 0 | 4 | offline | 2022-07-25 18:13:54.411 | status msg timeout | -Query OK, 4 row(s) in set (0.001323s) -``` - -### Error 2 - -If the number of nodes after a scale-in is less than the value of the replica parameter, the cluster will go down: - -Create a database with replica set to 2 and add data. - -```bash -kubectl exec -i -t tdengine-0 -- \ - taos -s \ - "create database if not exists test replica 2; - use test; - create table if not exists t1(ts timestamp, n int); - insert into t1 values(now, 1)(now+1s, 2);" - - + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | + 5 | tdengine-3.ta... | 0 | 16 | offline | 2023-07-20 16:31:34.092 | 2023-07-20 16:38:17.419 | status msg timeout | | | +Query OK, 4 row(s) in set (0.003862s) ``` -Scale in to one node: +## Finally -```bash -kubectl scale statefulsets tdengine --replicas=1 +For the high availability and high reliability of TDengine in a Kubernetes environment, hardware damage and disaster recovery are divided into two levels: -``` +1. The disaster recovery capability of the underlying distributed Block Storage, the multi-copy of Block Storage, the current popular distributed Block Storage such as Ceph, has the multi-copy capability, extending the storage copy to different racks, cabinets, computer rooms, Data center (or directly use the Block Storage service provided by Public Cloud vendors) +2. TDengine disaster recovery, in TDengine Enterprise, itself has when a dnode permanently offline (TCE-metal disk damage, data sorting loss), re-pull a blank dnode to restore the original dnode work. -In the TDengine CLI, you can see that no database operations succeed: +Finally, welcome to [TDengine Cloud ](https://cloud.tdengine.com/)to experience the one-stop fully managed TDengine Cloud as a Service. -``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 2 | 40 | ready | any | 2021-06-01 15:55:52.562 | | - 2 | tdengine-1.taosd.default.sv... | 1 | 40 | offline | any | 2021-06-01 15:56:07.212 | status msg timeout | -Query OK, 2 row(s) in set (0.000845s) - -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 2 | 40 | ready | any | 2021-06-01 15:55:52.562 | | - 2 | tdengine-1.taosd.default.sv... | 1 | 40 | offline | any | 2021-06-01 15:56:07.212 | status msg timeout | -Query OK, 2 row(s) in set (0.000837s) - -taos> use test; -Database changed. - -taos> insert into t1 values(now, 3); - -DB error: Unable to resolve FQDN (0.013874s) - -``` +> TDengine Cloud is a minimalist fully managed time series data processing Cloud as a Service platform developed based on the open source time series database TDengine. In addition to high-performance time series database, it also has system functions such as caching, subscription and stream computing, and provides convenient and secure data sharing, as well as numerous enterprise-level functions. It allows enterprises in the fields of Internet of Things, Industrial Internet, Finance, IT operation and maintenance monitoring to significantly reduce labor costs and operating costs in the management of time series data. diff --git a/docs/en/10-deployment/index.md b/docs/en/10-deployment/index.md index 865fbc2da57bb985dabc03a743c5e0f7186b98be..0079ad374084c4d91d8bdf74246b431fe60e56b9 100644 --- a/docs/en/10-deployment/index.md +++ b/docs/en/10-deployment/index.md @@ -5,7 +5,7 @@ description: This document describes how to deploy a TDengine cluster on a serve TDengine has a native distributed design and provides the ability to scale out. A few nodes can form a TDengine cluster. If you need higher processing power, you just need to add more nodes into the cluster. TDengine uses virtual node technology to virtualize a node into multiple virtual nodes to achieve load balancing. At the same time, TDengine can group virtual nodes on different nodes into virtual node groups, and use the replication mechanism to ensure the high availability of the system. The cluster feature of TDengine is completely open source. -This document describes how to manually deploy a cluster on a host as well as how to deploy on Kubernetes and by using Helm. +This document describes how to manually deploy a cluster on a host directly and deploy a cluster with Docker, Kubernetes or Helm. ```mdx-code-block import DocCardList from '@theme/DocCardList'; diff --git a/docs/en/12-taos-sql/01-data-type.md b/docs/en/12-taos-sql/01-data-type.md index cca256139d409ac009bb188ccb377bffc0dd42ef..f81aaceca37b4e8ad4fb734fcc9f2adab808c89e 100644 --- a/docs/en/12-taos-sql/01-data-type.md +++ b/docs/en/12-taos-sql/01-data-type.md @@ -42,11 +42,20 @@ In TDengine, the data types below can be used when specifying a column or tag. | 14 | NCHAR | User Defined | Multi-byte string that can include multi byte characters like Chinese characters. Each character of NCHAR type consumes 4 bytes storage. The string value should be quoted with single quotes. Literal single quote inside the string must be preceded with backslash, like `\'`. The length must be specified when defining a column or tag of NCHAR type, for example nchar(10) means it can store at most 10 characters of nchar type and will consume fixed storage of 40 bytes. An error will be reported if the string value exceeds the length defined. | | 15 | JSON | | JSON type can only be used on tags. A tag of json type is excluded with any other tags of any other type. | | 16 | VARCHAR | User-defined | Alias of BINARY | - +| 17 | GEOMETRY | User-defined | Geometry | :::note +- Each row of the table cannot be longer than 48KB (64KB since version 3.0.5.0) (note that each BINARY/NCHAR/GEOMETRY column takes up an additional 2 bytes of storage space). - Only ASCII visible characters are suggested to be used in a column or tag of BINARY type. Multi-byte characters must be stored in NCHAR type. -- The length of BINARY can be up to 16,374 bytes. The string value must be quoted with single quotes. You must specify a length in bytes for a BINARY value, for example binary(20) for up to twenty single-byte characters. If the data exceeds the specified length, an error will occur. The literal single quote inside the string must be preceded with back slash like `\'` +- The length of BINARY can be up to 16,374(data column is 65,517 and tag column is 16,382 since version 3.0.5.0) bytes. The string value must be quoted with single quotes. You must specify a length in bytes for a BINARY value, for example binary(20) for up to twenty single-byte characters. If the data exceeds the specified length, an error will occur. The literal single quote inside the string must be preceded with back slash like `\'` +- The maximum length of the GEOMETRY data column is 65,517 bytes, and the maximum length of the tag column is 16,382 bytes. Supports POINT, LINESTRING, and POLYGON subtypes of 2D. The following table describes the length calculation method: + + | # | **Syntax** | **MinLen** | **MaxLen** | **Growth of each point** | + |---|--------------------------------------|------------|------------|--------------------------| + | 1 | POINT(1.0 1.0) | 21 | 21 | NA | + | 2 | LINESTRING(1.0 1.0, 2.0 2.0) | 9+2*16 | 9+4094*16 | +16 | + | 3 | POLYGON((1.0 1.0, 2.0 2.0, 1.0 1.0)) | 13+3*16 | 13+4094*16 | +16 | + - Numeric values in SQL statements will be determined as integer or float type according to whether there is decimal point or whether scientific notation is used, so attention must be paid to avoid overflow. For example, 9999999999999999999 will be considered as overflow because it exceeds the upper limit of long integer, but 9999999999999999999.0 will be considered as a legal float number. ::: diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index af619c11a5c5ca1e72b7ee6edc4b3106da1e0f1c..e7f3aa8d1bd1a2b4c675af22813e1775c6105b7c 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -36,14 +36,12 @@ database_option: { | TSDB_PAGESIZE value | WAL_RETENTION_PERIOD value | WAL_RETENTION_SIZE value - | WAL_ROLL_PERIOD value - | WAL_SEGMENT_SIZE value } ``` ## Parameters -- BUFFER: specifies the size (in MB) of the write buffer for each vnode. Enter a value between 3 and 16384. The default value is 96. +- BUFFER: specifies the size (in MB) of the write buffer for each vnode. Enter a value between 3 and 16384. The default value is 256. - CACHEMODEL: specifies how the latest data in subtables is stored in the cache. The default value is none. - none: The latest data is not cached. - last_row: The last row of each subtable is cached. This option significantly improves the performance of the LAST_ROW function. @@ -58,7 +56,7 @@ database_option: { - WAL_FSYNC_PERIOD: specifies the interval (in milliseconds) at which data is written from the WAL to disk. This parameter takes effect only when the WAL parameter is set to 2. The default value is 3000. Enter a value between 0 and 180000. The value 0 indicates that incoming data is immediately written to disk. - MAXROWS: specifies the maximum number of rows recorded in a block. The default value is 4096. - MINROWS: specifies the minimum number of rows recorded in a block. The default value is 100. -- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. The Enterprise Edition supports [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; the Community Edition does not support Tiered Storage function (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). +- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. TDengine Enterprise supports [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; TDengine OSS does not support Tiered Storage function (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). - PAGES: specifies the number of pages in the metadata storage engine cache on each vnode. Enter a value greater than or equal to 64. The default value is 256. The space occupied by metadata storage on each vnode is equal to the product of the values of the PAGESIZE and PAGES parameters. The space occupied by default is 1 MB. - PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode. The default value is 4. Enter a value between 1 and 16384. - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms. @@ -75,10 +73,8 @@ database_option: { - TABLE_PREFIX: The prefix in the table name that is ignored when distributing a table to a vgroup when it's a positive number, or only the prefix is used when distributing a table to a vgroup, the default value is 0; For example, if the table name v30001, then "0001" is used if TSDB_PREFIX is set to 2 but "v3" is used if TSDB_PREFIX is set to -2; It can help you to control the distribution of tables. - TABLE_SUFFIX: The suffix in the table name that is ignored when distributing a table to a vgroup when it's a positive number, or only the suffix is used when distributing a table to a vgroup, the default value is 0; For example, if the table name v30001, then "v300" is used if TSDB_SUFFIX is set to 2 but "01" is used if TSDB_SUFFIX is set to -2; It can help you to control the distribution of tables. - TSDB_PAGESIZE: The page size of the data storage engine in a vnode. The unit is KB. The default is 4 KB. The range is 1 to 16384, that is, 1 KB to 16 MB. -- WAL_RETENTION_PERIOD: specifies the maximum time of which WAL files are to be kept for consumption. This parameter is used for data subscription. Enter a time in seconds. The default value 0. A value of 0 indicates that WAL files are not required to keep for consumption. Alter it with a proper value at first to create topics. +- WAL_RETENTION_PERIOD: specifies the maximum time of which WAL files are to be kept for consumption. This parameter is used for data subscription. Enter a time in seconds. The default value is 3600, which means the data in latest 3600 seconds will be kept in WAL for data subscription. Please adjust this parameter to a more proper value for your data subscription. - WAL_RETENTION_SIZE: specifies the maximum total size of which WAL files are to be kept for consumption. This parameter is used for data subscription. Enter a size in KB. The default value is 0. A value of 0 indicates that the total size of WAL files to keep for consumption has no upper limit. -- WAL_ROLL_PERIOD: specifies the time after which WAL files are rotated. After this period elapses, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after TSDB data in memory are flushed to disk. -- WAL_SEGMENT_SIZE: specifies the maximum size of a WAL file. After the current WAL file reaches this size, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after TSDB data in memory are flushed to disk. ### Example Statement ```sql diff --git a/docs/en/12-taos-sql/03-table.md b/docs/en/12-taos-sql/03-table.md index f61d1f51471bf042aea79023bff4f74c43831eb8..10c44848c95632777c98b936929f615c550accb2 100644 --- a/docs/en/12-taos-sql/03-table.md +++ b/docs/en/12-taos-sql/03-table.md @@ -9,27 +9,27 @@ You create standard tables and subtables with the `CREATE TABLE` statement. ```sql CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definition] ...) [table_options] - + CREATE TABLE create_subtable_clause - + CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definition] ...) [TAGS (create_definition [, create_definition] ...)] [table_options] - + create_subtable_clause: { create_subtable_clause [create_subtable_clause] ... | [IF NOT EXISTS] [db_name.]tb_name USING [db_name.]stb_name [(tag_name [, tag_name] ...)] TAGS (tag_value [, tag_value] ...) } - + create_definition: col_name column_definition - + column_definition: type_name [comment 'string_value'] - + table_options: table_option ... - + table_option: { COMMENT 'string_value' | WATERMARK duration[,duration] @@ -45,9 +45,9 @@ table_option: { 1. The first column of a table MUST be of type TIMESTAMP. It is automatically set as the primary key. 2. The maximum length of the table name is 192 bytes. -3. The maximum length of each row is 48k bytes, please note that the extra 2 bytes used by each BINARY/NCHAR column are also counted. +3. The maximum length of each row is 48k(64k since version 3.0.5.0) bytes, please note that the extra 2 bytes used by each BINARY/NCHAR/GEOMETRY column are also counted. 4. The name of the subtable can only consist of characters from the English alphabet, digits and underscore. Table names can't start with a digit. Table names are case insensitive. -5. The maximum length in bytes must be specified when using BINARY or NCHAR types. +5. The maximum length in bytes must be specified when using BINARY/NCHAR/GEOMETRY types. 6. Escape character "\`" can be used to avoid the conflict between table names and reserved keywords, above rules will be bypassed when using escape character on table names, but the upper limit for the name length is still valid. The table names specified using escape character are case sensitive. For example \`aBc\` and \`abc\` are different table names but `abc` and `aBc` are same table names because they are both converted to `abc` internally. Only ASCII visible characters can be used with escape character. @@ -58,7 +58,7 @@ table_option: { 3. MAX_DELAY: specifies the maximum latency for pushing computation results. The default value is 15 minutes or the value of the INTERVAL parameter, whichever is smaller. Enter a value between 0 and 15 minutes in milliseconds, seconds, or minutes. You can enter multiple values separated by commas (,). Note: Retain the default value if possible. Configuring a small MAX_DELAY may cause results to be frequently pushed, affecting storage and query performance. This parameter applies only to supertables and takes effect only when the RETENTIONS parameter has been specified for the database. 4. ROLLUP: specifies aggregate functions to roll up. Rolling up a function provides downsampled results based on multiple axes. This parameter applies only to supertables and takes effect only when the RETENTIONS parameter has been specified for the database. You can specify only one function to roll up. The rollup takes effect on all columns except TS. Enter one of the following values: avg, sum, min, max, last, or first. 5. SMA: specifies functions on which to enable small materialized aggregates (SMA). SMA is user-defined precomputation of aggregates based on data blocks. Enter one of the following values: max, min, or sum This parameter can be used with supertables and standard tables. -6. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The default value is 0, i.e. never expire. +6. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The default value is 0, i.e. never expire. ## Create Subtables @@ -88,7 +88,7 @@ You can create multiple subtables in a single SQL statement provided that all su ```sql ALTER TABLE [db_name.]tb_name alter_table_clause - + alter_table_clause: { alter_table_options | ADD COLUMN col_name column_type @@ -96,10 +96,10 @@ alter_table_clause: { | MODIFY COLUMN col_name column_type | RENAME COLUMN old_col_name new_col_name } - + alter_table_options: alter_table_option ... - + alter_table_option: { TTL value | COMMENT 'string_value' @@ -142,15 +142,15 @@ ALTER TABLE tb_name RENAME COLUMN old_col_name new_col_name ```sql ALTER TABLE [db_name.]tb_name alter_table_clause - + alter_table_clause: { alter_table_options | SET TAG tag_name = new_tag_value } - + alter_table_options: alter_table_option ... - + alter_table_option: { TTL value | COMMENT 'string_value' diff --git a/docs/en/12-taos-sql/04-stable.md b/docs/en/12-taos-sql/04-stable.md index 3a4d6cc590279ad00a20a11ff95a53b4284a7b08..66d93aed580ae04e8293950b253f1b6429660b72 100644 --- a/docs/en/12-taos-sql/04-stable.md +++ b/docs/en/12-taos-sql/04-stable.md @@ -51,6 +51,11 @@ DESCRIBE [db_name.]stb_name; ### View tag information for all child tables in the supertable +``` +SHOW TABLE TAGS FROM table_name [FROM db_name]; +SHOW TABLE TAGS FROM [db_name.]table_name; +``` + ``` taos> SHOW TABLE TAGS FROM st1; tbname | id | loc | diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index ea0d7abc16b18a92147642d86b167d96903ebfda..b28d5acb1828ddb67cf1fc2615973409362063af 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -55,7 +55,7 @@ window_clause: { | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [WATERMARK(watermark_val)] [FILL(fill_mod_and_val)] interp_clause: - RANGE(ts_val, ts_val) EVERY(every_val) FILL(fill_mod_and_val) + RANGE(ts_val [, ts_val]) EVERY(every_val) FILL(fill_mod_and_val) partition_by_clause: PARTITION BY expr [, expr] ... diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 7e45ffa1df1f8b9814daafeee7a28016eca911e4..234625bfb4bc60e712d41291f8c18aa8cbe5a230 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -672,7 +672,7 @@ If you input a specific column, the number of non-null values in the column is r ELAPSED(ts_primary_key [, time_unit]) ``` -**Description**: `elapsed` function can be used to calculate the continuous time length in which there is valid data. If it's used with `INTERVAL` clause, the returned result is the calculated time length within each time window. If it's used without `INTERVAL` caluse, the returned result is the calculated time length within the specified time range. Please be noted that the return value of `elapsed` is the number of `time_unit` in the calculated time length. +**Description**: `elapsed` function can be used to calculate the continuous time length in which there is valid data. If it's used with `INTERVAL` clause, the returned result is the calculated time length within each time window. If it's used without `INTERVAL` clause, the returned result is the calculated time length within the specified time range. Please be noted that the return value of `elapsed` is the number of `time_unit` in the calculated time length. **Return value type**: Double if the input value is not NULL; @@ -698,7 +698,7 @@ ELAPSED(ts_primary_key [, time_unit]) LEASTSQUARES(expr, start_val, step_val) ``` -**Description**: The linear regression function of the specified column and the timestamp column (primary key), `start_val` is the initial value and `step_val` is the step value. +**Description**: The linear regression function of a specified column, `start_val` is the initial value and `step_val` is the step value. **Return value type**: A string in the format of "(slope, intercept)" @@ -867,10 +867,16 @@ FIRST(expr) ### INTERP ```sql -INTERP(expr) +INTERP(expr [, ignore_null_values]) + +ignore_null_values: { + 0 + | 1 +} ``` -**Description**: The value that matches the specified timestamp range is returned, if existing; or an interpolation value is returned. +**Description**: The value that matches the specified timestamp range is returned, if existing; or an interpolation value is returned. The value of `ignore_null_values` can be 0 or 1, 1 means null values are ignored. The default value of this parameter is 0. + **Return value type**: Same as the column being operated upon @@ -883,9 +889,10 @@ INTERP(expr) - `INTERP` is used to get the value that matches the specified time slice from a column. If no such value exists an interpolation value will be returned based on `FILL` parameter. - The input data of `INTERP` is the value of the specified column and a `where` clause can be used to filter the original data. If no `where` condition is specified then all original data is the input. - `INTERP` must be used along with `RANGE`, `EVERY`, `FILL` keywords. -- The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1 <= timestamp2. timestamp1 is the starting point of the output time range and must be specified. timestamp2 is the ending point of the output time range and must be specified. +- The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1 <= timestamp2. timestamp1 is the starting point of the output time range. timestamp2 is the ending point of the output time range. - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY(time_unit)`. Starting from timestamp1, one interpolation is performed for every time interval specified `time_unit` parameter. The parameter `time_unit` must be an integer, with no quotes, with a time unit of: a(millisecond)), s(second), m(minute), h(hour), d(day), or w(week). For example, `EVERY(500a)` will interpolate every 500 milliseconds. - Interpolation is performed based on `FILL` parameter. For more information about FILL clause, see [FILL Clause](../distinguished/#fill-clause). +- When only one timestamp value is specified in `RANGE` clause, `INTERP` is used to generate interpolation at this point in time. In this case, `EVERY` clause can be omitted. For example, SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00') FILL(linear). - `INTERP` can be applied to supertable by interpolating primary key sorted data of all its childtables. It can also be used with `partition by tbname` when applied to supertable to generate interpolation on each single timeline. - Pseudocolumn `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.2.0). - Pseudocolumn `_isfilled` can be used along with `INTERP` to indicate whether the results are original records or data points generated by interpolation algorithm(support after version 3.0.3.0). @@ -896,7 +903,7 @@ INTERP(expr) - We want to downsample every 1 hour and use a linear fill for missing values. Note the order in which the "partition by" clause and the "range", "every" and "fill" parameters are used. ```sql -SELECT _irowts,INTERP(current) FROM test.meters PARTITION BY TBNAME RANGE('2017-07-22 00:00:00','2017-07-24 12:25:00') EVERY(1h) FILL(LINEAR) +SELECT _irowts,INTERP(current) FROM test.meters PARTITION BY TBNAME RANGE('2017-07-22 00:00:00','2017-07-24 12:25:00') EVERY(1h) FILL(LINEAR) ``` ### LAST @@ -992,19 +999,14 @@ SAMPLE(expr, k) **Description**: _k_ sampling values of a specific column. The applicable range of _k_ is [1,1000]. -**Return value type**: Same as the column being operated plus the associated timestamp +**Return value type**: Same as the column being operated -**Applicable data types**: Any data type except for tags of STable +**Applicable data types**: Any data type **Applicable nested query**: Inner query and Outer query **Applicable table types**: standard tables and supertables -**More explanations**: - -This function cannot be used in expression calculation. -- Must be used with `PARTITION BY tbname` when it's used on a STable to force the result on each single timeline - ### TAIL @@ -1049,11 +1051,11 @@ TOP(expr, k) UNIQUE(expr) ``` -**Description**: The values that occur the first time in the specified column. The effect is similar to `distinct` keyword, but it can also be used to match tags or timestamp. The first occurrence of a timestamp or tag is used. +**Description**: The values that occur the first time in the specified column. The effect is similar to `distinct` keyword. **Return value type**:Same as the data type of the column being operated upon -**Applicable column types**: Any data types except for timestamp +**Applicable column types**: Any data types **Applicable table types**: table, STable @@ -1082,7 +1084,6 @@ CSUM(expr) - Arithmetic operation can't be performed on the result of `csum` function - Can only be used with aggregate functions This function can be used with supertables and standard tables. -- Must be used with `PARTITION BY tbname` when it's used on a STable to force the result on each single timeline ### DERIVATIVE @@ -1106,7 +1107,6 @@ ignore_negative: { **More explanation**: -- It can be used together with `PARTITION BY tbname` against a STable. - It can be used together with a selected column. For example: select \_rowts, DERIVATIVE() from. ### DIFF @@ -1169,7 +1169,6 @@ MAVG(expr, k) - Arithmetic operation can't be performed on the result of `MAVG`. - Can only be used with data columns, can't be used with tags. - Can't be used with aggregate functions. -- Must be used with `PARTITION BY tbname` when it's used on a STable to force the result on each single timeline ### STATECOUNT @@ -1195,7 +1194,6 @@ STATECOUNT(expr, oper, val) **More explanations**: -- Must be used together with `PARTITION BY tbname` when it's used on a STable to force the result into each single timeline] - Can't be used with window operation, like interval/state_window/session_window @@ -1223,7 +1221,6 @@ STATEDURATION(expr, oper, val, unit) **More explanations**: -- Must be used together with `PARTITION BY tbname` when it's used on a STable to force the result into each single timeline] - Can't be used with window operation, like interval/state_window/session_window @@ -1241,7 +1238,6 @@ TWA(expr) **Applicable table types**: standard tables and supertables -- Must be used together with `PARTITION BY tbname` to force the result into each single timeline. ## System Information Functions @@ -1278,3 +1274,161 @@ SELECT SERVER_STATUS(); ``` **Description**: The server status. + + +## Geometry Functions + +### Geometry Input Functions + +Geometry input functions create geometry data from WTK. + +#### ST_GeomFromText + +```sql +ST_GeomFromText(VARCHAR WKT expr) +``` + +**Description**: Return a specified GEOMETRY value from Well-Known Text representation (WKT). + +**Return value type**: GEOMETRY + +**Applicable data types**: VARCHAR + +**Applicable table types**: standard tables and supertables + +**Explanations**： +- The input can be one of WTK string, like POINT, LINESTRING, POLYGON, MULTIPOINT, MULTILINESTRING, MULTIPOLYGON, GEOMETRYCOLLECTION. +- The output is a GEOMETRY data type, internal defined as binary string. + +### Geometry Output Functions + +Geometry output functions convert geometry data into WTK. + +#### ST_AsText + +```sql +ST_AsText(GEOMETRY geom) +``` + +**Description**: Return a specified Well-Known Text representation (WKT) value from GEOMETRY data. + +**Return value type**: VARCHAR + +**Applicable data types**: GEOMETRY + +**Applicable table types**: standard tables and supertables + +**Explanations**： +- The output can be one of WTK string, like POINT, LINESTRING, POLYGON, MULTIPOINT, MULTILINESTRING, MULTIPOLYGON, GEOMETRYCOLLECTION. + +### Geometry Relationships Functions + +Geometry relationships functions determine spatial relationships between geometries. + +#### ST_Intersects + +```sql +ST_Intersects(GEOMETRY geomA, GEOMETRY geomB) +``` + +**Description**: Compares two geometries and returns true if they intersect. + +**Return value type**: BOOL + +**Applicable data types**: GEOMETRY, GEOMETRY + +**Applicable table types**: standard tables and supertables + +**Explanations**： +- Geometries intersect if they have any point in common. + + +#### ST_Equals + +```sql +ST_Equals(GEOMETRY geomA, GEOMETRY geomB) +``` + +**Description**: Returns TRUE if the given geometries are "spatially equal". + +**Return value type**: BOOL + +**Applicable data types**: GEOMETRY, GEOMETRY + +**Applicable table types**: standard tables and supertables + +**Explanations**： +- 'Spatially equal' means ST_Contains(A,B) = true and ST_Contains(B,A) = true, and the ordering of points can be different but represent the same geometry structure. + + +#### ST_Touches + +```sql +ST_Touches(GEOMETRY geomA, GEOMETRY geomB) +``` + +**Description**: Returns TRUE if A and B intersect, but their interiors do not intersect. + +**Return value type**: BOOL + +**Applicable data types**: GEOMETRY, GEOMETRY + +**Applicable table types**: standard tables and supertables + +**Explanations**： +- A and B have at least one point in common, and the common points lie in at least one boundary. +- For Point/Point inputs the relationship is always FALSE, since points do not have a boundary. + + +#### ST_Covers + +```sql +ST_Covers(GEOMETRY geomA, GEOMETRY geomB) +``` + +**Description**: Returns TRUE if every point in Geometry B lies inside (intersects the interior or boundary of) Geometry A. + +**Return value type**: BOOL + +**Applicable data types**: GEOMETRY, GEOMETRY + +**Applicable table types**: standard tables and supertables + +**Explanations**： +- A covers B means no point of B lies outside (in the exterior of) A. + + +#### ST_Contains + +```sql +ST_Contains(GEOMETRY geomA, GEOMETRY geomB) +``` + +**Description**: Returns TRUE if geometry A contains geometry B. + +**Return value type**: BOOL + +**Applicable data types**: GEOMETRY, GEOMETRY + +**Applicable table types**: standard tables and supertables + +**Explanations**： +- A contains B if and only if all points of B lie inside (i.e. in the interior or boundary of) A (or equivalently, no points of B lie in the exterior of A), and the interiors of A and B have at least one point in common. + + +#### ST_ContainsProperly + +```sql +ST_ContainsProperly(GEOMETRY geomA, GEOMETRY geomB) +``` + +**Description**: Returns TRUE if every point of B lies inside A. + +**Return value type**: BOOL + +**Applicable data types**: GEOMETRY, GEOMETRY + +**Applicable table types**: standard tables and supertables + +**Explanations**： +- There is no point of B that lies on the boundary of A or in the exterior of A. diff --git a/docs/en/12-taos-sql/12-distinguished.md b/docs/en/12-taos-sql/12-distinguished.md index b082f7b888dc2e09e4a6ae5c38e7ece591d667e3..7f0b8c7769298b460ec7102d5e3fc0b8f2637ca7 100644 --- a/docs/en/12-taos-sql/12-distinguished.md +++ b/docs/en/12-taos-sql/12-distinguished.md @@ -21,7 +21,7 @@ part_list can be any scalar expression, such as a column, constant, scalar funct A PARTITION BY clause is processed as follows: - The PARTITION BY clause must occur after the WHERE clause -- The PARTITION BY caluse partitions the data according to the specified dimensions, then perform computation on each partition. The performed computation is determined by the rest of the statement - a window clause, GROUP BY clause, or SELECT clause. +- The PARTITION BY clause partitions the data according to the specified dimensions, then perform computation on each partition. The performed computation is determined by the rest of the statement - a window clause, GROUP BY clause, or SELECT clause. - The PARTITION BY clause can be used together with a window clause or GROUP BY clause. In this case, the window or GROUP BY clause takes effect on every partition. For example, the following statement partitions the table by the location tag, performs downsampling over a 10 minute window, and returns the maximum value: ```sql diff --git a/docs/en/12-taos-sql/16-operators.md b/docs/en/12-taos-sql/16-operators.md index 32ad4e7075b6a510cb537016effb6064e6c51794..6b7adb4a3d1acbbf32a7ec40a92c068ff1495c9b 100644 --- a/docs/en/12-taos-sql/16-operators.md +++ b/docs/en/12-taos-sql/16-operators.md @@ -39,7 +39,7 @@ TDengine supports the `UNION` and `UNION ALL` operations. UNION ALL collects all | 3 | \>, < | All types except BLOB, MEDIUMBLOB, and JSON | Greater than and less than | | 4 | \>=, <= | All types except BLOB, MEDIUMBLOB, and JSON | Greater than or equal to and less than or equal to | | 5 | IS [NOT] NULL | All types | Indicates whether the value is null | -| 6 | [NOT] BETWEEN AND | All types except BLOB, MEDIUMBLOB, and JSON | Closed interval comparison | +| 6 | [NOT] BETWEEN AND | All types except BLOB, MEDIUMBLOB, JSON and GEOMETRY | Closed interval comparison | | 7 | IN | All types except BLOB, MEDIUMBLOB, and JSON; the primary key (timestamp) is also not supported | Equal to any value in the list | | 8 | LIKE | BINARY, NCHAR, and VARCHAR | Wildcard match | | 9 | MATCH, NMATCH | BINARY, NCHAR, and VARCHAR | Regular expression match | @@ -54,7 +54,7 @@ LIKE is used together with wildcards to match strings. Its usage is described as MATCH and NMATCH are used together with regular expressions to match strings. Their usage is described as follows: - Use POSIX regular expression syntax. For more information, see Regular Expressions. -- Regular expression can be used against only table names, i.e. `tbname`, and tags of binary/nchar types, but can't be used against data columns. +- Regular expression can be used against only table names, i.e. `tbname`, and tags/columns of binary/nchar types. - The maximum length of regular expression string is 128 bytes. Configuration parameter `maxRegexStringLen` can be used to set the maximum allowed regular expression. It's a configuration parameter on the client side, and will take effect after restarting the client. ## Logical Operators diff --git a/docs/en/12-taos-sql/19-limit.md b/docs/en/12-taos-sql/19-limit.md index 654fae7560d3c24df1353b87952a3868ca307418..23bb8ce91705ddbb0a7e1d3580072d467efa95ad 100644 --- a/docs/en/12-taos-sql/19-limit.md +++ b/docs/en/12-taos-sql/19-limit.md @@ -26,7 +26,7 @@ The following characters cannot occur in a password: single quotation marks ('), - Maximum length of database name is 64 bytes - Maximum length of table name is 192 bytes, excluding the database name prefix and the separator. -- Maximum length of each data row is 48K bytes. Note that the upper limit includes the extra 2 bytes consumed by each column of BINARY/NCHAR type. +- Maximum length of each data row is 48K(64K since version 3.0.5.0) bytes. Note that the upper limit includes the extra 2 bytes consumed by each column of BINARY/NCHAR type. - The maximum length of a column name is 64 bytes. - Maximum number of columns is 4096. There must be at least 2 columns, and the first column must be timestamp. - The maximum length of a tag name is 64 bytes @@ -36,7 +36,7 @@ The following characters cannot occur in a password: single quotation marks ('), - Maximum numbers of databases, STables, tables are dependent only on the system resources. - The number of replicas can only be 1 or 3. - The maximum length of a username is 23 bytes. -- The maximum length of a password is 128 bytes. +- The maximum length of a password is 31 bytes. - The maximum number of rows depends on system resources. - The maximum number of vnodes in a database is 1024. diff --git a/docs/en/12-taos-sql/20-keywords.md b/docs/en/12-taos-sql/20-keywords.md index a2191c87ee17444d5341b8e97b33a954be84dc98..d563181b876afe0a2884f514de1cc09102f708c7 100644 --- a/docs/en/12-taos-sql/20-keywords.md +++ b/docs/en/12-taos-sql/20-keywords.md @@ -178,6 +178,7 @@ The following list shows all reserved keywords: - MATCH - MAX_DELAY +- MAX_SPEED - MAXROWS - MERGE - META @@ -334,8 +335,6 @@ The following list shows all reserved keywords: - WAL_LEVEL - WAL_RETENTION_PERIOD - WAL_RETENTION_SIZE -- WAL_ROLL_PERIOD -- WAL_SEGMENT_SIZE - WATERMARK - WHERE - WINDOW_CLOSE diff --git a/docs/en/12-taos-sql/22-meta.md b/docs/en/12-taos-sql/22-meta.md index bd745b1c6f9bbe3f3dfad65fe4664c7a136e722d..37304633e76b9c3c62106baa06debe54d5b922a0 100644 --- a/docs/en/12-taos-sql/22-meta.md +++ b/docs/en/12-taos-sql/22-meta.md @@ -28,47 +28,47 @@ This document introduces the tables of INFORMATION_SCHEMA and their structure. Provides information about dnodes. Similar to SHOW DNODES. -| # | **Column** | **Data Type** | **Description** | -| --- | :------------: | ------------ | ------------------------- | -| 1 | vnodes | SMALLINT | Current number of vnodes on the dnode. It should be noted that `vnodes` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 2 | support_vnodes | SMALLINT | Maximum number of vnodes on the dnode | -| 3 | status | BINARY(10) | Current status | -| 4 | note | BINARY(256) | Reason for going offline or other information | -| 5 | id | SMALLINT | Dnode ID | -| 6 | endpoint | BINARY(134) | Dnode endpoint | -| 7 | create | TIMESTAMP | Creation time | +| # | **Column** | **Data Type** | **Description** | +| --- | :------------: | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | vnodes | SMALLINT | Current number of vnodes on the dnode. It should be noted that `vnodes` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 2 | support_vnodes | SMALLINT | Maximum number of vnodes on the dnode | +| 3 | status | BINARY(10) | Current status | +| 4 | note | BINARY(256) | Reason for going offline or other information | +| 5 | id | SMALLINT | Dnode ID | +| 6 | endpoint | BINARY(134) | Dnode endpoint | +| 7 | create | TIMESTAMP | Creation time | ## INS_MNODES Provides information about mnodes. Similar to SHOW MNODES. -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------ | ------------------ | -| 1 | id | SMALLINT | Mnode ID | -| 2 | endpoint | BINARY(134) | Mnode endpoint | -| 3 | role | BINARY(10) | Current role | -| 4 | role_time | TIMESTAMP | Time at which the current role was assumed | -| 5 | create_time | TIMESTAMP | Creation time | +| # | **Column** | **Data Type** | **Description** | +| --- | :---------: | ------------- | ------------------------------------------ | +| 1 | id | SMALLINT | Mnode ID | +| 2 | endpoint | BINARY(134) | Mnode endpoint | +| 3 | role | BINARY(10) | Current role | +| 4 | role_time | TIMESTAMP | Time at which the current role was assumed | +| 5 | create_time | TIMESTAMP | Creation time | ## INS_QNODES Provides information about qnodes. Similar to SHOW QNODES. -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------ | ------------ | -| 1 | id | SMALLINT | Qnode ID | -| 2 | endpoint | BINARY(134) | Qnode endpoint | -| 3 | create_time | TIMESTAMP | Creation time | +| # | **Column** | **Data Type** | **Description** | +| --- | :---------: | ------------- | --------------- | +| 1 | id | SMALLINT | Qnode ID | +| 2 | endpoint | BINARY(134) | Qnode endpoint | +| 3 | create_time | TIMESTAMP | Creation time | ## INS_CLUSTER Provides information about the cluster. -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------ | ---------- | -| 1 | id | BIGINT | Cluster ID | -| 2 | name | BINARY(134) | Cluster name | -| 3 | create_time | TIMESTAMP | Creation time | +| # | **Column** | **Data Type** | **Description** | +| --- | :---------: | ------------- | --------------- | +| 1 | id | BIGINT | Cluster ID | +| 2 | name | BINARY(134) | Cluster name | +| 3 | create_time | TIMESTAMP | Creation time | ## INS_DATABASES @@ -81,7 +81,7 @@ Provides information about user-created databases. Similar to SHOW DATABASES. | 3 | ntables | INT | Number of standard tables and subtables (not including supertables) | | 4 | vgroups | INT | Number of vgroups. It should be noted that `vnodes` is a TDengine keyword and needs to be escaped with ` when used as a column name. | | 6 | replica | INT | Number of replicas. It should be noted that `replica` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 7 | strict | BINARY(3) | Strong consistency. It should be noted that `strict` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 7 | strict | BINARY(4) | Obsoleted | | 8 | duration | INT | Duration for storage of single files. It should be noted that `duration` is a TDengine keyword and needs to be escaped with ` when used as a column name. | | 9 | keep | INT | Data retention period. It should be noted that `keep` is a TDengine keyword and needs to be escaped with ` when used as a column name. | | 10 | buffer | INT | Write cache size per vnode, in MB. It should be noted that `buffer` is a TDengine keyword and needs to be escaped with ` when used as a column name. | @@ -98,202 +98,212 @@ Provides information about user-created databases. Similar to SHOW DATABASES. | 21 | cachesize | INT | Memory per vnode used for caching the newest data. It should be noted that `cachesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | | 22 | wal_level | INT | WAL level. It should be noted that `wal_level` is a TDengine keyword and needs to be escaped with ` when used as a column name. | | 23 | wal_fsync_period | INT | Interval at which WAL is written to disk. It should be noted that `wal_fsync_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 24 | wal_retention_period | INT | WAL retention period. It should be noted that `wal_retention_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 24 | wal_retention_period | INT | WAL retention period, in second. It should be noted that `wal_retention_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | | 25 | wal_retention_size | INT | Maximum WAL size. It should be noted that `wal_retention_size` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 26 | wal_roll_period | INT | WAL rotation period. It should be noted that `wal_roll_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 27 | wal_segment_size | BIGINT | WAL file size. It should be noted that `wal_segment_size` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 28 | stt_trigger | SMALLINT | The threshold for number of files to trigger file merging. It should be noted that `stt_trigger` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 29 | table_prefix | SMALLINT | The prefix length in the table name that is ignored when distributing table to vnode based on table name. It should be noted that `table_prefix` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 30 | table_suffix | SMALLINT | The suffix length in the table name that is ignored when distributing table to vnode based on table name. It should be noted that `table_suffix` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 31 | tsdb_pagesize | INT | The page size for internal storage engine, its unit is KB. It should be noted that `tsdb_pagesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 26 | stt_trigger | SMALLINT | The threshold for number of files to trigger file merging. It should be noted that `stt_trigger` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 27 | table_prefix | SMALLINT | The prefix length in the table name that is ignored when distributing table to vnode based on table name. It should be noted that `table_prefix` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 28 | table_suffix | SMALLINT | The suffix length in the table name that is ignored when distributing table to vnode based on table name. It should be noted that `table_suffix` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 29 | tsdb_pagesize | INT | The page size for internal storage engine, its unit is KB. It should be noted that `tsdb_pagesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | ## INS_FUNCTIONS Provides information about user-defined functions. -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------ | -------------- | -| 1 | name | BINARY(64) | Function name | -| 2 | comment | BINARY(255) | Function description. It should be noted that `comment` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 3 | aggregate | INT | Whether the UDF is an aggregate function. It should be noted that `aggregate` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 4 | output_type | BINARY(31) | Output data type | -| 5 | create_time | TIMESTAMP | Creation time | -| 6 | code_len | INT | Length of the source code | -| 7 | bufsize | INT | Buffer size | -| 8 | func_language | BINARY(31) | UDF programming language | -| 9 | func_body | BINARY(16384) | UDF function body | -| 10 | func_version | INT | UDF function version. starting from 0. Increasing by 1 each time it is updated| +| # | **Column** | **Data Type** | **Description** | +| --- | :-----------: | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | name | BINARY(64) | Function name | +| 2 | comment | BINARY(255) | Function description. It should be noted that `comment` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 3 | aggregate | INT | Whether the UDF is an aggregate function. It should be noted that `aggregate` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 4 | output_type | BINARY(31) | Output data type | +| 5 | create_time | TIMESTAMP | Creation time | +| 6 | code_len | INT | Length of the source code | +| 7 | bufsize | INT | Buffer size | +| 8 | func_language | BINARY(31) | UDF programming language | +| 9 | func_body | BINARY(16384) | UDF function body | +| 10 | func_version | INT | UDF function version. starting from 0. Increasing by 1 each time it is updated | ## INS_INDEXES Provides information about user-created indices. Similar to SHOW INDEX. -| # | **Column** | **Data Type** | **Description** | -| --- | :--------------: | ------------ | ---------------------------------------------------------------------------------- | -| 1 | db_name | BINARY(32) | Database containing the table with the specified index | -| 2 | table_name | BINARY(192) | Table containing the specified index | -| 3 | index_name | BINARY(192) | Index name | -| 4 | db_name | BINARY(64) | Index column | -| 5 | index_type | BINARY(10) | SMA or FULLTEXT index | -| 6 | index_extensions | BINARY(256) | Other information For SMA indices, this shows a list of functions. For FULLTEXT indices, this is null. | +| # | **Column** | **Data Type** | **Description** | +| --- | :--------------: | ------------- | --------------------------------------------------------------------- | +| 1 | db_name | BINARY(32) | Database containing the table with the specified index | +| 2 | table_name | BINARY(192) | Table containing the specified index | +| 3 | index_name | BINARY(192) | Index name | +| 4 | db_name | BINARY(64) | Index column | +| 5 | index_type | BINARY(10) | SMA or tag index | +| 6 | index_extensions | BINARY(256) | Other information For SMA/tag indices, this shows a list of functions | ## INS_STABLES Provides information about supertables. -| # | **Column** | **Data Type** | **Description** | -| --- | :-----------: | ------------ | ------------------------ | -| 1 | stable_name | BINARY(192) | Supertable name | -| 2 | db_name | BINARY(64) | All databases in the supertable | -| 3 | create_time | TIMESTAMP | Creation time | -| 4 | columns | INT | Number of columns | -| 5 | tags | INT | Number of tags. It should be noted that `tags` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 6 | last_update | TIMESTAMP | Last updated time | -| 7 | table_comment | BINARY(1024) | Table description | -| 8 | watermark | BINARY(64) | Window closing time. It should be noted that `watermark` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 9 | max_delay | BINARY(64) | Maximum delay for pushing stream processing results. It should be noted that `max_delay` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 10 | rollup | BINARY(128) | Rollup aggregate function. It should be noted that `rollup` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| # | **Column** | **Data Type** | **Description** | +| --- | :-----------: | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | stable_name | BINARY(192) | Supertable name | +| 2 | db_name | BINARY(64) | All databases in the supertable | +| 3 | create_time | TIMESTAMP | Creation time | +| 4 | columns | INT | Number of columns | +| 5 | tags | INT | Number of tags. It should be noted that `tags` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 6 | last_update | TIMESTAMP | Last updated time | +| 7 | table_comment | BINARY(1024) | Table description | +| 8 | watermark | BINARY(64) | Window closing time. It should be noted that `watermark` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 9 | max_delay | BINARY(64) | Maximum delay for pushing stream processing results. It should be noted that `max_delay` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 10 | rollup | BINARY(128) | Rollup aggregate function. It should be noted that `rollup` is a TDengine keyword and needs to be escaped with ` when used as a column name. | ## INS_TABLES Provides information about standard tables and subtables. -| # | **Column** | **Data Type** | **Description** | -| --- | :-----------: | ------------ | ---------------- | -| 1 | table_name | BINARY(192) | Table name | -| 2 | db_name | BINARY(64) | Database name | -| 3 | create_time | TIMESTAMP | Creation time | -| 4 | columns | INT | Number of columns | -| 5 | stable_name | BINARY(192) | Supertable name | -| 6 | uid | BIGINT | Table ID | -| 7 | vgroup_id | INT | Vgroup ID | -| 8 | ttl | INT | Table time-to-live. It should be noted that `ttl` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 9 | table_comment | BINARY(1024) | Table description | -| 10 | type | BINARY(20) | Table type | +| # | **Column** | **Data Type** | **Description** | +| --- | :-----------: | ------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| 1 | table_name | BINARY(192) | Table name | +| 2 | db_name | BINARY(64) | Database name | +| 3 | create_time | TIMESTAMP | Creation time | +| 4 | columns | INT | Number of columns | +| 5 | stable_name | BINARY(192) | Supertable name | +| 6 | uid | BIGINT | Table ID | +| 7 | vgroup_id | INT | Vgroup ID | +| 8 | ttl | INT | Table time-to-live. It should be noted that `ttl` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 9 | table_comment | BINARY(1024) | Table description | +| 10 | type | BINARY(20) | Table type | ## INS_TAGS -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------- | ---------------------- | -| 1 | table_name | BINARY(192) | Table name | -| 2 | db_name | BINARY(64) | Database name | -| 3 | stable_name | BINARY(192) | Supertable name | -| 4 | tag_name | BINARY(64) | Tag name | -| 5 | tag_type | BINARY(64) | Tag type | -| 6 | tag_value | BINARY(16384) | Tag value | +| # | **Column** | **Data Type** | **Description** | +| --- | :---------: | ------------- | --------------- | +| 1 | table_name | BINARY(192) | Table name | +| 2 | db_name | BINARY(64) | Database name | +| 3 | stable_name | BINARY(192) | Supertable name | +| 4 | tag_name | BINARY(64) | Tag name | +| 5 | tag_type | BINARY(64) | Tag type | +| 6 | tag_value | BINARY(16384) | Tag value | ## INS_COLUMNS -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------- | ---------------------- | -| 1 | table_name | BINARY(192) | Table name | -| 2 | db_name | BINARY(64) | Database name | -| 3 | table_type | BINARY(21) | Table type | -| 4 | col_name | BINARY(64) | Column name | -| 5 | col_type | BINARY(32) | Column type | -| 6 | col_length | INT | Column length | -| 7 | col_precision | INT | Column precision | -| 8 | col_scale | INT | Column scale | -| 9 | col_nullable | INT | Column nullable | +| # | **Column** | **Data Type** | **Description** | +| --- | :-----------: | ------------- | ---------------- | +| 1 | table_name | BINARY(192) | Table name | +| 2 | db_name | BINARY(64) | Database name | +| 3 | table_type | BINARY(21) | Table type | +| 4 | col_name | BINARY(64) | Column name | +| 5 | col_type | BINARY(32) | Column type | +| 6 | col_length | INT | Column length | +| 7 | col_precision | INT | Column precision | +| 8 | col_scale | INT | Column scale | +| 9 | col_nullable | INT | Column nullable | ## INS_USERS Provides information about TDengine users. -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------ | -------- | -| 1 | user_name | BINARY(23) | User name | -| 2 | privilege | BINARY(256) | User permissions | -| 3 | create_time | TIMESTAMP | Creation time | +| # | **Column** | **Data Type** | **Description** | +| --- | :---------: | ------------- | ---------------- | +| 1 | user_name | BINARY(23) | User name | +| 2 | privilege | BINARY(256) | User permissions | +| 3 | create_time | TIMESTAMP | Creation time | ## INS_GRANTS Provides information about TDengine Enterprise Edition permissions. -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------ | -------------------------------------------------- | -| 1 | version | BINARY(9) | Whether the deployment is a licensed or trial version | -| 2 | cpu_cores | BINARY(9) | CPU cores included in license | -| 3 | dnodes | BINARY(10) | Dnodes included in license. It should be noted that `dnodes` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 4 | streams | BINARY(10) | Streams included in license. It should be noted that `streams` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 5 | users | BINARY(10) | Users included in license. It should be noted that `users` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 6 | accounts | BINARY(10) | Accounts included in license. It should be noted that `accounts` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 7 | storage | BINARY(21) | Storage space included in license. It should be noted that `storage` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 8 | connections | BINARY(21) | Client connections included in license. It should be noted that `connections` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 9 | databases | BINARY(11) | Databases included in license. It should be noted that `databases` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 10 | speed | BINARY(9) | Write speed specified in license (data points per second) | -| 11 | querytime | BINARY(9) | Total query time specified in license | -| 12 | timeseries | BINARY(21) | Number of metrics included in license | -| 13 | expired | BINARY(5) | Whether the license has expired | -| 14 | expire_time | BINARY(19) | When the trial period expires | +| # | **Column** | **Data Type** | **Description** | +| --- | :---------: | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | version | BINARY(9) | Whether the deployment is a licensed or trial version | +| 2 | cpu_cores | BINARY(9) | CPU cores included in license | +| 3 | dnodes | BINARY(10) | Dnodes included in license. It should be noted that `dnodes` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 4 | streams | BINARY(10) | Streams included in license. It should be noted that `streams` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 5 | users | BINARY(10) | Users included in license. It should be noted that `users` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 6 | accounts | BINARY(10) | Accounts included in license. It should be noted that `accounts` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 7 | storage | BINARY(21) | Storage space included in license. It should be noted that `storage` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 8 | connections | BINARY(21) | Client connections included in license. It should be noted that `connections` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 9 | databases | BINARY(11) | Databases included in license. It should be noted that `databases` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 10 | speed | BINARY(9) | Write speed specified in license (data points per second) | +| 11 | querytime | BINARY(9) | Total query time specified in license | +| 12 | timeseries | BINARY(21) | Number of metrics included in license | +| 13 | expired | BINARY(5) | Whether the license has expired | +| 14 | expire_time | BINARY(19) | When the trial period expires | ## INS_VGROUPS Provides information about vgroups. -| # | **Column** | **Data Type** | **Description** | -| --- | :-------: | ------------ | ------------------------------------------------------ | -| 1 | vgroup_id | INT | Vgroup ID | -| 2 | db_name | BINARY(32) | Database name | -| 3 | tables | INT | Tables in vgroup. It should be noted that `tables` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 4 | status | BINARY(10) | Vgroup status | -| 5 | v1_dnode | INT | Dnode ID of first vgroup member | -| 6 | v1_status | BINARY(10) | Status of first vgroup member | -| 7 | v2_dnode | INT | Dnode ID of second vgroup member | -| 8 | v2_status | BINARY(10) | Status of second vgroup member | -| 9 | v3_dnode | INT | Dnode ID of third vgroup member | -| 10 | v3_status | BINARY(10) | Status of third vgroup member | -| 11 | nfiles | INT | Number of data and metadata files in the vgroup | -| 12 | file_size | INT | Size of the data and metadata files in the vgroup | -| 13 | tsma | TINYINT | Whether time-range-wise SMA is enabled. 1 means enabled; 0 means disabled. | +| # | **Column** | **Data Type** | **Description** | +| --- | :--------: | ------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| 1 | vgroup_id | INT | Vgroup ID | +| 2 | db_name | BINARY(32) | Database name | +| 3 | tables | INT | Tables in vgroup. It should be noted that `tables` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 4 | status | BINARY(10) | Vgroup status | +| 5 | v1_dnode | INT | Dnode ID of first vgroup member | +| 6 | v1_status | BINARY(10) | Status of first vgroup member | +| 7 | v2_dnode | INT | Dnode ID of second vgroup member | +| 8 | v2_status | BINARY(10) | Status of second vgroup member | +| 9 | v3_dnode | INT | Dnode ID of third vgroup member | +| 10 | v3_status | BINARY(10) | Status of third vgroup member | +| 11 | nfiles | INT | Number of data and metadata files in the vgroup | +| 12 | file_size | INT | Size of the data and metadata files in the vgroup | +| 13 | tsma | TINYINT | Whether time-range-wise SMA is enabled. 1 means enabled; 0 means disabled. | ## INS_CONFIGS Provides system configuration information. -| # | **Column** | **Data Type** | **Description** | -| --- | :------: | ------------ | ------------ | -| 1 | name | BINARY(32) | Parameter | -| 2 | value | BINARY(64) | Value. It should be noted that `value` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| # | **Column** | **Data Type** | **Description** | +| --- | :--------: | ------------- | ----------------------------------------------------------------------------------------------------------------------- | +| 1 | name | BINARY(32) | Parameter | +| 2 | value | BINARY(64) | Value. It should be noted that `value` is a TDengine keyword and needs to be escaped with ` when used as a column name. | ## INS_DNODE_VARIABLES Provides dnode configuration information. -| # | **Column** | **Data Type** | **Description** | -| --- | :------: | ------------ | ------------ | -| 1 | dnode_id | INT | Dnode ID | -| 2 | name | BINARY(32) | Parameter | -| 3 | value | BINARY(64) | Value. It should be noted that `value` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| # | **Column** | **Data Type** | **Description** | +| --- | :--------: | ------------- | ----------------------------------------------------------------------------------------------------------------------- | +| 1 | dnode_id | INT | Dnode ID | +| 2 | name | BINARY(32) | Parameter | +| 3 | value | BINARY(64) | Value. It should be noted that `value` is a TDengine keyword and needs to be escaped with ` when used as a column name. | ## INS_TOPICS -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------ | ------------------------------ | -| 1 | topic_name | BINARY(192) | Topic name | -| 2 | db_name | BINARY(64) | Database for the topic | -| 3 | create_time | TIMESTAMP | Creation time | -| 4 | sql | BINARY(1024) | SQL statement used to create the topic | +| # | **Column** | **Data Type** | **Description** | +| --- | :---------: | ------------- | -------------------------------------- | +| 1 | topic_name | BINARY(192) | Topic name | +| 2 | db_name | BINARY(64) | Database for the topic | +| 3 | create_time | TIMESTAMP | Creation time | +| 4 | sql | BINARY(1024) | SQL statement used to create the topic | ## INS_SUBSCRIPTIONS -| # | **Column** | **Data Type** | **Description** | -| --- | :------------: | ------------ | ------------------------ | -| 1 | topic_name | BINARY(204) | Subscribed topic | -| 2 | consumer_group | BINARY(193) | Subscribed consumer group | -| 3 | vgroup_id | INT | Vgroup ID for the consumer | -| 4 | consumer_id | BIGINT | Consumer ID | +| # | **Column** | **Data Type** | **Description** | +| --- | :------------: | ------------- | --------------------------- | +| 1 | topic_name | BINARY(204) | Subscribed topic | +| 2 | consumer_group | BINARY(193) | Subscribed consumer group | +| 3 | vgroup_id | INT | Vgroup ID for the consumer | +| 4 | consumer_id | BIGINT | Consumer ID | +| 5 | offset | BINARY(64) | Consumption progress | +| 6 | rows | BIGINT | Number of consumption items | ## INS_STREAMS -| # | **Column** | **Data Type** | **Description** | -| --- | :----------: | ------------ | --------------------------------------- | -| 1 | stream_name | BINARY(64) | Stream name | -| 2 | create_time | TIMESTAMP | Creation time | -| 3 | sql | BINARY(1024) | SQL statement used to create the stream | -| 4 | status | BINARY(20) | Current status | -| 5 | source_db | BINARY(64) | Source database | -| 6 | target_db | BINARY(64) | Target database | -| 7 | target_table | BINARY(192) | Target table | -| 8 | watermark | BIGINT | Watermark (see stream processing documentation). It should be noted that `watermark` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 9 | trigger | INT | Method of triggering the result push (see stream processing documentation). It should be noted that `trigger` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| # | **Column** | **Data Type** | **Description** | +| --- | :----------: | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | stream_name | BINARY(64) | Stream name | +| 2 | create_time | TIMESTAMP | Creation time | +| 3 | sql | BINARY(1024) | SQL statement used to create the stream | +| 4 | status | BINARY(20) | Current status | +| 5 | source_db | BINARY(64) | Source database | +| 6 | target_db | BINARY(64) | Target database | +| 7 | target_table | BINARY(192) | Target table | +| 8 | watermark | BIGINT | Watermark (see stream processing documentation). It should be noted that `watermark` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 9 | trigger | INT | Method of triggering the result push (see stream processing documentation). It should be noted that `trigger` is a TDengine keyword and needs to be escaped with ` when used as a column name. | + +## INS_USER_PRIVILEGES + +| # | **Column** | **Data Type** | **Description** |** | +| --- | :----------: | ------------ | -------------------------------------------| +| 1 | user_name | VARCHAR(24) | Username | +| 2 | privilege | VARCHAR(10) | Privilege description | +| 3 | db_name | VARCHAR(65) | Database name | +| 4 | table_name | VARCHAR(193) | Table name | +| 5 | condition | VARCHAR(49152) | The privilege filter for child tables | diff --git a/docs/en/12-taos-sql/24-show.md b/docs/en/12-taos-sql/24-show.md index eb70a7664b857a7971f24ea3ec7bc7f707f9fe38..b663fbd43521d03931deb7e3f9bfc65ac99d85ff 100644 --- a/docs/en/12-taos-sql/24-show.md +++ b/docs/en/12-taos-sql/24-show.md @@ -36,7 +36,7 @@ Shows information about connections to the system. SHOW CONSUMERS; ``` -Shows information about all active consumers in the system. +Shows information about all consumers in the system. ## SHOW CREATE DATABASE @@ -101,6 +101,7 @@ Note: TDengine Enterprise Edition only. ```sql SHOW INDEXES FROM tbl_name [FROM db_name]; +SHOW INDEXES FROM [db_name.]tbl_name; ``` Shows indices that have been created. @@ -326,6 +327,7 @@ Note that only the information about the data blocks in the data file will be di ```sql SHOW TAGS FROM child_table_name [FROM db_name]; +SHOW TAGS FROM [db_name.]child_table_name; ``` Shows all tag information in a subtable. diff --git a/docs/en/12-taos-sql/25-grant.md b/docs/en/12-taos-sql/25-grant.md index 8b4c4393523c90595c7dd5461fc2abc73fa44c93..c214e11876c77f4c617e1103b14ee46e6d211ca9 100644 --- a/docs/en/12-taos-sql/25-grant.md +++ b/docs/en/12-taos-sql/25-grant.md @@ -16,7 +16,7 @@ This statement creates a user account. The maximum length of user_name is 23 bytes. -The maximum length of password is 128 bytes. The password can include leters, digits, and special characters excluding single quotation marks, double quotation marks, backticks, backslashes, and spaces. The password cannot be empty. +The maximum length of password is 31 bytes. The password can include leters, digits, and special characters excluding single quotation marks, double quotation marks, backticks, backslashes, and spaces. The password cannot be empty. `SYSINFO` indicates whether the user is allowed to view system information. `1` means allowed, `0` means not allowed. System information includes server configuration, dnode, vnode, storage. The default value is `1`. diff --git a/docs/en/12-taos-sql/26-udf.md b/docs/en/12-taos-sql/26-udf.md index c4d6d4fca48fd3b4e9d3febc2013d14dd4d2cae5..b533b98b3d0d71dadce3979ce7f8950e6794cc79 100644 --- a/docs/en/12-taos-sql/26-udf.md +++ b/docs/en/12-taos-sql/26-udf.md @@ -17,7 +17,7 @@ CREATE [OR REPLACE] FUNCTION function_name AS library_path OUTPUTTYPE output_typ ``` - OR REPLACE: if the UDF exists, the UDF properties are modified - function_name: The scalar function name to be used in the SQL statement - - LANGUAGE 'C|Python': the programming language of UDF. Now C or Python is supported. If this clause is omitted, C is assumed as the programming language. + - LANGUAGE 'C|Python': the programming language of UDF. Now C or Python (v3.7+) is supported. If this clause is omitted, C is assumed as the programming language. - library_path: For C programming language, The absolute path of the DLL file including the name of the shared object file (.so). For Python programming language, the absolute path of the Python UDF script. The path must be quoted with single or double quotes. - output_type: The data type of the results of the UDF. diff --git a/docs/en/12-taos-sql/27-index.md b/docs/en/12-taos-sql/27-indexing.md similarity index 62% rename from docs/en/12-taos-sql/27-index.md rename to docs/en/12-taos-sql/27-indexing.md index 7586e4af76983e785b2c6b3f03870a8bcd6df4a3..a89c8929c1b632e5a2deb8d493459d58f57b0cc1 100644 --- a/docs/en/12-taos-sql/27-index.md +++ b/docs/en/12-taos-sql/27-indexing.md @@ -4,12 +4,12 @@ sidebar_label: Indexing description: This document describes the SQL statements related to indexing in TDengine. --- -TDengine supports SMA and FULLTEXT indexing. +TDengine supports SMA and tag indexing. ## Create an Index ```sql -CREATE FULLTEXT INDEX index_name ON tb_name (col_name [, col_name] ...) +CREATE INDEX index_name ON tb_name (col_name [, col_name] ...) CREATE SMA INDEX index_name ON tb_name index_option @@ -28,9 +28,23 @@ Performs pre-aggregation on the specified column over the time window defined by - WATERMARK: Enter a value between 0ms and 900000ms. The most precise unit supported is milliseconds. The default value is 5 seconds. This option can be used only on supertables. - MAX_DELAY: Enter a value between 1ms and 900000ms. The most precise unit supported is milliseconds. The default value is the value of interval provided that it does not exceed 900000ms. This option can be used only on supertables. Note: Retain the default value if possible. Configuring a small MAX_DELAY may cause results to be frequently pushed, affecting storage and query performance. -### FULLTEXT Indexing - -Creates a text index for the specified column. FULLTEXT indexing improves performance for queries with text filtering. The index_option syntax is not supported for FULLTEXT indexing. FULLTEXT indexing is supported for JSON tag columns only. Multiple columns cannot be indexed together. However, separate indices can be created for each column. +```sql +DROP DATABASE IF EXISTS d0; +CREATE DATABASE d0; +USE d0; +CREATE TABLE IF NOT EXISTS st1 (ts timestamp, c1 int, c2 float, c3 double) TAGS (t1 int unsigned); +CREATE TABLE ct1 USING st1 TAGS(1000); +CREATE TABLE ct2 USING st1 TAGS(2000); +INSERT INTO ct1 VALUES(now+0s, 10, 2.0, 3.0); +INSERT INTO ct1 VALUES(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3); +CREATE SMA INDEX sma_index_name1 ON st1 FUNCTION(max(c1),max(c2),min(c1)) INTERVAL(5m,10s) SLIDING(5m) WATERMARK 5s MAX_DELAY 1m; +-- query from SMA Index +ALTER LOCAL 'querySmaOptimize' '1'; +SELECT max(c2),min(c1) FROM st1 INTERVAL(5m,10s) SLIDING(5m); +SELECT _wstart,_wend,_wduration,max(c2),min(c1) FROM st1 INTERVAL(5m,10s) SLIDING(5m); +-- query from raw data +ALTER LOCAL 'querySmaOptimize' '0'; +``` ## Delete an Index @@ -41,8 +55,8 @@ DROP INDEX index_name; ## View Indices ````sql -```sql SHOW INDEXES FROM tbl_name [FROM db_name]; +SHOW INDEXES FROM [db_name.]tbl_name ; ```` Shows indices that have been created for the specified database or table. diff --git a/docs/en/12-taos-sql/29-changes.md b/docs/en/12-taos-sql/29-changes.md index 086aee59fe0583e905c04320c9fdd3a4a93351c7..bbb52db4d9100c6643b6ec84460f273e9be3f2e6 100644 --- a/docs/en/12-taos-sql/29-changes.md +++ b/docs/en/12-taos-sql/29-changes.md @@ -18,6 +18,7 @@ description: This document describes how TDengine SQL has changed in version 3.0 | 8 | Mixed operations | Enhanced | Mixing scalar and vector operations in queries has been enhanced and is supported in all SELECT clauses. | 9 | Tag operations | Added | Tag columns can be used in queries and clauses like data columns. | 10 | Timeline clauses and time functions in supertables | Enhanced | When PARTITION BY is not used, data in supertables is merged into a single timeline. +| 11 | GEOMETRY | Added | Geometry ## SQL Syntax @@ -33,7 +34,7 @@ The following data types can be used in the schema for standard tables. | 6 | ALTER USER | Modified | Deprecated
  • PRIVILEGE: Specified user permissions. Replaced by GRANT and REVOKE.
    Added
  • ENABLE: Enables or disables a user.
  • SYSINFO: Specifies whether a user can query system information.
| 7 | COMPACT VNODES | Not supported | Compacted the data on a vnode. Not supported. | 8 | CREATE ACCOUNT | Deprecated| This Enterprise Edition-only statement has been removed. It returns the error "This statement is no longer supported." -| 9 | CREATE DATABASE | Modified | Deprecated
  • BLOCKS: Specified the number of blocks for each vnode. BUFFER is now used to specify the size of the write cache pool for each vnode.
  • CACHE: Specified the size of the memory blocks used by each vnode. BUFFER is now used to specify the size of the write cache pool for each vnode.
  • CACHELAST: Specified how to cache the newest row of data. CACHEMODEL now replaces CACHELAST.
  • DAYS: The length of time to store in a single file. Replaced by DURATION.
  • FSYNC: Specified the fsync interval when WAL was set to 2. Replaced by WAL_FSYNC_PERIOD.
  • QUORUM: Specified the number of confirmations required. STRICT is now used to specify strong or weak consistency.
  • UPDATE: Specified whether update operations were supported. All databases now support updating data in certain columns.
  • WAL: Specified the WAL level. Replaced by WAL_LEVEL.
    Added
  • BUFFER: Specifies the size of the write cache pool for each vnode.
  • CACHEMODEL: Specifies whether to cache the latest subtable data.
  • CACHESIZE: Specifies the size of the cache for the newest subtable data.
  • DURATION: Replaces DAYS. Now supports units.
  • PAGES: Specifies the number of pages in the metadata storage engine cache on each vnode.
  • PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode.
  • RETENTIONS: Specifies the aggregation interval and retention period
  • STRICT: Specifies whether strong data consistency is enabled.
  • SINGLE_STABLE: Specifies whether a database can contain multiple supertables.
  • VGROUPS: Specifies the initial number of vgroups when a database is created.
  • WAL_FSYNC_PERIOD: Replaces the FSYNC parameter.
  • WAL_LEVEL: Replaces the WAL parameter.
  • WAL_RETENTION_PERIOD: specifies the time after which WAL files are deleted. This parameter is used for data subscription.
  • WAL_RETENTION_SIZE: specifies the size at which WAL files are deleted. This parameter is used for data subscription.
  • WAL_ROLL_PERIOD: Specifies the WAL rotation period.
  • WAL_SEGMENT_SIZE: specifies the maximum size of a WAL file.
    Modified
  • KEEP: Now supports units.
+| 9 | CREATE DATABASE | Modified | Deprecated
  • BLOCKS: Specified the number of blocks for each vnode. BUFFER is now used to specify the size of the write cache pool for each vnode.
  • CACHE: Specified the size of the memory blocks used by each vnode. BUFFER is now used to specify the size of the write cache pool for each vnode.
  • CACHELAST: Specified how to cache the newest row of data. CACHEMODEL now replaces CACHELAST.
  • DAYS: The length of time to store in a single file. Replaced by DURATION.
  • FSYNC: Specified the fsync interval when WAL was set to 2. Replaced by WAL_FSYNC_PERIOD.
  • QUORUM: Specified the number of confirmations required. STRICT is now used to specify strong or weak consistency.
  • UPDATE: Specified whether update operations were supported. All databases now support updating data in certain columns.
  • WAL: Specified the WAL level. Replaced by WAL_LEVEL.
    Added
  • BUFFER: Specifies the size of the write cache pool for each vnode.
  • CACHEMODEL: Specifies whether to cache the latest subtable data.
  • CACHESIZE: Specifies the size of the cache for the newest subtable data.
  • DURATION: Replaces DAYS. Now supports units.
  • PAGES: Specifies the number of pages in the metadata storage engine cache on each vnode.
  • PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode.
  • RETENTIONS: Specifies the aggregation interval and retention period
  • STRICT: Specifies whether strong data consistency is enabled.
  • SINGLE_STABLE: Specifies whether a database can contain multiple supertables.
  • VGROUPS: Specifies the initial number of vgroups when a database is created.
  • WAL_FSYNC_PERIOD: Replaces the FSYNC parameter.
  • WAL_LEVEL: Replaces the WAL parameter.
  • WAL_RETENTION_PERIOD: specifies the time after which WAL files are deleted. This parameter is used for data subscription.
  • WAL_RETENTION_SIZE: specifies the size at which WAL files are deleted. This parameter is used for data subscription.
    Modified
  • KEEP: Now supports units.
| 10 | CREATE DNODE | Modified | Now supports specifying hostname and port separately
  • CREATE DNODE dnode_host_name PORT port_val
| 11 | CREATE INDEX | Added | Creates an SMA index. | 12 | CREATE MNODE | Added | Creates an mnode. diff --git a/docs/en/13-operation/10-monitor.md b/docs/en/13-operation/10-monitor.md index 80c33dd475d5c8ea1aa1553571c69a7d2406e5fd..c1c6ac3c4ca1dd047a0501415095abb721017f73 100644 --- a/docs/en/13-operation/10-monitor.md +++ b/docs/en/13-operation/10-monitor.md @@ -36,7 +36,7 @@ You can use below command to setup Grafana alert notification. An existing Grafana Notification Channel can be specified with parameter `-E`, the notifier uid of the channel can be obtained by `curl -u admin:admin localhost:3000/api/alert-notifications |jq` ```bash - sudo ./TDinsight.sh -a http://localhost:6041 -u root -p taosdata -E + ./TDinsight.sh -a http://localhost:6041 -u root -p taosdata -E ``` Launch `TDinsight.sh` with the command above and restart Grafana, then open Dashboard `http://localhost:3000/d/tdinsight`. @@ -214,19 +214,6 @@ The data of tdinsight dashboard is stored in `log` database (default. You can ch |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### logs table - -`logs` table contains login information records. - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|level|VARCHAR||log level| -|content|NCHAR||log content| -|dnode\_id|INT|TAG|dnode id| -|dnode\_ep|NCHAR|TAG|dnode endpoint| -|cluster\_id|NCHAR|TAG|cluster id| - ### log\_summary table `log_summary` table contains log summary information records. @@ -274,7 +261,7 @@ The data of tdinsight dashboard is stored in `log` database (default. You can ch |field|type|is\_tag|comment| |:----|:---|:-----|:------| |\_ts|TIMESTAMP||timestamp| -|guage|DOUBLE||metric value| +|gauge|DOUBLE||metric value| |client\_ip|NCHAR|TAG|client ip| |endpoint|NCHAR|TAG|taosadpater endpoint| |request\_method|NCHAR|TAG|request method| @@ -288,7 +275,7 @@ The data of tdinsight dashboard is stored in `log` database (default. You can ch |field|type|is\_tag|comment| |:----|:---|:-----|:------| |\_ts|TIMESTAMP||timestamp| -|guage|DOUBLE||metric value| +|gauge|DOUBLE||metric value| |client\_ip|NCHAR|TAG|client ip| |endpoint|NCHAR|TAG|taosadpater endpoint| |request\_method|NCHAR|TAG|request method| @@ -302,7 +289,7 @@ The data of tdinsight dashboard is stored in `log` database (default. You can ch |field|type|is\_tag|comment| |:----|:---|:-----|:------| |\_ts|TIMESTAMP||timestamp| -|guage|DOUBLE||metric value| +|gauge|DOUBLE||metric value| |endpoint|NCHAR|TAG|taosadpater endpoint| ### taosadapter\_restful\_http\_request\_summary\_milliseconds table @@ -330,7 +317,7 @@ The data of tdinsight dashboard is stored in `log` database (default. You can ch |field|type|is\_tag|comment| |:----|:---|:-----|:------| |\_ts|TIMESTAMP||timestamp| -|guage|DOUBLE||metric value| +|gauge|DOUBLE||metric value| |endpoint|NCHAR|TAG|taosadpater endpoint| ### taosadapter\_system\_cpu\_percent table @@ -340,6 +327,6 @@ The data of tdinsight dashboard is stored in `log` database (default. You can ch |field|type|is\_tag|comment| |:----|:---|:-----|:------| |\_ts|TIMESTAMP||timestamp| -|guage|DOUBLE||mertic value| +|gauge|DOUBLE||mertic value| |endpoint|NCHAR|TAG|taosadpater endpoint| diff --git a/docs/en/14-reference/02-rest-api/02-rest-api.mdx b/docs/en/14-reference/02-rest-api/02-rest-api.mdx index ea010f42db8b988c597bcd4c9278c0b5d50a3ca7..4da987213cb8c5f21fd7c8defe1f0952b8f65e41 100644 --- a/docs/en/14-reference/02-rest-api/02-rest-api.mdx +++ b/docs/en/14-reference/02-rest-api/02-rest-api.mdx @@ -79,6 +79,12 @@ Parameter Description: - tz: Optional parameter that specifies the timezone of the returned time, following the IANA Time Zone rules, e.g. `America/New_York`. - req_id: Optional parameter that specifies the request id for tracing. +:::note + +URL Encoding. Make sure that parameters are properly encoded. For example, when specifying a timezone you must properly encode special characters. ?tz=Etc/GMT+10 will not work because the <+> plus symbol is recognized as a space in the url. It's best practice to encode all special characters in a parameter. Instead use ?tz=Etc%2FGMT%2B10 for the parameter. + +::: + For example, `http://h1.taos.com:6041/rest/sql/test` is a URL to `h1.taos.com:6041` and sets the default database name to `test`. TDengine supports both Basic authentication and custom authentication mechanisms, and subsequent versions will provide a standard secure digital signature mechanism for authentication. diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index 260b38b24ffd00682b393fc2e3d14142c0e6f3af..69bbd287edbbeae608cee4ec1f1070f10d96532b 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -32,25 +32,24 @@ TDengine's JDBC driver implementation is as consistent as possible with the rela Native connections are supported on the same platforms as the TDengine client driver. REST connection supports all platforms that can run Java. -## Version support - -Please refer to [version support list](/reference/connector#version-support) - ## Recent update logs -| taos-jdbcdriver version | major changes | -| :---------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------: | -| 3.2.1 | JDBC REST connection supports schemaless/prepareStatement over WebSocket | -| 3.2.0 | This version has been deprecated | -| 3.1.0 | JDBC REST connection supports subscription over WebSocket | -| 3.0.1 - 3.0.4 | fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use other version in the JDK 8 environment | -| 3.0.0 | Support for TDengine 3.0 | -| 2.0.42 | fix wasNull interface return value in WebSocket connection | -| 2.0.41 | fix decode method of username and password in REST connection | -| 2.0.39 - 2.0.40 | Add REST connection/request timeout parameters | -| 2.0.38 | JDBC REST connections add bulk pull function | -| 2.0.37 | Support json tags | -| 2.0.36 | Support schemaless writing | +| taos-jdbcdriver version | major changes | TDengine version | +| :---------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------: | :--------------: | +| 3.2.4 | Subscription add the enable.auto.commit parameter and the unsubscribe() method in the WebSocket connection | - | +| 3.2.3 | Fixed resultSet data parsing failure in some cases | - | +| 3.2.2 | Subscription add seek function | 3.0.5.0 or later | +| 3.2.1 | JDBC REST connection supports schemaless/prepareStatement over WebSocket | 3.0.3.0 or later | +| 3.2.0 | This version has been deprecated | - | +| 3.1.0 | JDBC REST connection supports subscription over WebSocket | - | +| 3.0.1 - 3.0.4 | fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use other version in the JDK 8 environment | - | +| 3.0.0 | Support for TDengine 3.0 | 3.0.0.0 or later | +| 2.0.42 | Fix wasNull interface return value in WebSocket connection | - | +| 2.0.41 | Fix decode method of username and password in REST connection | - | +| 2.0.39 - 2.0.40 | Add REST connection/request timeout parameters | - | +| 2.0.38 | JDBC REST connections add bulk pull function | - | +| 2.0.37 | Support json tags | - | +| 2.0.36 | Support schemaless writing | - | **Note**: adding `batchfetch` to the REST connection and setting it to true will enable the WebSocket connection. @@ -102,6 +101,8 @@ For specific error codes, please refer to. | 0x2319 | user is required | The user name information is missing when creating the connection | | 0x231a | password is required | Password information is missing when creating a connection | | 0x231c | httpEntity is null, sql: | Execution exception occurred during the REST connection | +| 0x231d | can't create connection with server within | Increase the connection time by adding the httpConnectTimeout parameter, or check the connection to the taos adapter. | +| 0x231e | failed to complete the task within the specified time | Increase the execution time by adding the messageWaitTimeout parameter, or check the connection to the taos adapter. | | 0x2350 | unknown error | Unknown exception, please return to the developer on github. | | 0x2352 | Unsupported encoding | An unsupported character encoding set is specified under the native Connection. | | 0x2353 | internal error of database, please see taoslog for more details | An error occurs when the prepare statement is executed on the native connection. Check the taos log to locate the fault. | @@ -117,8 +118,8 @@ For specific error codes, please refer to. | 0x2376 | failed to set consumer topic, topic name is empty | During data subscription creation, the subscription topic name is empty. Check that the specified topic name is correct. | | 0x2377 | consumer reference has been destroyed | The subscription data transfer channel has been closed. Please check the connection to TDengine. | | 0x2378 | consumer create error | Failed to create a data subscription. Check the taos log according to the error message to locate the fault. | -| - | can't create connection with server within | Increase the connection time by adding the httpConnectTimeout parameter, or check the connection to the taos adapter. | -| - | failed to complete the task within the specified time | Increase the execution time by adding the messageWaitTimeout parameter, or check the connection to the taos adapter. | +| 0x2379 | seek offset must not be a negative number | The seek interface parameter cannot be negative. Use the correct parameter | +| 0x237a | vGroup not found in result set | subscription is not bound to the VGroup due to the rebalance mechanism | - [TDengine Java Connector](https://github.com/taosdata/taos-connector-jdbc/blob/main/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java) @@ -169,7 +170,7 @@ Add following dependency in the `pom.xml` file of your Maven project: com.taosdata.jdbc taos-jdbcdriver - 3.2.1 + 3.2.2 ``` @@ -285,10 +286,11 @@ The configuration parameters in the URL are as follows: - batchfetch: true: pulls result sets in batches when executing queries; false: pulls result sets row by row. The default value is: false. batchfetch uses HTTP for data transfer. JDBC REST supports batch pulls. taos-jdbcdriver and TDengine transfer data via WebSocket connection. Compared with HTTP, WebSocket enables JDBC REST connection to support large data volume querying and improve query performance. - charset: specify the charset to parse the string, this parameter is valid only when set batchfetch to true. - batchErrorIgnore: true: when executing executeBatch of Statement, if one SQL execution fails in the middle, continue to execute the following SQL. false: no longer execute any statement after the failed SQL. The default value is: false. -- httpConnectTimeout: REST connection timeout in milliseconds, the default value is 5000 ms. -- httpSocketTimeout: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when batchfetch is false. -- messageWaitTimeout: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when batchfetch is true. +- httpConnectTimeout: REST connection timeout in milliseconds, the default value is 60000 ms. +- httpSocketTimeout: socket timeout in milliseconds, the default value is 60000 ms. It only takes effect when batchfetch is false. +- messageWaitTimeout: message transmission timeout in milliseconds, the default value is 60000 ms. It only takes effect when batchfetch is true. - useSSL: connecting Securely Using SSL. true: using SSL connection, false: not using SSL connection. +- httpPoolSize: size of REST concurrent requests. The default value is 20. **Note**: Some configuration items (e.g., locale, timezone) do not work in the REST connection. @@ -352,10 +354,11 @@ The configuration parameters in properties are as follows. - TSDBDriver.PROPERTY_KEY_CHARSET: In the character set used by the client, the default value is the system character set. - TSDBDriver.PROPERTY_KEY_LOCALE: this only takes effect when using JDBC native connection. Client language environment, the default value is system current locale. - TSDBDriver.PROPERTY_KEY_TIME_ZONE: only takes effect when using JDBC native connection. In the time zone used by the client, the default value is the system's current time zone. -- TSDBDriver.HTTP_CONNECT_TIMEOUT: REST connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection. -- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection and batchfetch is false. -- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when using JDBC REST connection and batchfetch is true. +- TSDBDriver.HTTP_CONNECT_TIMEOUT: REST connection timeout in milliseconds, the default value is 60000 ms. It only takes effect when using JDBC REST connection. +- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 60000 ms. It only takes effect when using JDBC REST connection and batchfetch is false. +- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 60000 ms. It only takes effect when using JDBC REST connection and batchfetch is true. - TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL connection, false: not using SSL connection. It only takes effect when using JDBC REST connection. +- TSDBDriver.HTTP_POOL_SIZE: size of REST concurrent requests. The default value is 20. For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](/reference/config/#Client-Only). ### Priority of configuration parameters @@ -420,6 +423,19 @@ while(resultSet.next()){ > The query is consistent with operating a relational database. When using subscripts to get the contents of the returned fields, you have to start from 1. However, we recommend using the field names to get the values of the fields in the result set. +### execute SQL with reqId + +This reqId can be used to request link tracing. + +```java +AbstractStatement aStmt = (AbstractStatement) connection.createStatement(); +aStmt.execute("create database if not exists db", 1L); +aStmt.executeUpdate("use db", 2L); +try (ResultSet rs = aStmt.executeQuery("select * from tb", 3L)) { + Timestamp ts = rs.getTimestamp(1); +} +``` + ### Writing data via parameter binding TDengine has significantly improved the bind APIs to support data writing (INSERT) scenarios. Writing data in this way avoids the resource consumption of SQL syntax parsing, resulting in significant write performance improvements in many cases. @@ -913,14 +929,15 @@ public class SchemalessWsTest { public static void main(String[] args) throws SQLException { final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata&batchfetch=true"; - Connection connection = DriverManager.getConnection(url); - init(connection); - - SchemalessWriter writer = new SchemalessWriter(connection, "test_ws_schemaless"); - writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS); - writer.write(telnetDemo, SchemalessProtocolType.TELNET, SchemalessTimestampType.MILLI_SECONDS); - writer.write(jsonDemo, SchemalessProtocolType.JSON, SchemalessTimestampType.SECONDS); - System.exit(0); + try(Connection connection = DriverManager.getConnection(url)){ + init(connection); + + try(SchemalessWriter writer = new SchemalessWriter(connection, "test_ws_schemaless")){ + writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS); + writer.write(telnetDemo, SchemalessProtocolType.TELNET, SchemalessTimestampType.MILLI_SECONDS); + writer.write(jsonDemo, SchemalessProtocolType.JSON, SchemalessTimestampType.SECONDS); + } + } } private static void init(Connection connection) throws SQLException { @@ -936,6 +953,14 @@ public class SchemalessWsTest { +### Schemaless with reqId + +This reqId can be used to request link tracing. + +```java +writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS, 1L); +``` + ### Data Subscription The TDengine Java Connector supports subscription functionality with the following application API. @@ -959,6 +984,7 @@ The preceding example uses the SQL statement `select ts, speed from speed_table` ```java Properties config = new Properties(); +config.setProperty("bootstrap.servers", "localhost:6030"); config.setProperty("enable.auto.commit", "true"); config.setProperty("group.id", "group1"); config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ResultDeserializer"); @@ -966,12 +992,14 @@ config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.Res TaosConsumer consumer = new TaosConsumer<>(config); ``` +- bootstrap.servers: `ip:port` where the TDengine server is located, or `ip:port` where the taosAdapter is located if WebSocket connection is used. - enable.auto.commit: Specifies whether to commit automatically. - group.id: consumer: Specifies the group that the consumer is in. - value.deserializer: To deserialize the results, you can inherit `com.taosdata.jdbc.tmq.ReferenceDeserializer` and specify the result set bean. You can also inherit `com.taosdata.jdbc.tmq.Deserializer` and perform custom deserialization based on the SQL result set. - td.connect.type: Specifies the type connect with TDengine, `jni` or `WebSocket`. default is `jni` - httpConnectTimeout: WebSocket connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using WebSocket type. - messageWaitTimeout: socket timeout in milliseconds, the default value is 10000 ms. It only takes effect when using WebSocket type. +- httpPoolSize: Maximum number of concurrent requests on the a connection。It only takes effect when using WebSocket type. - For more information, see [Consumer Parameters](../../../develop/tmq). #### Subscribe to consume data @@ -988,6 +1016,42 @@ while(true) { `poll` obtains one message each time it is run. +#### Assignment subscription Offset + +```java +// get offset +long position(TopicPartition partition) throws SQLException; +Map position(String topic) throws SQLException; +Map beginningOffsets(String topic) throws SQLException; +Map endOffsets(String topic) throws SQLException; + +// Overrides the fetch offsets that the consumer will use on the next poll(timeout). +void seek(TopicPartition partition, long offset) throws SQLException; +``` + +Example usage is as follows. + +```java +String topic = "offset_seek_test"; +Map offset = null; +try (TaosConsumer consumer = new TaosConsumer<>(properties)) { + consumer.subscribe(Collections.singletonList(topic)); + for (int i = 0; i < 10; i++) { + if (i == 3) { + // Saving consumption position + offset = consumer.position(topic); + } + if (i == 5) { + // reset consumption to the previously saved position + for (Map.Entry entry : offset.entrySet()) { + consumer.seek(entry.getKey(), entry.getValue()); + } + } + ConsumerRecords records = consumer.poll(Duration.ofMillis(500)); + } +} +``` + #### Close subscriptions ```java @@ -1015,10 +1079,20 @@ public abstract class ConsumerLoop { public ConsumerLoop() throws SQLException { Properties config = new Properties(); + config.setProperty("td.connect.type", "jni"); + config.setProperty("bootstrap.servers", "localhost:6030"); + config.setProperty("td.connect.user", "root"); + config.setProperty("td.connect.pass", "taosdata"); + config.setProperty("auto.offset.reset", "earliest"); config.setProperty("msg.with.table.name", "true"); config.setProperty("enable.auto.commit", "true"); + config.setProperty("auto.commit.interval.ms", "1000"); config.setProperty("group.id", "group1"); + config.setProperty("client.id", "1"); config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer"); + config.setProperty("value.deserializer.encoding", "UTF-8"); + config.setProperty("experimental.snapshot.enable", "true"); + this.consumer = new TaosConsumer<>(config); this.topics = Collections.singletonList("topic_speed"); @@ -1090,12 +1164,19 @@ public abstract class ConsumerLoop { public ConsumerLoop() throws SQLException { Properties config = new Properties(); - config.setProperty("bootstrap.servers", "localhost:6041"); config.setProperty("td.connect.type", "ws"); + config.setProperty("bootstrap.servers", "localhost:6041"); + config.setProperty("td.connect.user", "root"); + config.setProperty("td.connect.pass", "taosdata"); + config.setProperty("auto.offset.reset", "earliest"); config.setProperty("msg.with.table.name", "true"); config.setProperty("enable.auto.commit", "true"); + config.setProperty("auto.commit.interval.ms", "1000"); config.setProperty("group.id", "group2"); + config.setProperty("client.id", "1"); config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer"); + config.setProperty("value.deserializer.encoding", "UTF-8"); + config.setProperty("experimental.snapshot.enable", "true"); this.consumer = new TaosConsumer<>(config); this.topics = Collections.singletonList("topic_speed"); @@ -1236,6 +1317,7 @@ The source code of the sample application is under `TDengine/examples/JDBC`: - connectionPools: using taos-jdbcdriver in connection pools such as HikariCP, Druid, dbcp, c3p0, etc. - SpringJdbcTemplate: using taos-jdbcdriver in Spring JdbcTemplate. - mybatisplus-demo: using taos-jdbcdriver in Springboot + Mybatis. +- consumer-demo: consumer TDengine data example, the consumption rate can be controlled by parameters. [JDBC example](https://github.com/taosdata/TDengine/tree/3.0/examples/JDBC) @@ -1276,3 +1358,7 @@ For additional troubleshooting, see [FAQ](../../../train-faq/faq). ## API Reference [taos-jdbcdriver doc](https://docs.taosdata.com/api/taos-jdbcdriver) + +``` + +``` diff --git a/docs/en/14-reference/03-connector/05-go.mdx b/docs/en/14-reference/03-connector/05-go.mdx index 0088f230069b15f12e455b5b6d2d6fa261fc5a14..b3d4857d75e22f18c0dbcb4f2798c268f6fbcd3a 100644 --- a/docs/en/14-reference/03-connector/05-go.mdx +++ b/docs/en/14-reference/03-connector/05-go.mdx @@ -29,65 +29,80 @@ REST connections are supported on all platforms that can run Go. ## Version support -Please refer to [version support list](/reference/connector#version-support) +Please refer to [version support list](https://github.com/taosdata/driver-go#remind) -## Supported features +## Handling exceptions -### Native connections - -A "native connection" is established by the connector directly to the TDengine instance via the TDengine client driver (taosc). The supported functional features are: - -* Normal queries -* Continuous queries -* Subscriptions -* Schemaless interface -* Parameter binding interface - -### REST connection - -A "REST connection" is a connection between the application and the TDengine instance via the REST API provided by the taosAdapter component. The following features are supported: +If it is a TDengine error, you can get the error code and error information in the following ways. +```go +// import "github.com/taosdata/driver-go/v3/errors" + if err != nil { + tError, is := err.(*errors.TaosError) + if is { + fmt.Println("errorCode:", int(tError.Code)) + fmt.Println("errorMessage:", tError.ErrStr) + } else { + fmt.Println(err.Error()) + } + } +``` -* Normal queries -* Continuous queries +## TDengine DataType vs. Go DataType + +| TDengine DataType | Go Type | +|-------------------|-----------| +| TIMESTAMP | time.Time | +| TINYINT | int8 | +| SMALLINT | int16 | +| INT | int32 | +| BIGINT | int64 | +| TINYINT UNSIGNED | uint8 | +| SMALLINT UNSIGNED | uint16 | +| INT UNSIGNED | uint32 | +| BIGINT UNSIGNED | uint64 | +| FLOAT | float32 | +| DOUBLE | float64 | +| BOOL | bool | +| BINARY | string | +| NCHAR | string | +| JSON | []byte | + +**Note**: Only TAG supports JSON types ## Installation Steps ### Pre-installation preparation * Install Go development environment (Go 1.14 and above, GCC 4.8.5 and above) -- If you use the native connector, please install the TDengine client driver. Please refer to [Install Client Driver](/reference/connector/#install-client-driver) for specific steps +* If you use the native connector, please install the TDengine client driver. Please refer to [Install Client Driver](/reference/connector/#install-client-driver) for specific steps Configure the environment variables and check the command. * ```go env``` * ```gcc -v``` -### Use go get to install - -`go get -u github.com/taosdata/driver-go/v3@latest` - -### Manage with go mod +### Install the connectors 1. Initialize the project with the `go mod` command. - ```text - go mod init taos-demo - ``` + ```text + go mod init taos-demo + ``` 2. Introduce taosSql - ```go - import ( - "database/sql" - _ "github.com/taosdata/driver-go/v3/taosSql" - ) - ``` + ```go + import ( + "database/sql" + _ "github.com/taosdata/driver-go/v3/taosSql" + ) + ``` 3. Update the dependency packages with `go mod tidy`. - ```text - go mod tidy - ``` + ```text + go mod tidy + ``` 4. Run the program with `go run taos-demo` or compile the binary with the `go build` command. @@ -98,8 +113,6 @@ Configure the environment variables and check the command. ## Establishing a connection -### Data source name (DSN) - Data source names have a standard format, e.g. [PEAR DB](http://pear.php.net/manual/en/package.database.db.intro-dsn.php), but no type prefix (square brackets indicate optionally): ``` text @@ -111,9 +124,7 @@ DSN in full form. ```text username:password@protocol(address)/dbname?param=value ``` -### Connecting via connector - - + _taosSql_ implements Go's `database/sql/driver` interface via cgo. You can use the [`database/sql`](https://golang.org/pkg/database/sql/) interface by simply introducing the driver. @@ -209,322 +220,902 @@ func main() { -## Usage examples - -### Write data +### Specify the URL and Properties to get the connection -#### SQL Write +The Go connector does not support this feature - +### Priority of configuration parameters -#### InfluxDB line protocol write +The Go connector does not support this feature - +## Usage examples -#### OpenTSDB Telnet line protocol write +### Create database and tables - +```go +var taosDSN = "root:taosdata@tcp(localhost:6030)/" +taos, err := sql.Open("taosSql", taosDSN) +if err != nil { + log.Fatalln("failed to connect TDengine, err:", err) +} +defer taos.Close() +_, err := taos.Exec("CREATE DATABASE power") +if err != nil { + log.Fatalln("failed to create database, err:", err) +} +_, err = taos.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") +if err != nil { + log.Fatalln("failed to create stable, err:", err) +} +``` -#### OpenTSDB JSON line protocol write +### Insert data - + -### Query data +### Querying data -### More sample programs +### execute SQL with reqId -* [sample program](https://github.com/taosdata/driver-go/tree/3.0/examples) +This reqId can be used to request link tracing. +```go +db, err := sql.Open("taosSql", "root:taosdata@tcp(localhost:6030)/") +if err != nil { + panic(err) +} +defer db.Close() +ctx := context.WithValue(context.Background(), common.ReqIDKey, common.GetReqID()) +_, err = db.ExecContext(ctx, "create database if not exists example_taos_sql") +if err != nil { + panic(err) +} +``` -## Usage limitations - -Since the REST interface is stateless, the `use db` syntax will not work. You need to put the db name into the SQL command, e.g. `create table if not exists tb1 (ts timestamp, a int)` to `create table if not exists test.tb1 (ts timestamp, a int)` otherwise it will report the error `[0x217] Database not specified or available`. - -You can also put the db name in the DSN by changing `root:taosdata@http(localhost:6041)/` to `root:taosdata@http(localhost:6041)/test`. Executing the `create database` statement when the specified db does not exist will not report an error while executing other queries or writing against that db will report an error. +### Writing data via parameter binding -The complete example is as follows. + + ```go package main import ( - "database/sql" - "fmt" "time" - _ "github.com/taosdata/driver-go/v3/taosRestful" + "github.com/taosdata/driver-go/v3/af" + "github.com/taosdata/driver-go/v3/common" + "github.com/taosdata/driver-go/v3/common/param" ) func main() { - var taosDSN = "root:taosdata@http(localhost:6041)/test" - taos, err := sql.Open("taosRestful", taosDSN) + db, err := af.Open("", "root", "taosdata", "", 0) if err != nil { - fmt.Println("failed to connect TDengine, err:", err) - return + panic(err) } - defer taos.Close() - taos.Exec("create database if not exists test") - taos.Exec("create table if not exists tb1 (ts timestamp, a int)") - _, err = taos.Exec("insert into tb1 values(now, 0)(now+1s,1)(now+2s,2)(now+3s,3)") + defer db.Close() + _, err = db.Exec("create database if not exists example_stmt") if err != nil { - fmt.Println("failed to insert, err:", err) - return + panic(err) } - rows, err := taos.Query("select * from tb1") + _, err = db.Exec("create table if not exists example_stmt.tb1(ts timestamp," + + "c1 bool," + + "c2 tinyint," + + "c3 smallint," + + "c4 int," + + "c5 bigint," + + "c6 tinyint unsigned," + + "c7 smallint unsigned," + + "c8 int unsigned," + + "c9 bigint unsigned," + + "c10 float," + + "c11 double," + + "c12 binary(20)," + + "c13 nchar(20)" + + ")") if err != nil { - fmt.Println("failed to select from table, err:", err) - return + panic(err) } - - defer rows.Close() - for rows.Next() { - var r struct { - ts time.Time - a int - } - err := rows.Scan(&r.ts, &r.a) - if err != nil { - fmt.Println("scan error:\n", err) - return - } - fmt.Println(r.ts, r.a) + stmt := db.InsertStmt() + err = stmt.Prepare("insert into example_stmt.tb1 values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + if err != nil { + panic(err) + } + now := time.Now() + params := make([]*param.Param, 14) + params[0] = param.NewParam(2). + AddTimestamp(now, common.PrecisionMilliSecond). + AddTimestamp(now.Add(time.Second), common.PrecisionMilliSecond) + params[1] = param.NewParam(2).AddBool(true).AddNull() + params[2] = param.NewParam(2).AddTinyint(2).AddNull() + params[3] = param.NewParam(2).AddSmallint(3).AddNull() + params[4] = param.NewParam(2).AddInt(4).AddNull() + params[5] = param.NewParam(2).AddBigint(5).AddNull() + params[6] = param.NewParam(2).AddUTinyint(6).AddNull() + params[7] = param.NewParam(2).AddUSmallint(7).AddNull() + params[8] = param.NewParam(2).AddUInt(8).AddNull() + params[9] = param.NewParam(2).AddUBigint(9).AddNull() + params[10] = param.NewParam(2).AddFloat(10).AddNull() + params[11] = param.NewParam(2).AddDouble(11).AddNull() + params[12] = param.NewParam(2).AddBinary([]byte("binary")).AddNull() + params[13] = param.NewParam(2).AddNchar("nchar").AddNull() + + paramTypes := param.NewColumnType(14). + AddTimestamp(). + AddBool(). + AddTinyint(). + AddSmallint(). + AddInt(). + AddBigint(). + AddUTinyint(). + AddUSmallint(). + AddUInt(). + AddUBigint(). + AddFloat(). + AddDouble(). + AddBinary(6). + AddNchar(5) + err = stmt.BindParam(params, paramTypes) + if err != nil { + panic(err) } + err = stmt.AddBatch() + if err != nil { + panic(err) + } + err = stmt.Execute() + if err != nil { + panic(err) + } + err = stmt.Close() + if err != nil { + panic(err) + } + // select * from example_stmt.tb1 } ``` -## Frequently Asked Questions - -1. bind interface in database/sql crashes - - REST does not support parameter binding related interface. It is recommended to use `db.Exec` and `db.Query`. - -2. error `[0x217] Database not specified or available` after executing other statements with `use db` statement - - The execution of SQL command in the REST interface is not contextual, so using `use db` statement will not work, see the usage restrictions section above. - -3. use `taosSql` without error but use `taosRestful` with error `[0x217] Database not specified or available` - - Because the REST interface is stateless, using the `use db` statement will not take effect. See the usage restrictions section above. - -4. `readBufferSize` parameter has no significant effect after being increased - - Increasing `readBufferSize` will reduce the number of `syscall` calls when fetching results. If the query result is smaller, modifying this parameter will not improve performance significantly. If you increase the parameter value too much, the bottleneck will be parsing JSON data. If you need to optimize the query speed, you must adjust the value based on the actual situation to achieve the best query performance. - -5. `disableCompression` parameter is set to `false` when the query efficiency is reduced - - When set `disableCompression` parameter to `false`, the query result will be compressed by `gzip` and then transmitted, so you have to decompress the data by `gzip` after getting it. - -6. `go get` command can't get the package, or timeout to get the package - - Set Go proxy `go env -w GOPROXY=https://goproxy.cn,direct`. - -## Common APIs - -### database/sql API - -* `sql.Open(DRIVER_NAME string, dataSourceName string) *DB` - - Use This API to open a DB, returning an object of type \*DB. - -:::info -This API is created successfully without checking permissions, but only when you execute a Query or Exec, and check if user/password/host/port is legal. -::: - -* `func (db *DB) Exec(query string, args ...interface{}) (Result, error)` - - `sql.Open` built-in method to execute non-query related SQL. - -* `func (db *DB) Query(query string, args ...interface{}) (*Rows, error)` - - `sql.Open` Built-in method to execute query statements. - -### Advanced functions (af) API - -The `af` package encapsulates TDengine advanced functions such as connection management, subscriptions, schemaless, parameter binding, etc. - -#### Connection management - -* `af.Open(host, user, pass, db string, port int) (*Connector, error)` - - This API creates a connection to taosd via cgo. - -* `func (conn *Connector) Close() error` - - Closes the connection. - -#### Subscribe - -* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` - -Creates consumer group. - -* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` -Note: `rebalanceCb` is reserved for compatibility purpose - -Subscribes a topic. - -* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` -Note: `rebalanceCb` is reserved for compatibility purpose - -Subscribes to topics. - -* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` - -Polling information. - -* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` -Note: `tmq.TopicPartition` is reserved for compatibility purpose - -Commit information. - -* `func (c *Consumer) Unsubscribe() error` - -Unsubscribe. + + -* `func (c *Consumer) Close() error` +```go +package main -Close consumer. +import ( + "database/sql" + "fmt" + "time" -#### schemaless + "github.com/taosdata/driver-go/v3/common" + "github.com/taosdata/driver-go/v3/common/param" + _ "github.com/taosdata/driver-go/v3/taosRestful" + "github.com/taosdata/driver-go/v3/ws/stmt" +) -* `func (conn *Connector) InfluxDBInsertLines(lines []string, precision string) error` +func main() { + db, err := sql.Open("taosRestful", "root:taosdata@http(localhost:6041)/") + if err != nil { + panic(err) + } + defer db.Close() + prepareEnv(db) + + config := stmt.NewConfig("ws://127.0.0.1:6041/rest/stmt", 0) + config.SetConnectUser("root") + config.SetConnectPass("taosdata") + config.SetConnectDB("example_ws_stmt") + config.SetMessageTimeout(common.DefaultMessageTimeout) + config.SetWriteWait(common.DefaultWriteWait) + config.SetErrorHandler(func(connector *stmt.Connector, err error) { + panic(err) + }) + config.SetCloseHandler(func() { + fmt.Println("stmt connector closed") + }) + + connector, err := stmt.NewConnector(config) + if err != nil { + panic(err) + } + now := time.Now() + { + stmt, err := connector.Init() + if err != nil { + panic(err) + } + err = stmt.Prepare("insert into ? using all_json tags(?) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + if err != nil { + panic(err) + } + err = stmt.SetTableName("tb1") + if err != nil { + panic(err) + } + err = stmt.SetTags(param.NewParam(1).AddJson([]byte(`{"tb":1}`)), param.NewColumnType(1).AddJson(0)) + if err != nil { + panic(err) + } + params := []*param.Param{ + param.NewParam(3).AddTimestamp(now, 0).AddTimestamp(now.Add(time.Second), 0).AddTimestamp(now.Add(time.Second*2), 0), + param.NewParam(3).AddBool(true).AddNull().AddBool(true), + param.NewParam(3).AddTinyint(1).AddNull().AddTinyint(1), + param.NewParam(3).AddSmallint(1).AddNull().AddSmallint(1), + param.NewParam(3).AddInt(1).AddNull().AddInt(1), + param.NewParam(3).AddBigint(1).AddNull().AddBigint(1), + param.NewParam(3).AddUTinyint(1).AddNull().AddUTinyint(1), + param.NewParam(3).AddUSmallint(1).AddNull().AddUSmallint(1), + param.NewParam(3).AddUInt(1).AddNull().AddUInt(1), + param.NewParam(3).AddUBigint(1).AddNull().AddUBigint(1), + param.NewParam(3).AddFloat(1).AddNull().AddFloat(1), + param.NewParam(3).AddDouble(1).AddNull().AddDouble(1), + param.NewParam(3).AddBinary([]byte("test_binary")).AddNull().AddBinary([]byte("test_binary")), + param.NewParam(3).AddNchar("test_nchar").AddNull().AddNchar("test_nchar"), + } + paramTypes := param.NewColumnType(14). + AddTimestamp(). + AddBool(). + AddTinyint(). + AddSmallint(). + AddInt(). + AddBigint(). + AddUTinyint(). + AddUSmallint(). + AddUInt(). + AddUBigint(). + AddFloat(). + AddDouble(). + AddBinary(0). + AddNchar(0) + err = stmt.BindParam(params, paramTypes) + if err != nil { + panic(err) + } + err = stmt.AddBatch() + if err != nil { + panic(err) + } + err = stmt.Exec() + if err != nil { + panic(err) + } + affected := stmt.GetAffectedRows() + fmt.Println("all_json affected rows:", affected) + err = stmt.Close() + if err != nil { + panic(err) + } + } + { + stmt, err := connector.Init() + if err != nil { + panic(err) + } + err = stmt.Prepare("insert into ? using all_all tags(?,?,?,?,?,?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + err = stmt.SetTableName("tb1") + if err != nil { + panic(err) + } - Write to InfluxDB line protocol. + err = stmt.SetTableName("tb2") + if err != nil { + panic(err) + } + err = stmt.SetTags( + param.NewParam(14). + AddTimestamp(now, 0). + AddBool(true). + AddTinyint(2). + AddSmallint(2). + AddInt(2). + AddBigint(2). + AddUTinyint(2). + AddUSmallint(2). + AddUInt(2). + AddUBigint(2). + AddFloat(2). + AddDouble(2). + AddBinary([]byte("tb2")). + AddNchar("tb2"), + param.NewColumnType(14). + AddTimestamp(). + AddBool(). + AddTinyint(). + AddSmallint(). + AddInt(). + AddBigint(). + AddUTinyint(). + AddUSmallint(). + AddUInt(). + AddUBigint(). + AddFloat(). + AddDouble(). + AddBinary(0). + AddNchar(0), + ) + if err != nil { + panic(err) + } + params := []*param.Param{ + param.NewParam(3).AddTimestamp(now, 0).AddTimestamp(now.Add(time.Second), 0).AddTimestamp(now.Add(time.Second*2), 0), + param.NewParam(3).AddBool(true).AddNull().AddBool(true), + param.NewParam(3).AddTinyint(1).AddNull().AddTinyint(1), + param.NewParam(3).AddSmallint(1).AddNull().AddSmallint(1), + param.NewParam(3).AddInt(1).AddNull().AddInt(1), + param.NewParam(3).AddBigint(1).AddNull().AddBigint(1), + param.NewParam(3).AddUTinyint(1).AddNull().AddUTinyint(1), + param.NewParam(3).AddUSmallint(1).AddNull().AddUSmallint(1), + param.NewParam(3).AddUInt(1).AddNull().AddUInt(1), + param.NewParam(3).AddUBigint(1).AddNull().AddUBigint(1), + param.NewParam(3).AddFloat(1).AddNull().AddFloat(1), + param.NewParam(3).AddDouble(1).AddNull().AddDouble(1), + param.NewParam(3).AddBinary([]byte("test_binary")).AddNull().AddBinary([]byte("test_binary")), + param.NewParam(3).AddNchar("test_nchar").AddNull().AddNchar("test_nchar"), + } + paramTypes := param.NewColumnType(14). + AddTimestamp(). + AddBool(). + AddTinyint(). + AddSmallint(). + AddInt(). + AddBigint(). + AddUTinyint(). + AddUSmallint(). + AddUInt(). + AddUBigint(). + AddFloat(). + AddDouble(). + AddBinary(0). + AddNchar(0) + err = stmt.BindParam(params, paramTypes) + if err != nil { + panic(err) + } + err = stmt.AddBatch() + if err != nil { + panic(err) + } + err = stmt.Exec() + if err != nil { + panic(err) + } + affected := stmt.GetAffectedRows() + fmt.Println("all_all affected rows:", affected) + err = stmt.Close() + if err != nil { + panic(err) + } -* `func (conn *Connector) OpenTSDBInsertTelnetLines(lines []string) error` + } +} - Write OpenTDSB telnet protocol data. +func prepareEnv(db *sql.DB) { + steps := []string{ + "create database example_ws_stmt", + "create table example_ws_stmt.all_json(ts timestamp," + + "c1 bool," + + "c2 tinyint," + + "c3 smallint," + + "c4 int," + + "c5 bigint," + + "c6 tinyint unsigned," + + "c7 smallint unsigned," + + "c8 int unsigned," + + "c9 bigint unsigned," + + "c10 float," + + "c11 double," + + "c12 binary(20)," + + "c13 nchar(20)" + + ")" + + "tags(t json)", + "create table example_ws_stmt.all_all(" + + "ts timestamp," + + "c1 bool," + + "c2 tinyint," + + "c3 smallint," + + "c4 int," + + "c5 bigint," + + "c6 tinyint unsigned," + + "c7 smallint unsigned," + + "c8 int unsigned," + + "c9 bigint unsigned," + + "c10 float," + + "c11 double," + + "c12 binary(20)," + + "c13 nchar(20)" + + ")" + + "tags(" + + "tts timestamp," + + "tc1 bool," + + "tc2 tinyint," + + "tc3 smallint," + + "tc4 int," + + "tc5 bigint," + + "tc6 tinyint unsigned," + + "tc7 smallint unsigned," + + "tc8 int unsigned," + + "tc9 bigint unsigned," + + "tc10 float," + + "tc11 double," + + "tc12 binary(20)," + + "tc13 nchar(20))", + } + for _, step := range steps { + _, err := db.Exec(step) + if err != nil { + panic(err) + } + } +} -* `func (conn *Connector) OpenTSDBInsertJsonPayload(payload string) error` +``` - Writes OpenTSDB JSON protocol data. + + -#### parameter binding -* `func (conn *Connector) StmtExecute(sql string, params *param.Param) (res driver.Result, err error)` +### Schemaless Writing - Parameter bound single row insert. + + -* `func (conn *Connector) InsertStmt() *insertstmt.InsertStmt` +```go +import ( + "fmt" - Initialize the parameters. + "github.com/taosdata/driver-go/v3/af" +) -* `func (stmt *InsertStmt) Prepare(sql string) error` +func main() { + conn, err := af.Open("localhost", "root", "taosdata", "", 6030) + if err != nil { + fmt.Println("fail to connect, err:", err) + } + defer conn.Close() + _, err = conn.Exec("create database if not exists example") + if err != nil { + panic(err) + } + _, err = conn.Exec("use example") + if err != nil { + panic(err) + } + influxdbData := "st,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000" + err = conn.InfluxDBInsertLines([]string{influxdbData}, "ns") + if err != nil { + panic(err) + } + telnetData := "stb0_0 1626006833 4 host=host0 interface=eth0" + err = conn.OpenTSDBInsertTelnetLines([]string{telnetData}) + if err != nil { + panic(err) + } + jsonData := "{\"metric\": \"meter_current\",\"timestamp\": 1626846400,\"value\": 10.3, \"tags\": {\"groupid\": 2, \"location\": \"California.SanFrancisco\", \"id\": \"d1001\"}}" + err = conn.OpenTSDBInsertJsonPayload(jsonData) + if err != nil { + panic(err) + } +} +``` - Parameter binding preprocessing SQL statement. + + -* `func (stmt *InsertStmt) SetTableName(name string) error` +```go +import ( + "database/sql" + "log" + "time" - Bind the table name parameter. + "github.com/taosdata/driver-go/v3/common" + _ "github.com/taosdata/driver-go/v3/taosWS" + "github.com/taosdata/driver-go/v3/ws/schemaless" +) -* `func (stmt *InsertStmt) SetSubTableName(name string) error` +func main() { + db, err := sql.Open("taosWS", "root:taosdata@ws(localhost:6041)/") + if err != nil { + log.Fatal(err) + } + defer db.Close() + _, err = db.Exec("create database if not exists schemaless_ws") + if err != nil { + log.Fatal(err) + } + s, err := schemaless.NewSchemaless(schemaless.NewConfig("ws://localhost:6041/rest/schemaless", 1, + schemaless.SetDb("schemaless_ws"), + schemaless.SetReadTimeout(10*time.Second), + schemaless.SetWriteTimeout(10*time.Second), + schemaless.SetUser("root"), + schemaless.SetPassword("taosdata"), + schemaless.SetErrorHandler(func(err error) { + log.Fatal(err) + }), + )) + if err != nil { + panic(err) + } + influxdbData := "st,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000" + telnetData := "stb0_0 1626006833 4 host=host0 interface=eth0" + jsonData := "{\"metric\": \"meter_current\",\"timestamp\": 1626846400,\"value\": 10.3, \"tags\": {\"groupid\": 2, \"location\": \"California.SanFrancisco\", \"id\": \"d1001\"}}" - Parameter binding to set the sub table name. + err = s.Insert(influxdbData, schemaless.InfluxDBLineProtocol, "ns", 0, common.GetReqID()) + if err != nil { + panic(err) + } + err = s.Insert(telnetData, schemaless.OpenTSDBTelnetLineProtocol, "ms", 0, common.GetReqID()) + if err != nil { + panic(err) + } + err = s.Insert(jsonData, schemaless.OpenTSDBJsonFormatProtocol, "ms", 0, common.GetReqID()) + if err != nil { + panic(err) + } +} +``` -* `func (stmt *InsertStmt) BindParam(params []*param.Param, bindType *param.ColumnType) error` + + - Parameter bind multiple rows of data. -* `func (stmt *InsertStmt) AddBatch() error` +### Schemaless with reqId - Add to a parameter-bound batch. +```go +func (s *Schemaless) Insert(lines string, protocol int, precision string, ttl int, reqID int64) error +``` -* `func (stmt *InsertStmt) Execute() error` +You can get the unique id by `common.GetReqID()`. - Execute a parameter binding. +### Data Subscription -* `func (stmt *InsertStmt) GetAffectedRows() int` +The TDengine Go Connector supports subscription functionality with the following application API. - Gets the number of affected rows inserted by the parameter binding. +#### Create a Topic -* `func (stmt *InsertStmt) Close() error` +```go + db, err := af.Open("", "root", "taosdata", "", 0) + if err != nil { + panic(err) + } + defer db.Close() + _, err = db.Exec("create database if not exists example_tmq WAL_RETENTION_PERIOD 86400") + if err != nil { + panic(err) + } + _, err = db.Exec("create topic if not exists example_tmq_topic as DATABASE example_tmq") + if err != nil { + panic(err) + } +``` - Closes the parameter binding. +#### Create a Consumer -### Subscribe via WebSocket +```go + consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ + "group.id": "test", + "auto.offset.reset": "earliest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_client", + "enable.auto.commit": "false", + "msg.with.table.name": "true", + }) + if err != nil { + panic(err) + } +``` -* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` +#### Subscribe to consume data -Creates consumer group. +```go + err = consumer.Subscribe("example_tmq_topic", nil) + if err != nil { + panic(err) + } + for i := 0; i < 5; i++ { + ev := consumer.Poll(500) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Printf("get message:%v\n", e) + case tmqcommon.Error: + fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() + } + } +``` -* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` -Note: `rebalanceCb` is reserved for compatibility purpose +#### Assignment subscription Offset -Subscribes a topic. +```go + partitions, err := consumer.Assignment() + if err != nil { + panic(err) + } + for i := 0; i < len(partitions); i++ { + fmt.Println(partitions[i]) + err = consumer.Seek(tmqcommon.TopicPartition{ + Topic: partitions[i].Topic, + Partition: partitions[i].Partition, + Offset: 0, + }, 0) + if err != nil { + panic(err) + } + } +``` -* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` -Note: `rebalanceCb` is reserved for compatibility purpose +#### Close subscriptions -Subscribes to topics. +```go + err = consumer.Close() + if err != nil { + panic(err) + } +``` -* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` +#### Full Sample Code -Polling information. + + -* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` -Note: `tmq.TopicPartition` is reserved for compatibility purpose +```go +package main -Commit information. +import ( + "fmt" + "os" -* `func (c *Consumer) Unsubscribe() error` + "github.com/taosdata/driver-go/v3/af" + "github.com/taosdata/driver-go/v3/af/tmq" + tmqcommon "github.com/taosdata/driver-go/v3/common/tmq" +) -Unsubscribe. +func main() { + db, err := af.Open("", "root", "taosdata", "", 0) + if err != nil { + panic(err) + } + defer db.Close() + _, err = db.Exec("create database if not exists example_tmq WAL_RETENTION_PERIOD 86400") + if err != nil { + panic(err) + } + _, err = db.Exec("create topic if not exists example_tmq_topic as DATABASE example_tmq") + if err != nil { + panic(err) + } + if err != nil { + panic(err) + } + consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ + "group.id": "test", + "auto.offset.reset": "earliest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_client", + "enable.auto.commit": "false", + "msg.with.table.name": "true", + }) + if err != nil { + panic(err) + } + err = consumer.Subscribe("example_tmq_topic", nil) + if err != nil { + panic(err) + } + _, err = db.Exec("create table example_tmq.t1 (ts timestamp,v int)") + if err != nil { + panic(err) + } + _, err = db.Exec("insert into example_tmq.t1 values(now,1)") + if err != nil { + panic(err) + } + for i := 0; i < 5; i++ { + ev := consumer.Poll(500) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Printf("get message:%v\n", e) + case tmqcommon.Error: + fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() + } + } + partitions, err := consumer.Assignment() + if err != nil { + panic(err) + } + for i := 0; i < len(partitions); i++ { + fmt.Println(partitions[i]) + err = consumer.Seek(tmqcommon.TopicPartition{ + Topic: partitions[i].Topic, + Partition: partitions[i].Partition, + Offset: 0, + }, 0) + if err != nil { + panic(err) + } + } -* `func (c *Consumer) Close() error` + partitions, err = consumer.Assignment() + if err != nil { + panic(err) + } + for i := 0; i < len(partitions); i++ { + fmt.Println(partitions[i]) + } -Close consumer. + err = consumer.Close() + if err != nil { + panic(err) + } +} +``` -For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) + + -### parameter binding via WebSocket +```go +package main -* `func NewConnector(config *Config) (*Connector, error)` +import ( + "database/sql" + "fmt" - Create a connection. + "github.com/taosdata/driver-go/v3/common" + tmqcommon "github.com/taosdata/driver-go/v3/common/tmq" + _ "github.com/taosdata/driver-go/v3/taosRestful" + "github.com/taosdata/driver-go/v3/ws/tmq" +) -* `func (c *Connector) Init() (*Stmt, error)` +func main() { + db, err := sql.Open("taosRestful", "root:taosdata@http(localhost:6041)/") + if err != nil { + panic(err) + } + defer db.Close() + prepareEnv(db) + consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ + "ws.url": "ws://127.0.0.1:6041/rest/tmq", + "ws.message.channelLen": uint(0), + "ws.message.timeout": common.DefaultMessageTimeout, + "ws.message.writeWait": common.DefaultWriteWait, + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "group.id": "example", + "client.id": "example_consumer", + "auto.offset.reset": "earliest", + }) + if err != nil { + panic(err) + } + err = consumer.Subscribe("example_ws_tmq_topic", nil) + if err != nil { + panic(err) + } + go func() { + _, err := db.Exec("create table example_ws_tmq.t_all(ts timestamp," + + "c1 bool," + + "c2 tinyint," + + "c3 smallint," + + "c4 int," + + "c5 bigint," + + "c6 tinyint unsigned," + + "c7 smallint unsigned," + + "c8 int unsigned," + + "c9 bigint unsigned," + + "c10 float," + + "c11 double," + + "c12 binary(20)," + + "c13 nchar(20)" + + ")") + if err != nil { + panic(err) + } + _, err = db.Exec("insert into example_ws_tmq.t_all values(now,true,2,3,4,5,6,7,8,9,10.123,11.123,'binary','nchar')") + if err != nil { + panic(err) + } + }() + for i := 0; i < 5; i++ { + ev := consumer.Poll(500) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Printf("get message:%v\n", e) + case tmqcommon.Error: + fmt.Printf("%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() + } + } + partitions, err := consumer.Assignment() + if err != nil { + panic(err) + } + for i := 0; i < len(partitions); i++ { + fmt.Println(partitions[i]) + err = consumer.Seek(tmqcommon.TopicPartition{ + Topic: partitions[i].Topic, + Partition: partitions[i].Partition, + Offset: 0, + }, 0) + if err != nil { + panic(err) + } + } - Initialize the parameters. + partitions, err = consumer.Assignment() + if err != nil { + panic(err) + } + for i := 0; i < len(partitions); i++ { + fmt.Println(partitions[i]) + } -* `func (c *Connector) Close() error` + err = consumer.Close() + if err != nil { + panic(err) + } +} - Close the connection. +func prepareEnv(db *sql.DB) { + _, err := db.Exec("create database example_ws_tmq WAL_RETENTION_PERIOD 86400") + if err != nil { + panic(err) + } + _, err = db.Exec("create topic example_ws_tmq_topic as database example_ws_tmq") + if err != nil { + panic(err) + } +} +``` -* `func (s *Stmt) Prepare(sql string) error` + + - Parameter binding preprocessing SQL statement. +### More sample programs -* `func (s *Stmt) SetTableName(name string) error` +* [sample program](https://github.com/taosdata/driver-go/tree/3.0/examples) - Bind the table name parameter. -* `func (s *Stmt) SetTags(tags *param.Param, bindType *param.ColumnType) error` +## Frequently Asked Questions - Set tags. +1. bind interface in database/sql crashes -* `func (s *Stmt) BindParam(params []*param.Param, bindType *param.ColumnType) error` + REST does not support parameter binding related interface. It is recommended to use `db.Exec` and `db.Query`. - Parameter bind multiple rows of data. +2. error `[0x217] Database not specified or available` after executing other statements with `use db` statement -* `func (s *Stmt) AddBatch() error` + The execution of SQL command in the REST interface is not contextual, so using `use db` statement will not work, see the usage restrictions section above. - Add to a parameter-bound batch. +3. use `taosSql` without error but use `taosRestful` with error `[0x217] Database not specified or available` -* `func (s *Stmt) Exec() error` + Because the REST interface is stateless, using the `use db` statement will not take effect. See the usage restrictions section above. - Execute a parameter binding. +4. `readBufferSize` parameter has no significant effect after being increased -* `func (s *Stmt) GetAffectedRows() int` + Increasing `readBufferSize` will reduce the number of `syscall` calls when fetching results. If the query result is smaller, modifying this parameter will not improve performance significantly. If you increase the parameter value too much, the bottleneck will be parsing JSON data. If you need to optimize the query speed, you must adjust the value based on the actual situation to achieve the best query performance. - Gets the number of affected rows inserted by the parameter binding. +5. `disableCompression` parameter is set to `false` when the query efficiency is reduced -* `func (s *Stmt) Close() error` + When set `disableCompression` parameter to `false`, the query result will be compressed by `gzip` and then transmitted, so you have to decompress the data by `gzip` after getting it. - Closes the parameter binding. +6. `go get` command can't get the package, or timeout to get the package -For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/3.0/examples/stmtoverws/main.go) + Set Go proxy `go env -w GOPROXY=https://goproxy.cn,direct`. ## API Reference diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index 99c3d2c066d9e0e269ecc5cc03f6efa106b1906f..a98683d43c169c6a2f76dc154035c0af84464287 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -27,20 +27,62 @@ The source code for the Rust connectors is located on [GitHub](https://github.co Native connections are supported on the same platforms as the TDengine client driver. Websocket connections are supported on all platforms that can run Go. -## Version support +## Version history -Please refer to [version support list](/reference/connector#version-support) +| connector-rust version | TDengine version | major features | +| :----------------: | :--------------: | :--------------------------------------------------: | +| v0.9.2 | 3.0.7.0 or later | STMT: Get tag_fields and col_fields under ws. | +| v0.8.12 | 3.0.5.0 | TMQ: Get consuming progress and seek offset to consume. | +| v0.8.0 | 3.0.4.0 | Support schemaless insert. | +| v0.7.6 | 3.0.3.0 | Support req_id in query. | +| v0.6.0 | 3.0.0.0 | Base features. | The Rust Connector is still under rapid development and is not guaranteed to be backward compatible before 1.0. We recommend using TDengine version 3.0 or higher to avoid known issues. -## Installation +## Handling exceptions + +After the error is reported, the specific information of the error can be obtained: + +```rust +match conn.exec(sql) { + Ok(_) => { + Ok(()) + } + Err(e) => { + eprintln!("ERROR: {:?}", e); + Err(e) + } +} +``` + +## TDengine DataType vs. Rust DataType + +TDengine currently supports timestamp, number, character, Boolean type, and the corresponding type conversion with Rust is as follows: + +| TDengine DataType | Rust DataType | +| ----------------- | ----------------- | +| TIMESTAMP | Timestamp | +| INT | i32 | +| BIGINT | i64 | +| FLOAT | f32 | +| DOUBLE | f64 | +| SMALLINT | i16 | +| TINYINT | i8 | +| BOOL | bool | +| BINARY | Vec | +| NCHAR | String | +| JSON | serde_json::Value | + +Note: Only TAG supports JSON types + +## Installation Steps ### Pre-installation preparation * Install the Rust development toolchain * If using the native connection, please install the TDengine client driver. Please refer to [install client driver](/reference/connector#install-client-driver) -### Add taos dependency +### Install the connectors Depending on the connection method, add the [taos][taos] dependency in your Rust project as follows: @@ -141,7 +183,8 @@ let builder = TaosBuilder::from_dsn("taos://localhost:6030")?; let conn1 = builder.build(); // use websocket protocol. -let conn2 = TaosBuilder::from_dsn("taos+ws://localhost:6041")?; +let builder2 = TaosBuilder::from_dsn("taos+ws://localhost:6041")?; +let conn2 = builder2.build(); ``` After the connection is established, you can perform operations on your database. @@ -223,41 +266,191 @@ There are two ways to query data: Using built-in types or the [serde](https://se ## Usage examples -### Write data +### Create database and tables + +```rust +use taos::*; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let dsn = "taos://localhost:6030"; + let builder = TaosBuilder::from_dsn(dsn)?; + + let taos = builder.build()?; -#### SQL Write + let db = "query"; + + // create database + taos.exec_many([ + format!("DROP DATABASE IF EXISTS `{db}`"), + format!("CREATE DATABASE `{db}`"), + format!("USE `{db}`"), + ]) + .await?; + + // create table + taos.exec_many([ + // create super table + "CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) \ + TAGS (`groupid` INT, `location` BINARY(16))", + // create child table + "CREATE TABLE `d0` USING `meters` TAGS(0, 'Los Angles')", + ]).await?; +} +``` + +> The query is consistent with operating a relational database. When using subscripts to get the contents of the returned fields, you have to start from 1. However, we recommend using the field names to get the values of the fields in the result set. + +### Insert data -#### STMT Write +### Query data + + + +### execute SQL with req_id + +This req_id can be used to request link tracing. + +```rust +let rs = taos.query_with_req_id("select * from stable where tag1 is null", 1)?; +``` + +### Writing data via parameter binding + +TDengine has significantly improved the bind APIs to support data writing (INSERT) scenarios. Writing data in this way avoids the resource consumption of SQL syntax parsing, resulting in significant write performance improvements in many cases. + +Parameter binding details see [API Reference](#stmt-api) -#### Schemaless Write +### Schemaless Writing + +TDengine supports schemaless writing. It is compatible with InfluxDB's Line Protocol, OpenTSDB's telnet line protocol, and OpenTSDB's JSON format protocol. For more information, see [Schemaless Writing](../../schemaless). -### Query data +### Schemaless with req_id - +This req_id can be used to request link tracing. -## API Reference +```rust +let sml_data = SmlDataBuilder::default() + .protocol(SchemalessProtocol::Line) + .data(data) + .req_id(100u64) + .build()?; + +client.put(&sml_data)? +``` + +### Data Subscription + +TDengine starts subscriptions through [TMQ](../../../taos-sql/tmq/). + +#### Create a Topic + +```rust +taos.exec_many([ + // create topic for subscription + format!("CREATE TOPIC tmq_meters with META AS DATABASE {db}") +]) +.await?; +``` + +#### Create a Consumer + +You create a TMQ connector by using a DSN. + +```rust +let tmq = TmqBuilder::from_dsn("taos://localhost:6030/?group.id=test")?; +``` + +Create a consumer: + +```rust +let mut consumer = tmq.build()?; +``` + +#### Subscribe to consume data + +A single consumer can subscribe to one or more topics. + +```rust +consumer.subscribe(["tmq_meters"]).await?; +``` + +The TMQ is of [futures::Stream](https://docs.rs/futures/latest/futures/stream/index.html) type. You can use the corresponding API to consume each message in the queue and then use `.commit` to mark them as consumed. + +```rust +{ + let mut stream = consumer.stream(); + + while let Some((offset, message)) = stream.try_next().await? { + // get information from offset + + // the topic + let topic = offset.topic(); + // the vgroup id, like partition id in kafka. + let vgroup_id = offset.vgroup_id(); + println!("* in vgroup id {vgroup_id} of topic {topic}\n"); + + if let Some(data) = message.into_data() { + while let Some(block) = data.fetch_raw_block().await? { + // one block for one table, get table name if needed + let name = block.table_name(); + let records: Vec = block.deserialize().try_collect()?; + println!( + "** table: {}, got {} records: {:#?}\n", + name.unwrap(), + records.len(), + records + ); + } + } + consumer.commit(offset).await?; + } +} +``` + +Get assignments： + +Version requirements connector-rust >= v0.8.8, TDengine >= 3.0.5.0 -### Connector Constructor +```rust +let assignments = consumer.assignments().await.unwrap(); +``` + +#### Assignment subscription Offset + +Seek offset： -You create a connector constructor by using a DSN. +Version requirements connector-rust >= v0.8.8, TDengine >= 3.0.5.0 ```rust -let cfg = TaosBuilder::default().build()?; +consumer.offset_seek(topic, vgroup_id, offset).await; ``` -You use the builder object to create multiple connections. +#### Close subscriptions ```rust -let conn: Taos = cfg.build(); +consumer.unsubscribe().await; ``` -### Connection pooling +The following parameters can be configured for the TMQ DSN. Only `group.id` is mandatory. + +- `group.id`: Within a consumer group, load balancing is implemented by consuming messages on an at-least-once basis. +- `client.id`: Subscriber client ID. +- `auto.offset.reset`: Initial point of subscription. *earliest* subscribes from the beginning, and *latest* subscribes from the newest message. The default is earliest. Note: This parameter is set per consumer group. +- `enable.auto.commit`: Automatically commits. This can be enabled when data consistency is not essential. +- `auto.commit.interval.ms`: Interval for automatic commits. + +#### Full Sample Code + +For more information, see [GitHub sample file](https://github.com/taosdata/TDengine/blob/3.0/docs/examples/rust/nativeexample/examples/subscribe_demo.rs). + +### Use with connection pool In complex applications, we recommend enabling connection pools. [taos] implements connection pools based on [r2d2]. @@ -287,7 +480,17 @@ In the application code, use `pool.get()? ` to get a connection object [Taos]. let taos = pool.get()?; ``` -### Connectors +### More sample programs + +The source code of the sample application is under `TDengine/examples/rust` : + +[rust example](https://github.com/taosdata/TDengine/tree/3.0/examples/rust) + +## Frequently Asked Questions + +For additional troubleshooting, see [FAQ](../../../train-faq/faq). + +## API Reference The [Taos][struct.Taos] object provides an API to perform operations on multiple databases. @@ -373,9 +576,13 @@ Note that Rust asynchronous functions and an asynchronous runtime are required. - `.create_database(database: &str)`: Executes the `CREATE DATABASE` statement. - `.use_database(database: &str)`: Executes the `USE` statement. -In addition, this structure is also the entry point for [Parameter Binding](#Parameter Binding Interface) and [Line Protocol Interface](#Line Protocol Interface). Please refer to the specific API descriptions for usage. +In addition, this structure is also the entry point for Parameter Binding and Line Protocol Interface. Please refer to the specific API descriptions for usage. -### Bind Interface +

+ +Bind Interface + +

Similar to the C interface, Rust provides the bind interface's wrapping. First, the [Taos][struct.taos] object creates a parameter binding object [Stmt] for an SQL statement. @@ -386,7 +593,7 @@ stmt.prepare("INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)")?; The bind object provides a set of interfaces for implementing parameter binding. -#### `.set_tbname(name)` +`.set_tbname(name)` To bind table names. @@ -395,7 +602,7 @@ let mut stmt = taos.stmt("insert into ? values(? ,?)")?; stmt.set_tbname("d0")?; ``` -#### `.set_tags(&[tag])` +`.set_tags(&[tag])` Bind sub-table table names and tag values when the SQL statement uses a super table. @@ -405,7 +612,7 @@ stmt.set_tbname("d0")?; stmt.set_tags(&[Value::VarChar("taos".to_string())])?; ``` -#### `.bind(&[column])` +`.bind(&[column])` Bind value types. Use the [ColumnView] structure to create and bind the required types. @@ -429,7 +636,7 @@ let params = vec![ let rows = stmt.bind(¶ms)?.add_batch()?.execute()?; ``` -#### `.execute()` +`.execute()` Execute SQL. [Stmt] objects can be reused, re-binded, and executed after execution. Before execution, ensure that all data has been added to the queue with `.add_batch`. @@ -442,82 +649,12 @@ stmt.execute()?; //stmt.execute()?; ``` -For a working example, see [GitHub](https://github.com/taosdata/taos-connector-rust/blob/main/examples/bind.rs). - -### Subscriptions - -TDengine starts subscriptions through [TMQ](../../../taos-sql/tmq/). - -You create a TMQ connector by using a DSN. - -```rust -let tmq = TmqBuilder::from_dsn("taos://localhost:6030/?group.id=test")?; -``` - -Create a consumer: - -```rust -let mut consumer = tmq.build()?; -``` - -A single consumer can subscribe to one or more topics. +For a working example, see [GitHub](https://github.com/taosdata/taos-connector-rust/blob/main/taos/examples/bind.rs). -```rust -consumer.subscribe(["tmq_meters"]).await?; -``` - -The TMQ is of [futures::Stream](https://docs.rs/futures/latest/futures/stream/index.html) type. You can use the corresponding API to consume each message in the queue and then use `.commit` to mark them as consumed. - -```rust -{ - let mut stream = consumer.stream(); - - while let Some((offset, message)) = stream.try_next().await? { - // get information from offset - - // the topic - let topic = offset.topic(); - // the vgroup id, like partition id in kafka. - let vgroup_id = offset.vgroup_id(); - println!("* in vgroup id {vgroup_id} of topic {topic}\n"); - - if let Some(data) = message.into_data() { - while let Some(block) = data.fetch_raw_block().await? { - // one block for one table, get table name if needed - let name = block.table_name(); - let records: Vec = block.deserialize().try_collect()?; - println!( - "** table: {}, got {} records: {:#?}\n", - name.unwrap(), - records.len(), - records - ); - } - } - consumer.commit(offset).await?; - } -} -``` - -Unsubscribe: - -```rust -consumer.unsubscribe().await; -``` - -The following parameters can be configured for the TMQ DSN. Only `group.id` is mandatory. - -- `group.id`: Within a consumer group, load balancing is implemented by consuming messages on an at-least-once basis. -- `client.id`: Subscriber client ID. -- `auto.offset.reset`: Initial point of subscription. *earliest* subscribes from the beginning, and *latest* subscribes from the newest message. The default is earliest. Note: This parameter is set per consumer group. -- `enable.auto.commit`: Automatically commits. This can be enabled when data consistency is not essential. -- `auto.commit.interval.ms`: Interval for automatic commits. - -For more information, see [GitHub sample file](https://github.com/taosdata/taos-connector-rust/blob/main/examples/subscribe.rs). For information about other structure APIs, see the [Rust documentation](https://docs.rs/taos). -[taos]: https://github.com/taosdata/rust-connector-taos +[taos]: https://github.com/taosdata/taos-connector-rust [r2d2]: https://crates.io/crates/r2d2 [TaosBuilder]: https://docs.rs/taos/latest/taos/struct.TaosBuilder.html [TaosCfg]: https://docs.rs/taos/latest/taos/struct.TaosCfg.html diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index a83f2047d02f7161d85a2f7ecb1c2a983ecfc22e..5067c33e2d210090c2a68f3f9898193950aebd7c 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -20,18 +20,76 @@ The source code for the Python connector is hosted on [GitHub](https://github.co - The [supported platforms](/reference/connector/#supported-platforms) for the native connection are the same as the ones supported by the TDengine client. - REST connections are supported on all platforms that can run Python. +### Supported features + +- Native connections support all the core features of TDengine, including connection management, SQL execution, bind interface, subscriptions, and schemaless writing. +- REST connections support features such as connection management and SQL execution. (SQL execution allows you to: manage databases, tables, and supertables, write data, query data, create continuous queries, etc.). + ## Version selection We recommend using the latest version of `taospy`, regardless of the version of TDengine. -## Supported features +|Python Connector Version|major changes| +|:-------------------:|:----:| +|2.7.9|support for getting assignment and seek function on subscription| +|2.7.8|add `execute_many` method| -- Native connections support all the core features of TDengine, including connection management, SQL execution, bind interface, subscriptions, and schemaless writing. -- REST connections support features such as connection management and SQL execution. (SQL execution allows you to: manage databases, tables, and supertables, write data, query data, create continuous queries, etc.). +|Python Websocket Connector Version|major changes| +|:----------------------------:|:-----:| +|0.2.5|1. support for getting assignment and seek function on subscription
2. support schemaless
3. support STMT| +|0.2.4|support `unsubscribe` on subscription| -## Installation +## Handling Exceptions -### Preparation +There are 4 types of exception in python connector. + +- The exception of Python Connector itself. +- The exception of native library. +- The exception of websocket +- The exception of subscription. +- The exception of other TDengine function modules. + +|Error Type|Description|Suggested Actions| +|:--------:|:---------:|:---------------:| +|InterfaceError|the native library is too old that it cannot support the function|please check the TDengine client version| +|ConnectionError|connection error|please check TDengine's status and the connection params| +|DatabaseError|database error|please upgrade Python connector to latest| +|OperationalError|operation error|| +|ProgrammingError||| +|StatementError|the exception of stmt|| +|ResultError||| +|SchemalessError|the exception of stmt schemaless|| +|TmqError|the exception of stmt tmq|| + +It usually uses try-expect to handle exceptions in python. For exception handling, please refer to [Python Errors and Exceptions Documentation](https://docs.python.org/3/tutorial/errors.html). + +All exceptions from the Python Connector are thrown directly. Applications should handle these exceptions. For example: + +```python +{{#include docs/examples/python/handle_exception.py}} +``` + +## TDengine DataType vs. Python DataType + +TDengine currently supports timestamp, number, character, Boolean type, and the corresponding type conversion with Python is as follows: + +|TDengine DataType|Python DataType| +|:---------------:|:-------------:| +|TIMESTAMP|datetime| +|INT|int| +|BIGINT|int| +|FLOAT|float| +|DOUBLE|int| +|SMALLINT|int| +|TINYINT|int| +|BOOL|bool| +|BINARY|str| +|NCHAR|str| +|JSON|str| + +## Installation Steps + +### Pre-installation preparation 1. Install Python. The recent taospy package requires Python 3.6.2+. The earlier versions of taospy require Python 3.7+. The taos-ws-py package requires Python 3.7+. If Python is not available on your system, refer to the [Python BeginnersGuide](https://wiki.python.org/moin/BeginnersGuide/Download) to install it. 2. Install [pip](https://pypi.org/project/pip/). In most cases, the Python installer comes with the pip utility. If not, please refer to [pip documentation](https://pip.pypa.io/en/stable/installation/) to install it. @@ -217,7 +275,7 @@ Transfer-Encoding: chunked -### Using connectors to establish connections +### Specify the Host and Properties to get the connection The following example code assumes that TDengine is installed locally and that the default configuration is used for both FQDN and serverPort. @@ -273,7 +331,69 @@ The parameter of `connect()` is the url of TDengine, and the protocol is `taosws -## Example program +### Priority of configuration parameters + +If the configuration parameters are duplicated in the parameters or client configuration file, the priority of the parameters, from highest to lowest, are as follows: + +1. Parameters in `connect` function. +2. the configuration file taos.cfg of the TDengine client driver when using a native connection. + +## Usage examples + +### Create database and tables + + + + +```python +conn = taos.connect() +# Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. +conn.execute("DROP DATABASE IF EXISTS test") +conn.execute("CREATE DATABASE test") +# change database. same as execute "USE db" +conn.select_db("test") +conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") +``` + + + + + +```python +conn = taosrest.connect(url="http://localhost:6041") +# Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. +conn.execute("DROP DATABASE IF EXISTS test") +conn.execute("CREATE DATABASE test") +conn.execute("USE test") +conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") +``` + + + + + +```python +conn = taosws.connect("taosws://localhost:6041") +# Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. +conn.execute("DROP DATABASE IF EXISTS test") +conn.execute("CREATE DATABASE test") +conn.execute("USE test") +conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") +``` + + + + +### Insert data + +```python +conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m, 24.4)") +``` + +::: +now is an internal function. The default is the current time of the client's computer. now + 1s represents the current time of the client plus 1 second, followed by the number representing the unit of time: a (milliseconds), s (seconds), m (minutes), h (hours), d (days), w (weeks), n (months), y (years). +::: + ### Basic Usage @@ -343,6 +463,8 @@ For a more detailed description of the `sql()` method, please refer to [RestClie +The `Connection` class contains both an implementation of the PEP249 Connection interface (e.g., the `cursor()` method and the `close()` method) and many extensions (e.g., the `execute()`, `query()`, `schemaless_insert()`, and `subscribe()` methods). + ```python {{#include docs/examples/python/connect_websocket_examples.py:basic}} ``` @@ -353,7 +475,47 @@ For a more detailed description of the `sql()` method, please refer to [RestClie -### Usage with req_id +### Querying Data + + + + +The `query` method of the `TaosConnection` class can be used to query data and return the result data of type `TaosResult`. + +```python +{{#include docs/examples/python/connection_usage_native_reference.py:query}} +``` + +:::tip +The queried results can only be fetched once. For example, only one of `fetch_all()` and `fetch_all_into_dict()` can be used in the example above. Repeated fetches will result in an empty list. +::: + + + + + +The `RestClient` class is a direct wrapper for the [REST API](/reference/rest-api). It contains only a `sql()` method for executing arbitrary SQL statements and returning the result. + +```python +{{#include docs/examples/python/rest_client_example.py}} +``` + +For a more detailed description of the `sql()` method, please refer to [RestClient](https://docs.taosdata.com/api/taospy/taosrest/restclient.html). + + + + + +The `query` method of the `TaosConnection` class can be used to query data and return the result data of type `TaosResult`. + +```python +{{#include docs/examples/python/connect_websocket_examples.py:basic}} +``` + + + + +### Execute SQL with reqId By using the optional req_id parameter, you can specify a request ID that can be used for tracing. @@ -362,7 +524,7 @@ By using the optional req_id parameter, you can specify a request ID that can be ##### TaosConnection class -The `TaosConnection` class contains both an implementation of the PEP249 Connection interface (e.g., the `cursor()` method and the `close()` method) and many extensions (e.g., the `execute()`, `query()`, `schemaless_insert()`, and `subscribe()` methods). +As the way to connect introduced above but add `req_id` argument. ```python title="execute method" {{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:insert}} @@ -372,13 +534,9 @@ The `TaosConnection` class contains both an implementation of the PEP249 Connect {{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:query}} ``` -:::tip -The queried results can only be fetched once. For example, only one of `fetch_all()` and `fetch_all_into_dict()` can be used in the example above. Repeated fetches will result in an empty list. -::: - ##### Use of TaosResult class -In the above example of using the `TaosConnection` class, we have shown two ways to get the result of a query: `fetch_all()` and `fetch_all_into_dict()`. In addition, `TaosResult` also provides methods to iterate through the result set by rows (`rows_iter`) or by data blocks (`blocks_iter`). Using these two methods will be more efficient in scenarios where the query has a large amount of data. +As the way to fetch data introduced above but add `req_id` argument. ```python title="blocks_iter method" {{#include docs/examples/python/result_set_with_req_id_examples.py}} @@ -391,17 +549,12 @@ The `TaosConnection` class and the `TaosResult` class already implement all the {{#include docs/examples/python/cursor_usage_native_reference_with_req_id.py}} ``` -:::note -The TaosCursor class uses native connections for write and query operations. In a client-side multi-threaded scenario, this cursor instance must remain thread exclusive and cannot be shared across threads for use, otherwise, it will result in errors in the returned results. - -::: - ##### Use of TaosRestCursor class -The `TaosRestCursor` class is an implementation of the PEP249 Cursor interface. +As the way to connect introduced above but add `req_id` argument. ```python title="Use of TaosRestCursor" {{#include docs/examples/python/connect_rest_with_req_id_examples.py:basic}} @@ -421,8 +574,11 @@ The `RestClient` class is a direct wrapper for the [REST API](/reference/rest-ap For a more detailed description of the `sql()` method, please refer to [RestClient](https://docs.taosdata.com/api/taospy/taosrest/restclient.html). + +As the way to connect introduced above but add `req_id` argument. + ```python {{#include docs/examples/python/connect_websocket_with_req_id_examples.py:basic}} ``` @@ -459,26 +615,157 @@ For a more detailed description of the `sql()` method, please refer to [RestClie -### Schemaless Insert +### Writing data via parameter binding + +The Python connector provides a parameter binding api for inserting data. Similar to most databases, TDengine currently only supports the question mark `?` to indicate the parameters to be bound. + + + + +##### Create Stmt + +Call the `statement` method in `Connection` to create the `stmt` for parameter binding. + +``` +import taos + +conn = taos.connect() +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") +``` + +##### parameter binding + +Call the `new_multi_binds` function to create the parameter list for parameter bindings. + +``` +params = new_multi_binds(16) +params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) +params[1].bool((True, None, False)) +params[2].tinyint([-128, -128, None]) # -128 is tinyint null +params[3].tinyint([0, 127, None]) +params[4].smallint([3, None, 2]) +params[5].int([3, 4, None]) +params[6].bigint([3, 4, None]) +params[7].tinyint_unsigned([3, 4, None]) +params[8].smallint_unsigned([3, 4, None]) +params[9].int_unsigned([3, 4, None]) +params[10].bigint_unsigned([3, 4, None]) +params[11].float([3, None, 1]) +params[12].double([3, None, 1.2]) +params[13].binary(["abc", "dddafadfadfadfadfa", None]) +params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) +params[15].timestamp([None, None, 1626861392591]) +``` + +Call the `bind_param` (for a single row) method or the `bind_param_batch` (for multiple rows) method to set the values. + +``` +stmt.bind_param_batch(params) +``` + +##### execute sql + +Call `execute` method to execute sql. + +``` +stmt.execute() +``` + +##### Close Stmt + +``` +stmt.close() +``` + +##### Example + +```python +{{#include docs/examples/python/stmt_example.py}} +``` + + + + +##### Create Stmt + +Call the `statement` method in `Connection` to create the `stmt` for parameter binding. + +``` +import taosws + +conn = taosws.connect('taosws://localhost:6041/test') +stmt = conn.statement() +``` + +##### Prepare sql + +Call `prepare` method in stmt to prepare sql. + +``` +stmt.prepare("insert into t1 values (?, ?, ?, ?)") +``` + +##### parameter binding + +Call the `bind_param` method to bind parameters. + +``` +stmt.bind_param([ + taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]), + taosws.ints_to_column([1, 2, 3, 4]), + taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]), + taosws.varchar_to_column(['a', 'b', 'c', 'd']), +]) +``` + +Call the `add_batch` method to add parameters to the batch. + +``` +stmt.add_batch() +``` + +##### execute sql + +Call `execute` method to execute sql. + +``` +stmt.execute() +``` + +##### Close Stmt + +``` +stmt.close() +``` + +##### Example + +```python +{{#include docs/examples/python/stmt_websocket_example.py}} +``` + + + +### Schemaless Writing Connector support schemaless insert. -Simple insert +##### Simple insert ```python {{#include docs/examples/python/schemaless_insert.py}} ``` -Insert with ttl argument +##### Insert with ttl argument ```python {{#include docs/examples/python/schemaless_insert_ttl.py}} ``` -Insert with req_id argument +##### Insert with req_id argument ```python {{#include docs/examples/python/schemaless_insert_req_id.py}} @@ -488,19 +775,19 @@ Insert with req_id argument -Simple insert +##### Simple insert ```python {{#include docs/examples/python/schemaless_insert_raw.py}} ``` -Insert with ttl argument +##### Insert with ttl argument ```python {{#include docs/examples/python/schemaless_insert_raw_ttl.py}} ``` -Insert with req_id argument +##### Insert with req_id argument ```python {{#include docs/examples/python/schemaless_insert_raw_req_id.py}} @@ -509,26 +796,226 @@ Insert with req_id argument +### Schemaless with reqId + +There is a optional parameter called `req_id` in `schemaless_insert` and `schemaless_insert_raw` method. This reqId can be used to request link tracing. + +```python +{{#include docs/examples/python/schemaless_insert_req_id.py}} +``` + +```python +{{#include docs/examples/python/schemaless_insert_raw_req_id.py}} +``` + +### Data Subscription + +Connector support data subscription. For more information about subscroption, please refer to [Data Subscription](../../../develop/tmq/). + +#### Create a Topic + +To create topic, please refer to [Data Subscription](../../../develop/tmq/#create-a-topic). + +#### Create a Consumer + + + + + +The consumer in the connector contains the subscription api. The syntax for creating a consumer is consumer = Consumer(configs). For more subscription api parameters, please refer to [Data Subscription](../../../develop/tmq/#create-a-consumer). + +```python +from taos.tmq import Consumer + +consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) +``` + + + + +In addition to native connections, the connector also supports subscriptions via websockets. + +The syntax for creating a consumer is "consumer = consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../../develop/tmq/#create-a-consumer). + +```python +import taosws + +consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) +``` + + + + +#### Subscribe to a Topic + + + + + +The `subscribe` function is used to subscribe to a list of topics. + +```python +consumer.subscribe(['topic1', 'topic2']) +``` + + + + +The `subscribe` function is used to subscribe to a list of topics. + +```python +consumer.subscribe(['topic1', 'topic2']) +``` + + + + +#### Consume messages + + + + + +The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. You have to handle error messages in response data. + +```python +while True: + res = consumer.poll(1) + if not res: + continue + err = res.error() + if err is not None: + raise err + val = res.value() + + for block in val: + print(block.fetchall()) +``` + + + + +The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. You have to handle error messages in response data. + +```python +while True: + res = consumer.poll(timeout=1.0) + if not res: + continue + err = res.error() + if err is not None: + raise err + for block in message: + for row in block: + print(row) +``` + + + + +#### Assignment subscription Offset + + + + + +The `assignment` function is used to get the assignment of the topic. + +```python +assignments = consumer.assignment() +``` + +The `seek` function is used to reset the assignment of the topic. + +```python +tp = TopicPartition(topic='topic1', partition=0, offset=0) +consumer.seek(tp) +``` + + + + +The `assignment` function is used to get the assignment of the topic. + +```python +assignments = consumer.assignment() +``` + +The `seek` function is used to reset the assignment of the topic. + +```python +consumer.seek(topic='topic1', partition=0, offset=0) +``` + + + + +#### Close subscriptions + + + + + +You should unsubscribe to the topics and close the consumer after consuming. + +```python +consumer.unsubscribe() +consumer.close() +``` + + + + +You should unsubscribe to the topics and close the consumer after consuming. + +```python +consumer.unsubscribe() +consumer.close() +``` + + + + +#### Full Sample Code + + + + + +```python +{{#include docs/examples/python/tmq_example.py}} +``` + +```python +{{#include docs/examples/python/tmq_assignment_example.py:taos_get_assignment_and_seek_demo}} +``` + + + + +```python +{{#include docs/examples/python/tmq_websocket_example.py}} +``` + +```python +{{#include docs/examples/python/tmq_websocket_assgnment_example.py:taosws_get_assignment_and_seek_demo}} +``` + + + + ### Other sample programs | Example program links | Example program content | -| ------------------------------------------------------------------------------------------------------------- | ------------------- ---- | +|-----------------------|-------------------------| | [bind_multi.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-multi.py) | parameter binding, bind multiple rows at once | -| [bind_row.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-row.py) | bind_row.py +| [bind_row.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-row.py) | parameter binding, bind one row at once | | [insert_lines.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/insert-lines.py) | InfluxDB line protocol writing | | [json_tag.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/json-tag.py) | Use JSON type tags | -| [tmq.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/tmq.py) | TMQ subscription | +| [tmq_consumer.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/tmq_consumer.py) | TMQ subscription | ## Other notes -### Exception handling - -All errors from database operations are thrown directly as exceptions and the error message from the database is passed up the exception stack. The application is responsible for exception handling. For example: - -```python -{{#include docs/examples/python/handle_exception.py}} -``` - ### About nanoseconds Due to the current imperfection of Python's nanosecond support (see link below), the current implementation returns integers at nanosecond precision instead of the `datetime` type produced by `ms` and `us`, which application developers will need to handle on their own. And it is recommended to use pandas' to_datetime(). The Python Connector may modify the interface in the future if Python officially supports nanoseconds in full. diff --git a/docs/en/14-reference/03-connector/60-r-lang.mdx b/docs/en/14-reference/03-connector/60-r-lang.mdx new file mode 100644 index 0000000000000000000000000000000000000000..f1cbb89f7b1bbda310578cdee064821b8444cc09 --- /dev/null +++ b/docs/en/14-reference/03-connector/60-r-lang.mdx @@ -0,0 +1,87 @@ +--- +toc_max_heading_level: 4 +sidebar_label: R +title: R Language Connector +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +import Rdemo from "../../07-develop/01-connect/_connect_r.mdx" + +By using the RJDBC library in R, you can enable R programs to access TDengine data. Here are the installation process, configuration steps, and an example code in R. + +## Installation Process + +Before getting started, make sure you have installed the R language environment. Then, follow these steps to install and configure the RJDBC library: + +1. Install Java Development Kit (JDK): RJDBC library requires Java environment. Download the appropriate JDK for your operating system from the official Oracle website and follow the installation guide. + +2. Install the RJDBC library: Execute the following command in the R console to install the RJDBC library. + +```r +install.packages("RJDBC", repos='http://cran.us.r-project.org') +``` + +:::note +1. The default R language package version 4.2 which shipped with Ubuntu might lead unresponsive bug. Please install latest version of R language package from the [official website](https://www.r-project.org/). +2. On Linux systems, installing the RJDBC package may require installing the necessary components for compilation. For example, on Ubuntu, you can execute the command ``apt install -y libbz2-dev libpcre2-dev libicu-dev`` to install the required components. +3. On Windows systems, you need to set the **JAVA_HOME** environment variable. +::: + +3. Download the TDengine JDBC driver: Visit the Maven website and download the TDengine JDBC driver (taos-jdbcdriver-X.X.X-dist.jar) to your local machine. + +## Configuration Process + +Once you have completed the installation steps, you need to do some configuration to enable the RJDBC library to connect and access the TDengine time-series database. + +1. Load the RJDBC library and other necessary libraries in your R script: + +```r +library(DBI) +library(rJava) +library(RJDBC) +``` + +2. Set the JDBC driver and JDBC URL: + +```r +# Set the JDBC driver path (specify the location on your local machine) +driverPath <- "/path/to/taos-jdbcdriver-X.X.X-dist.jar" + +# Set the JDBC URL (specify the FQDN and credentials of your TDengine cluster) +url <- "jdbc:TAOS://localhost:6030/?user=root&password=taosdata" +``` + +3. Load the JDBC driver: + +```r +# Load the JDBC driver +drv <- JDBC("com.taosdata.jdbc.TSDBDriver", driverPath) +``` + +4. Create a TDengine database connection: + +```r +# Create a database connection +conn <- dbConnect(drv, url) +``` + +5. Once the connection is established, you can use the ``conn`` object for various database operations such as querying data and inserting data. + +6. Finally, don't forget to close the database connection after you are done: + +```r +# Close the database connection +dbDisconnect(conn) +``` + +## Example Code Using RJDBC in R + +Here's an example code that uses the RJDBC library to connect to a TDengine time-series database and perform a query operation: + + + +Please modify the JDBC driver, JDBC URL, username, password, and SQL query statement according to your specific TDengine time-series database environment and requirements. + +By following the steps and using the provided example code, you can use the RJDBC library in the R language to access the TDengine time-series database and perform tasks such as data querying and analysis. diff --git a/docs/en/14-reference/03-connector/10-php.mdx b/docs/en/14-reference/03-connector/80-php.mdx similarity index 100% rename from docs/en/14-reference/03-connector/10-php.mdx rename to docs/en/14-reference/03-connector/80-php.mdx diff --git a/docs/en/14-reference/03-connector/index.mdx b/docs/en/14-reference/03-connector/index.mdx index 28b7b83b58f1c755d147981dc80a443f21e9860d..4a3e9195d6820d251ae484b6476b971a55956226 100644 --- a/docs/en/14-reference/03-connector/index.mdx +++ b/docs/en/14-reference/03-connector/index.mdx @@ -48,7 +48,6 @@ Comparing the connector support for TDengine functional features as follows. | **Parameter Binding** | Support | Support | Support | Support | Support | Support | | **Subscription (TMQ)** | Support | Support | Support | Support | Support | Support | | **Schemaless** | Support | Support | Support | Support | Support | Support | -| **DataFrame** | Not Supported | Support | Not Supported | Not Supported | Not Supported | Not Supported | :::info The different database framework specifications for various programming languages do not mean that all C/C++ interfaces need a wrapper. @@ -60,11 +59,10 @@ The different database framework specifications for various programming language | -------------------------------------- | ------------- | --------------- | ------------- | ------------- | ------------- | ------------- | | **Connection Management** | Support | Support | Support | Support | Support | Support | | **Regular Query** | Support | Support | Support | Support | Support | Support | -| **Parameter Binding** | Not Supported | Not Supported | Support | Support | Not Supported | Support | +| **Parameter Binding** | Supported | Supported | Support | Support | Not Supported | Support | | **Subscription (TMQ) ** | Supported | Support | Support | Not Supported | Not Supported | Support | -| **Schemaless** | Supported | Not Supported | Not Supported | Not Supported | Not Supported | Not Supported | +| **Schemaless** | Supported | Supported | Supported | Not Supported | Not Supported | Not Supported | | **Bulk Pulling (based on WebSocket) ** | Support | Support | Support | Support | Support | Support | -| **DataFrame** | Not Supported | Support | Not Supported | Not Supported | Not Supported | Not Supported | :::warning diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index 8fc20c149f5f5dbaf56ed7fc7d065d42b8af81c5..38a8048a21706415c4712f25aec366a4cd2afa3c 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -364,6 +364,7 @@ The configuration parameters for specifying super table tag columns and data col - **min**: The minimum value of the column/label of the data type. The generated value will equal or large than the minimum value. - **max**: The maximum value of the column/label of the data type. The generated value will less than the maximum value. +- **fun**: This column of data is filled with functions. Currently, only the sin and cos functions are supported. The input parameter is the timestamp and converted to an angle value. The conversion formula is: angle x=input time column ts value % 360. At the same time, it supports coefficient adjustment and random fluctuation factor adjustment, presented in a fixed format expression, such as fun="10\*sin(x)+100\*random(5)", where x represents the angle, ranging from 0 to 360 degrees, and the growth step size is consistent with the time column step size. 10 represents the coefficient of multiplication, 100 represents the coefficient of addition or subtraction, and 5 represents the fluctuation range within a random range of 5%. The currently supported data types are int, bigint, float, and double. Note: The expression is fixed and cannot be reversed. - **values**: The value field of the nchar/binary column/label, which will be chosen randomly from the values. @@ -470,3 +471,26 @@ The configuration parameters for subscribing to a super table are set in `super_ - **sql**: The SQL command to be executed. For the query SQL of super table, keep "xxxx" in the SQL command. The program will automatically replace it with all the sub-table names of the super table. Replace it with all the sub-table names in the super table. - **result**: The file to save the query result. If not specified, taosBenchmark will not save result. + +#### data type on taosBenchmark + +| # | **TDengine** | **taosBenchmark** +| --- | :----------------: | :---------------: +| 1 | TIMESTAMP | timestamp +| 2 | INT | int +| 3 | INT UNSIGNED | uint +| 4 | BIGINT | bigint +| 5 | BIGINT UNSIGNED | ubigint +| 6 | FLOAT | float +| 7 | DOUBLE | double +| 8 | BINARY | binary +| 9 | SMALLINT | smallint +| 10 | SMALLINT UNSIGNED | usmallint +| 11 | TINYINT | tinyint +| 12 | TINYINT UNSIGNED | utinyint +| 13 | BOOL | bool +| 14 | NCHAR | nchar +| 15 | VARCHAR | varchar +| 15 | JSON | json + +note：Lowercase characters must be used on taosBenchmark datatype diff --git a/docs/en/14-reference/06-taosdump.md b/docs/en/14-reference/06-taosdump.md index 7348add4bd276c9f105c47f8b09b12dc645e3206..6d5547e7a97ad92d5e37f6c46f5298ac62ec8768 100644 --- a/docs/en/14-reference/06-taosdump.md +++ b/docs/en/14-reference/06-taosdump.md @@ -79,8 +79,6 @@ Usage: taosdump [OPTION...] dbname [tbname ...] -e, --escape-character Use escaped character for database name -N, --without-property Dump database without its properties. -s, --schemaonly Only dump table schemas. - -y, --answer-yes Input yes for prompt. It will skip data file - checking! -d, --avro-codec=snappy Choose an avro codec among null, deflate, snappy, and lzma. -S, --start-time=START_TIME Start time to dump. Either epoch or diff --git a/docs/en/14-reference/07-tdinsight/index.md b/docs/en/14-reference/07-tdinsight/index.md index 965a38168ad93691b37a1b7e74aa995e6346987a..d6d1cd4f1a6fab1bc16981aa7ad65e45c2d6466a 100644 --- a/docs/en/14-reference/07-tdinsight/index.md +++ b/docs/en/14-reference/07-tdinsight/index.md @@ -149,7 +149,7 @@ curl --no-progress-meter -u admin:admin http://localhost:3000/api/alert-notifica Use the `uid` value obtained above as `-E` input. ```bash -sudo ./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord -E existing-notifier +./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord -E existing-notifier ``` If you want to monitor multiple TDengine clusters, you need to set up numerous TDinsight dashboards. Setting up non-default TDinsight requires some changes: the `-n` `-i` `-t` options need to be changed to non-default names, and `-N` and `-L` should also be changed if using the built-in SMS alerting feature. diff --git a/docs/en/14-reference/09-support-platform/index.md b/docs/en/14-reference/09-support-platform/index.md index 7dfa8ac93a9e91eb9238f7b56033a592c241079f..779882f5827dcb7780f636cfda8c8ee91ab1e9d2 100644 --- a/docs/en/14-reference/09-support-platform/index.md +++ b/docs/en/14-reference/09-support-platform/index.md @@ -5,12 +5,12 @@ description: This document describes the supported platforms for the TDengine se ## List of supported platforms for TDengine server -| | **Windows Server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18/20** | **macOS** | +| | **Windows Server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18 or later** | **macOS** | | ------------ | ---------------------------- | ----------------- | ---------------- | ---------------- | --------- | -| X64 | ● | ● | ● | ● | ● | +| X64 | ●/E | ●/E | ● | ● | ● | | ARM64 | | | ● | | ● | -Note: ● means officially tested and verified, ○ means unofficially tested and verified. +Note: 1) ● means officially tested and verified, ○ means unofficially tested and verified, E means only supported by the enterprise edition. 2) The community edition only supports newer versions of mainstream operating systems, including Ubuntu 18+/CentOS 7+/RetHat/Debian/CoreOS/FreeBSD/OpenSUSE/SUSE Linux/Fedora/macOS, etc. If you have requirements for other operating systems and editions, please contact support of the enterprise edition. ## List of supported platforms for TDengine clients and connectors diff --git a/docs/en/14-reference/11-docker/_category_.yml b/docs/en/14-reference/11-docker/_category_.yml deleted file mode 100644 index f89ef7112c6d850726e5bcf20f1d1bd9f65d02fd..0000000000000000000000000000000000000000 --- a/docs/en/14-reference/11-docker/_category_.yml +++ /dev/null @@ -1 +0,0 @@ -label: TDengine Docker images \ No newline at end of file diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md old mode 100644 new mode 100755 index c3d28fa48e0bde79bae873519edc4d1ea059a364..c0512aef7b5822adf757446e9cb4c698c7b93075 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -5,7 +5,7 @@ description: This document describes the configuration parameters for the TDengi ## Configuration File on Server Side -On the server side, the actual service of TDengine is provided by an executable `taosd` whose parameters can be configured in file `taos.cfg` to meet the requirements of different use cases. The default location of `taos.cfg` is `/etc/taos`, but can be changed by using `-c` parameter on the CLI of `taosd`. For example, the configuration file can be put under `/home/user` and used like below +On the server side, the actual service of TDengine is provided by an executable `taosd` whose parameters can be configured in file `taos.cfg` to meet the requirements of different use cases. The default location of `taos.cfg` is `/etc/taos` on Linux system, it's located under `C:\TDengine` on Windows system. The location of configuration file can be specified by using `-c` parameter on the CLI of `taosd`. For example, on Linux system the configuration file can be put under `/home/user` and used like below ``` taosd -c /home/user @@ -19,14 +19,18 @@ taosd -C ## Configuration File on Client Side -TDengine CLI `taos` is the tool for users to interact with TDengine. It can share same configuration file as `taosd` or use a separate configuration file. When launching `taos`, parameter `-c` can be used to specify the location where its configuration file is. For example `taos -c /home/cfg` means `/home/cfg/taos.cfg` will be used. If `-c` is not used, the default location of the configuration file is `/etc/taos`. For more details please use `taos --help` to get. +TDengine CLI `taos` is the tool for users to interact with TDengine. It can share same configuration file as `taosd` or use a separate configuration file. When launching `taos`, parameter `-c` can be used to specify the location where its configuration file is. For example: -```bash -taos -C ``` +taos -c /home/cfg +``` + +means `/home/cfg/taos.cfg` will be used. If `-c` is not used, the default location of the configuration file is `/etc/taos`. For more details please use `taos --help` to get. + +Parameter `-C` can be used on the CLI of `taos` to show its configuration, like below: ```bash -taos --dump-config +taos -C ``` ## Configuration Parameters @@ -45,19 +49,19 @@ The parameters described in this document by the effect that they have on the sy ### firstEp -| Attribute | Description | -| -------- | -------------------------------------------------------------- | -| Applicable | Server and Client | -| Meaning | The end point of the first dnode in the cluster to be connected to when `taosd` or `taos` is started | -| Default | localhost:6030 | +| Attribute | Description | +| ---------- | ---------------------------------------------------------------------------------------------------- | +| Applicable | Server and Client | +| Meaning | The end point of the first dnode in the cluster to be connected to when `taosd` or `taos` is started | +| Default | localhost:6030 | ### secondEp -| Attribute | Description | -| -------- | ------------------------------------------------------------------------------------- | -| Applicable | Server and Client | -| Meaning | The end point of the second dnode to be connected to if the firstEp is not available when `taosd` or `taos` is started | -| Default | None | +| Attribute | Description | +| ---------- | ---------------------------------------------------------------------------------------------------------------------- | +| Applicable | Server and Client | +| Meaning | The end point of the second dnode to be connected to if the firstEp is not available when `taosd` or `taos` is started | +| Default | None | ### fqdn @@ -65,28 +69,29 @@ The parameters described in this document by the effect that they have on the sy | ------------- | ------------------------------------------------------------------------ | | Applicable | Server Only | | Meaning | The FQDN of the host where `taosd` will be started. It can be IP address | -| Default Value | The first hostname configured for the host | -| Note | It should be within 96 bytes | | +| Default Value | The first hostname configured for the host | +| Note | It should be within 96 bytes | | ### serverPort -| Attribute | Description | -| -------- | ----------------------------------------------------------------------------------------------------------------------- | -| Applicable | Server Only | -| Meaning | The port for external access after `taosd` is started | -| Default Value | 6030 | +| Attribute | Description | +| ------------- | ----------------------------------------------------- | +| Applicable | Server Only | +| Meaning | The port for external access after `taosd` is started | +| Default Value | 6030 | :::note -- Ensure that your firewall rules do not block TCP port 6042 on any host in the cluster. Below table describes the ports used by TDengine in details. +Ensure that your firewall rules do not block TCP port 6042 on any host in the cluster. Below table describes the ports used by TDengine in details. ::: -| Protocol | Default Port | Description | How to configure | -| :------- | :----------- | :----------------------------------------------- | :--------------------------------------------------------------------------------------------- | -| TCP | 6030 | Communication between client and server. In a multi-node cluster, communication between nodes. serverPort | -| TCP | 6041 | REST connection between client and server | Prior to 2.4.0.0: serverPort+11; After 2.4.0.0 refer to [taosAdapter](/reference/taosadapter/) | -| TCP | 6043 | Service Port of taosKeeper | The parameter of taosKeeper | -| TCP | 6044 | Data access port for StatsD | Configurable through taosAdapter parameters. -| UDP | 6045 | Data access for statsd | Configurable through taosAdapter parameters. -| TCP | 6060 | Port of Monitoring Service in Enterprise version | | + +| Protocol | Default Port | Description | How to configure | +| :------- | :----------- | :-------------------------------------------------------------------------------------------------------- | :--------------------------------------------------------------------------------------------- | +| TCP | 6030 | Communication between client and server. In a multi-node cluster, communication between nodes. serverPort | +| TCP | 6041 | REST connection between client and server | Prior to 2.4.0.0: serverPort+11; After 2.4.0.0 refer to [taosAdapter](/reference/taosadapter/) | +| TCP | 6043 | Service Port of taosKeeper | The parameter of taosKeeper | +| TCP | 6044 | Data access port for StatsD | Configurable through taosAdapter parameters. | +| UDP | 6045 | Data access for statsd | Configurable through taosAdapter parameters. | +| TCP | 6060 | Port of Monitoring Service in Enterprise version | | ### maxShellConns @@ -97,121 +102,141 @@ The parameters described in this document by the effect that they have on the sy | Value Range | 10-50000000 | | Default Value | 5000 | +### numOfRpcSessions + +| Attribute | Description | +| ------------- | ------------------------------------------ | +| Applicable | Client/Server | +| Meaning | The maximum number of connection to create | +| Value Range | 100-100000 | +| Default Value | 10000 | + +### timeToGetAvailableConn + +| Attribute | Description | +| ------------- | ---------------------------------------------- | +| Applicable | Client/Server | +| Meaning | The maximum waiting time to get available conn | +| Value Range | 10-50000000(ms) | +| Default Value | 500000 | + ## Monitoring Parameters :::note Please note the `taoskeeper` needs to be installed and running to create the `log` database and receiving metrics sent by `taosd` as the full monitoring solution. +::: + ### monitor -| Attribute | Description | -| -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Applicable | Server only | -| Meaning | The switch for monitoring inside server. The main object of monitoring is to collect information about load on physical nodes, including CPU usage, memory usage, disk usage, and network bandwidth. Monitoring information is sent over HTTP to the taosKeeper service specified by `monitorFqdn` and `monitorProt`. -| Value Range | 0: monitoring disabled, 1: monitoring enabled | -| Default | 0 | +| Attribute | Description | +| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Applicable | Server only | +| Meaning | The switch for monitoring inside server. The main object of monitoring is to collect information about load on physical nodes, including CPU usage, memory usage, disk usage, and network bandwidth. Monitoring information is sent over HTTP to the taosKeeper service specified by `monitorFqdn` and `monitorProt`. | +| Value Range | 0: monitoring disabled, 1: monitoring enabled | +| Default | 0 | ### monitorFqdn -| Attribute | Description | -| -------- | -------------------------- | -| Applicable | Server Only | -| Meaning | FQDN of taosKeeper monitoring service | -| Default | None | +| Attribute | Description | +| ---------- | ------------------------------------- | +| Applicable | Server Only | +| Meaning | FQDN of taosKeeper monitoring service | +| Default | None | ### monitorPort -| Attribute | Description | -| -------- | --------------------------- | -| Applicable | Server Only | -| Meaning | Port of taosKeeper monitoring service | -| Default Value | 6043 | +| Attribute | Description | +| ------------- | ------------------------------------- | +| Applicable | Server Only | +| Meaning | Port of taosKeeper monitoring service | +| Default Value | 6043 | ### monitorInterval -| Attribute | Description | -| -------- | -------------------------------------------- | -| Applicable | Server Only | +| Attribute | Description | +| ------------- | ------------------------------------------ | +| Applicable | Server Only | | Meaning | The interval of collecting system workload | | Unit | second | -| Value Range | 1-200000 | -| Default Value | 30 | +| Value Range | 1-200000 | +| Default Value | 30 | ### telemetryReporting -| Attribute | Description | -| -------- | ---------------------------------------- | -| Applicable | Server Only | +| Attribute | Description | +| ------------- | ---------------------------------------------------------------------------- | +| Applicable | Server and Client | | Meaning | Switch for allowing TDengine to collect and report service usage information | | Value Range | 0: Not allowed; 1: Allowed | -| Default Value | 1 | +| Default Value | 1 | ### crashReporting -| Attribute | Description | -| -------- | -------------------------------------------- | -| Applicable | Server Only | -| Meaning |Switch for allowing TDengine to collect and report crash related information | -| Value Range | 0,1 0: Not allowed; 1: allowed | -| Default Value | 1 | +| Attribute | Description | +| ------------- | ---------------------------------------------------------------------------- | +| Applicable | Server and Client | +| Meaning | Switch for allowing TDengine to collect and report crash related information | +| Value Range | 0,1 0: Not allowed; 1: allowed | +| Default Value | 1 | ## Query Parameters ### queryPolicy -| Attribute | Description | -| -------- | ----------------------------- | -| Applicable | Client only | -| Meaning | Execution policy for query statements | -| Unit | None | -| Default | 1 | +| Attribute | Description | +| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Applicable | Client only | +| Meaning | Execution policy for query statements | +| Unit | None | +| Default | 1 | | Value Range | 1: Run queries on vnodes and not on qnodes; 2: Run subtasks without scan operators on qnodes and subtasks with scan operators on vnodes; 3: Only run scan operators on vnodes, and run all other operators on qnodes. | ### querySmaOptimize -| Attribute | Description | -| -------- | -------------------- | -| Applicable | Client only | -| Meaning | SMA index optimization policy | -| Unit | None | -| Default Value | 0 | -| Notes |0: Disable SMA indexing and perform all queries on non-indexed data; 1: Enable SMA indexing and perform queries from suitable statements on precomputation results.| +| Attribute | Description | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Applicable | Client only | +| Meaning | SMA index optimization policy | +| Unit | None | +| Default Value | 0 | +| Notes | 0: Disable SMA indexing and perform all queries on non-indexed data; 1: Enable SMA indexing and perform queries from suitable statements on precomputation results. | -### countAlwaysReturnValue +### countAlwaysReturnValue -| Attribute | Description | -| -------- | -------------------------------- | -| Applicable | Server only | -| Meaning | count()/hyperloglog() return value or not if the input data is empty or NULL | -| Vlue Range | 0: Return empty line, 1: Return 0 | -| Default | 1 | -| Notes | When this parameter is setting to 1, for queries containing GROUP BY, PARTITION BY and INTERVAL clause, and input data in certain groups or windows is empty or NULL, the corresponding groups or windows have no return values | +| Attribute | Description | +| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Applicable | Server only | +| Meaning | count()/hyperloglog() return value or not if the input data is empty or NULL | +| Vlue Range | 0: Return empty line, 1: Return 0 | +| Default | 1 | +| Notes | When this parameter is setting to 1, for queries containing GROUP BY, PARTITION BY and INTERVAL clause, and input data in certain groups or windows is empty or NULL, the corresponding groups or windows have no return values | ### maxNumOfDistinctRes -| Attribute | Description | -| -------- | -------------------------------- | -| Applicable | Server Only | +| Attribute | Description | +| ------------- | -------------------------------------------- | +| Applicable | Server Only | | Meaning | The maximum number of distinct rows returned | | Value Range | [100,000 - 100,000,000] | | Default Value | 100,000 | ### keepColumnName -| Attribute | Description | -| -------- | -------------------------------- | -| Applicable | Client only | -| Meaning | When the Last, First, LastRow function is queried, whether the returned column name contains the function name. | -| Value Range | 0 means including the function name, 1 means not including the function name. | -| Default Value | 0 | +| Attribute | Description | +| ------------- | --------------------------------------------------------------------------------------------------------------- | +| Applicable | Client only | +| Meaning | When the Last, First, LastRow function is queried, whether the returned column name contains the function name. | +| Value Range | 0 means including the function name, 1 means not including the function name. | +| Default Value | 0 | ## Locale Parameters ### timezone -| Attribute | Description | -| -------- | ------------------------------ | -| Applicable | Server and Client | +| Attribute | Description | +| ------------- | ------------------------------- | +| Applicable | Server and Client | | Meaning | TimeZone | | Default Value | TimeZone configured in the host | @@ -314,49 +339,59 @@ The charset that takes effect is UTF-8. ### dataDir -| Attribute | Description | -| -------- | ------------------------------------------ | -| Applicable | Server Only | -| Meaning | All data files are stored in this directory | -| Default Value | /var/lib/taos | +| Attribute | Description | +| ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Applicable | Server Only | +| Meaning | All data files are stored in this directory | +| Default Value | /var/lib/taos | | Note | The [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function needs to be used in conjunction with the [KEEP](https://docs.tdengine.com/taos-sql/database/#parameters) parameter | ### tempDir -| Attribute | Description | -| -------- | ------------------------------------------ | -| Applicable | Server only | -| Meaning | The directory where to put all the temporary files generated during system running | -| Default | /tmp | +| Attribute | Description | +| ---------- | ---------------------------------------------------------------------------------- | +| Applicable | Server only | +| Meaning | The directory where to put all the temporary files generated during system running | +| Default | /tmp | ### minimalTmpDirGB -| Attribute | Description | -| -------- | ------------------------------------------------ | -| Applicable | Server and Client | +| Attribute | Description | +| ------------- | ----------------------------------------------------------------------------------------------- | +| Applicable | Server and Client | | Meaning | When the available disk space in tmpDir is below this threshold, writing to tmpDir is suspended | -| Unit | GB | -| Default Value | 1.0 | +| Unit | GB | +| Default Value | 1.0 | ### minimalDataDirGB -| Attribute | Description | -| -------- | ------------------------------------------------ | -| Applicable | Server Only | +| Attribute | Description | +| ------------- | ------------------------------------------------------------------------------------------------- | +| Applicable | Server Only | | Meaning | When the available disk space in dataDir is below this threshold, writing to dataDir is suspended | -| Unit | GB | -| Default Value | 2.0 | +| Unit | GB | +| Default Value | 2.0 | + +### metaCacheMaxSize + +| Attribute | Description | +| ------------- | ------------------------------------------------------------------------------------------------- | +| Applicable | Client Only | +| Meaning | Maximum meta cache size in single client process | +| Unit | MB | +| Default Value | -1 (No limitation) | + ## Cluster Parameters ### supportVnodes -| Attribute | Description | -| -------- | --------------------------- | -| Applicable | Server Only | -| Meaning | Maximum number of vnodes per dnode | -| Value Range | 0-4096 | -| Default Value | 2x the CPU cores | +| Attribute | Description | +| ------------- | ---------------------------------- | +| Applicable | Server Only | +| Meaning | Maximum number of vnodes per dnode | +| Value Range | 0-4096 | +| Default Value | 2x the CPU cores | ## Performance Tuning @@ -373,345 +408,403 @@ The charset that takes effect is UTF-8. ### logDir -| Attribute | Description | -| -------- | -------------------------------------------------- | -| Applicable | Server and Client | +| Attribute | Description | +| ------------- | ----------------------------------- | +| Applicable | Server and Client | | Meaning | The directory for writing log files | | Default Value | /var/log/taos | ### minimalLogDirGB -| Attribute | Description | -| -------- | -------------------------------------------- | -| Applicable | Server and Client | +| Attribute | Description | +| ------------- | -------------------------------------------------------------------------------------------------- | +| Applicable | Server and Client | | Meaning | When the available disk space in logDir is below this threshold, writing to log files is suspended | -| Unit | GB | -| Default Value | 1.0 | +| Unit | GB | +| Default Value | 1.0 | ### numOfLogLines -| Attribute | Description | -| -------- | ---------------------------- | -| Applicable | Server and Client | +| Attribute | Description | +| ------------- | ------------------------------------------ | +| Applicable | Server and Client | | Meaning | Maximum number of lines in single log file | -| Default Value | 10000000 | +| Default Value | 10000000 | ### asyncLog -| Attribute | Description | -| -------- | -------------------- | -| Applicable | Server and Client | +| Attribute | Description | +| ------------- | ---------------------------- | +| Applicable | Server and Client | | Meaning | The mode of writing log file | | Value Range | 0: sync way; 1: async way | -| Default Value | 1 | +| Default Value | 1 | ### logKeepDays -| Attribute | Description | -| -------- | ----------------------------------------------------------------------------------- | -| Applicable | Server and Client | +| Attribute | Description | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| Applicable | Server and Client | | Meaning | The number of days for log files to be kept | -| Unit | day | -| Default Value | 0 | +| Unit | day | +| Default Value | 0 | | Note | When it's bigger than 0, the log file would be renamed to "taosdlog.xxx" in which "xxx" is the timestamp when the file is changed last time | +### slowLogThreshold + +| Attribute | Description | +| ------------- | -------------------------------------------------------------------------------------------------------- | +| Applicable | Client only | +| Meaning | When an operation execution time exceeds this threshold, the operation will be logged in slow log file | +| Unit | second | +| Default Value | 3 | +| Note | All slow operations will be logged in file "taosSlowLog" in the log directory | + +### slowLogScope + +| Attribute | Description | +| --------------- | ----------------------------------------------------------------------- | +| Applicable | Client only | +| Meaning | Slow log type to be logged | +| Optional Values | ALL, QUERY, INSERT, OTHERS, NONE | +| Default Value | ALL | +| Note | All slow operations will be logged by default, one option could be set | + ### debugFlag -| Attribute | Description | -| -------- | ------------------------------------------------------------------------------------------------- | -| Applicable | Server and Client | +| Attribute | Description | +| ------------- | --------------------------------------------------------- | +| Applicable | Server and Client | | Meaning | Log level | | Value Range | 131: INFO/WARNING/ERROR; 135: plus DEBUG; 143: plus TRACE | | Default Value | 131 or 135, depending on the module | ### tmrDebugFlag -| Attribute | Description | -| -------- | -------------------- | +| Attribute | Description | +| ------------- | ------------------------- | | Applicable | Server and Client | | Meaning | Log level of timer module | -| Value Range | same as debugFlag | -| Default Value | | +| Value Range | same as debugFlag | +| Default Value | | ### uDebugFlag -| Attribute | Description | -| -------- | ---------------------- | -| Applicable | Server and Client | +| Attribute | Description | +| ------------- | -------------------------- | +| Applicable | Server and Client | | Meaning | Log level of common module | -| Value Range | same as debugFlag | -| Default Value | | +| Value Range | same as debugFlag | +| Default Value | | ### rpcDebugFlag -| Attribute | Description | -| -------- | -------------------- | -| Applicable | Server and Client | +| Attribute | Description | +| ------------- | ----------------------- | +| Applicable | Server and Client | | Meaning | Log level of rpc module | -| Value Range | same as debugFlag | -| Default Value | | +| Value Range | same as debugFlag | +| Default Value | | ### jniDebugFlag -| Attribute | Description | -| -------- | ------------------ | -| Applicable | Client Only | +| Attribute | Description | +| ------------- | ----------------------- | +| Applicable | Client Only | | Meaning | Log level of jni module | -| Value Range | same as debugFlag | -| Default Value | | +| Value Range | same as debugFlag | +| Default Value | | ### qDebugFlag -| Attribute | Description | -| -------- | -------------------- | +| Attribute | Description | +| ------------- | ------------------------- | | Applicable | Server and Client | -| Meaning | Log level of query module | -| Value Range | same as debugFlag | -| Default Value | | +| Meaning | Log level of query module | +| Value Range | same as debugFlag | +| Default Value | | ### cDebugFlag -| Attribute | Description | -| -------- | --------------------- | +| Attribute | Description | +| ------------- | ------------------- | | Applicable | Client Only | | Meaning | Log level of Client | -| Value Range | same as debugFlag | -| Default Value | | +| Value Range | same as debugFlag | +| Default Value | | ### dDebugFlag -| Attribute | Description | -| -------- | -------------------- | -| Applicable | Server Only | +| Attribute | Description | +| ------------- | ------------------ | +| Applicable | Server Only | | Meaning | Log level of dnode | | Value Range | same as debugFlag | -| Default Value | 135 | +| Default Value | 135 | ### vDebugFlag -| Attribute | Description | -| -------- | -------------------- | -| Applicable | Server Only | +| Attribute | Description | +| ------------- | ------------------ | +| Applicable | Server Only | | Meaning | Log level of vnode | | Value Range | same as debugFlag | -| Default Value | | +| Default Value | | ### mDebugFlag -| Attribute | Description | -| -------- | -------------------- | -| Applicable | Server Only | -| Meaning | Log level of mnode module | -| Value Range | same as debugFlag | -| Default Value | 135 | +| Attribute | Description | +| ------------- | ------------------------- | +| Applicable | Server Only | +| Meaning | Log level of mnode module | +| Value Range | same as debugFlag | +| Default Value | 135 | ### wDebugFlag -| Attribute | Description | -| -------- | ------------------ | -| Applicable | Server Only | -| Meaning | Log level of WAL module | -| Value Range | same as debugFlag | -| Default Value | 135 | +| Attribute | Description | +| ------------- | ----------------------- | +| Applicable | Server Only | +| Meaning | Log level of WAL module | +| Value Range | same as debugFlag | +| Default Value | 135 | ### sDebugFlag -| Attribute | Description | -| -------- | -------------------- | -| Applicable | Server and Client | +| Attribute | Description | +| ------------- | ------------------------ | +| Applicable | Server and Client | | Meaning | Log level of sync module | -| Value Range | same as debugFlag | -| Default Value | 135 | +| Value Range | same as debugFlag | +| Default Value | 135 | ### tsdbDebugFlag -| Attribute | Description | -| -------- | ------------------- | -| Applicable | Server Only | -| Meaning | Log level of TSDB module | -| Value Range | same as debugFlag | -| Default Value | | +| Attribute | Description | +| ------------- | ------------------------ | +| Applicable | Server Only | +| Meaning | Log level of TSDB module | +| Value Range | same as debugFlag | +| Default Value | | ### tqDebugFlag -| Attribute | Description | -| -------- | ----------------- | -| Applicable | Server only | -| Meaning | Log level of TQ module | -| Value Range | same as debugFlag | -| Default Value | | +| Attribute | Description | +| ------------- | ---------------------- | +| Applicable | Server only | +| Meaning | Log level of TQ module | +| Value Range | same as debugFlag | +| Default Value | | ### fsDebugFlag -| Attribute | Description | -| -------- | ----------------- | -| Applicable | Server only | -| Meaning | Log level of FS module | -| Value Range | same as debugFlag | -| Default Value | | +| Attribute | Description | +| ------------- | ---------------------- | +| Applicable | Server only | +| Meaning | Log level of FS module | +| Value Range | same as debugFlag | +| Default Value | | ### udfDebugFlag -| Attribute | Description | -| -------- | ------------------ | -| Applicable | Server Only | +| Attribute | Description | +| ------------- | ----------------------- | +| Applicable | Server Only | | Meaning | Log level of UDF module | -| Value Range | same as debugFlag | -| Default Value | | +| Value Range | same as debugFlag | +| Default Value | | ### smaDebugFlag -| Attribute | Description | -| -------- | ------------------ | -| Applicable | Server Only | -| Meaning | Log level of SMA module | -| Value Range | same as debugFlag | -| Default Value | | +| Attribute | Description | +| ------------- | ----------------------- | +| Applicable | Server Only | +| Meaning | Log level of SMA module | +| Value Range | same as debugFlag | +| Default Value | | ### idxDebugFlag -| Attribute | Description | -| -------- | -------------------- | -| Applicable | Server Only | -| Meaning | Log level of index module | -| Value Range | same as debugFlag | -| Default Value | | +| Attribute | Description | +| ------------- | ------------------------- | +| Applicable | Server Only | +| Meaning | Log level of index module | +| Value Range | same as debugFlag | +| Default Value | | ### tdbDebugFlag -| Attribute | Description | -| -------- | ------------------ | -| Applicable | Server Only | -| Meaning | Log level of TDB module | -| Value Range | same as debugFlag | -| Default Value | | +| Attribute | Description | +| ------------- | ----------------------- | +| Applicable | Server Only | +| Meaning | Log level of TDB module | +| Value Range | same as debugFlag | +| Default Value | | ## Schemaless Parameters ### smlChildTableName -| Attribute | Description | -| -------- | ------------------------- | -| Applicable | Client only | -| Meaning | Custom subtable name for schemaless writes | -| Type | String | -| Default Value | None | +| Attribute | Description | +| ------------- | ------------------------------------------ | +| Applicable | Client only | +| Meaning | Custom subtable name for schemaless writes | +| Type | String | +| Default Value | None | ### smlTagName -| Attribute | Description | -| -------- | ------------------------------------ | -| Applicable | Client only | -| Meaning | Default tag for schemaless writes without tag value specified | -| Type | String | -| Default Value | _tag_null | +| Attribute | Description | +| ------------- | ------------------------------------------------------------- | +| Applicable | Client only | +| Meaning | Default tag for schemaless writes without tag value specified | +| Type | String | +| Default Value | _tag_null | ### smlDataFormat -| Attribute | Description | -| -------- | ----------------------------- | -| Applicable | Client only | -| Meaning | Whether schemaless columns are consistently ordered, depat, discarded since 3.0.3.0| -| Value Range | 0: not consistent; 1: consistent. | -| Default | 0 | +| Attribute | Description | +| ----------- | ----------------------------------------------------------------------------------- | +| Applicable | Client only | +| Meaning | Whether schemaless columns are consistently ordered, depat, discarded since 3.0.3.0 | +| Value Range | 0: not consistent; 1: consistent. | +| Default | 0 | + +### smlTsDefaultName + +| Attribute | Description | +| -------- | -------------------------------------------------------- | +| Applicable | Client only | +| Meaning | The name of the time column for schemaless automatic table creation is set through this configuration | +| Type | String | +| Default Value | _ts | ## Compress Parameters ### compressMsgSize -| Attribute | Description | -| -------- | ----------------------------- | -| Applicable | Both Client and Server side | -| Meaning | Whether RPC message is compressed | -| Value Range | -1: none message is compressed; 0: all messages are compressed; N (N>0): messages exceeding N bytes are compressed | -| Default | -1 | +| Attribute | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------ | +| Applicable | Both Client and Server side | +| Meaning | Whether RPC message is compressed | +| Value Range | -1: none message is compressed; 0: all messages are compressed; N (N>0): messages exceeding N bytes are compressed | +| Default | -1 | ## Other Parameters ### enableCoreFile -| Attribute | Description | -| -------- | ------------------------------------------------------------------------------------------------------------------------------------------ | -| Applicable | Server and Client | -| Meaning | Whether to generate core file when server crashes | -| Value Range | 0: false, 1: true | -| Default Value | 1 | +| Attribute | Description | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Applicable | Server and Client | +| Meaning | Whether to generate core file when server crashes | +| Value Range | 0: false, 1: true | +| Default Value | 1 | | Note | The core file is generated under root directory `systemctl start taosd`/`launchctl start com.tdengine.taosd` is used to start, or under the working directory if `taosd` is started directly on Linux/macOS Shell. | ### enableScience -| Attribute | Description | -| -------- | ------------------------------------------------------------------------------------------------------------------------------------------ | -| Applicable | Only taos-CLI client | -| Meaning | Whether to show float and double with the scientific notation | -| Value Range | 0: false, 1: true | -| Default Value | 0 | +| Attribute | Description | +| ------------- | ------------------------------------------------------------- | +| Applicable | Only taos-CLI client | +| Meaning | Whether to show float and double with the scientific notation | +| Value Range | 0: false, 1: true | +| Default Value | 0 | ### udf -| Attribute | Description | -| -------- | ------------------ | -| Applicable | Server Only | -| Meaning | Whether the UDF service is enabled | -| Value Range | 0: disable UDF; 1: enabled UDF | -| Default Value | 1 | +| Attribute | Description | +| ------------- | ---------------------------------- | +| Applicable | Server Only | +| Meaning | Whether the UDF service is enabled | +| Value Range | 0: disable UDF; 1: enabled UDF | +| Default Value | 1 | +### ttlChangeOnWrite + +| Attribute | Description | +| ------------- | ----------------------------------------------------------------------------- | +| Applicable | Server Only | +| Meaning | Whether the ttl expiration time changes with the table modification operation | +| Value Range | 0: not change; 1: change by modification | +| Default Value | 0 | + +### keepTimeOffset + +| Attribute | Description | +| ------------- | ------------------------- | +| Applicable | Server Only | +| Meaning | Latency of data migration | +| Unit | hour | +| Value Range | 0-23 | +| Default Value | 0 | + +### tmqMaxTopicNum + +| Attribute | Description | +| -------- | ------------------ | +| Applicable | Server Only | +| Meaning | The max num of topics | +| Value Range | 1-10000| +| Default Value | 20 | ## 3.0 Parameters -| # | **Parameter** | **Applicable to 2.x ** | **Applicable to 3.0 ** | Current behavior in 3.0 | -| --- | :---------------------: | --------------- | --------------- | ------------------------------------------------- | -| 1 | firstEp | Yes | Yes | | -| 2 | secondEp | Yes | Yes | | -| 3 | fqdn | Yes | Yes | | -| 4 | serverPort | Yes | Yes | | -| 5 | maxShellConns | Yes | Yes | | -| 6 | monitor | Yes | Yes | | -| 7 | monitorFqdn | No | Yes | | -| 8 | monitorPort | No | Yes | | -| 9 | monitorInterval | Yes | Yes | | -| 10 | queryPolicy | No | Yes | | -| 11 | querySmaOptimize | No | Yes | | -| 12 | maxNumOfDistinctRes | Yes | Yes | | -| 15 | countAlwaysReturnValue | Yes | Yes | | -| 16 | dataDir | Yes | Yes | | -| 17 | minimalDataDirGB | Yes | Yes | | -| 18 | supportVnodes | No | Yes | | -| 19 | tempDir | Yes | Yes | | -| 20 | minimalTmpDirGB | Yes | Yes | | -| 21 | smlChildTableName | Yes | Yes | | -| 22 | smlTagName | Yes | Yes | | -| 23 | smlDataFormat | No | Yes(discarded since 3.0.3.0) | | -| 24 | statusInterval | Yes | Yes | | -| 25 | logDir | Yes | Yes | | -| 26 | minimalLogDirGB | Yes | Yes | | -| 27 | numOfLogLines | Yes | Yes | | -| 28 | asyncLog | Yes | Yes | | -| 29 | logKeepDays | Yes | Yes | | -| 30 | debugFlag | Yes | Yes | | -| 31 | tmrDebugFlag | Yes | Yes | | -| 32 | uDebugFlag | Yes | Yes | | -| 33 | rpcDebugFlag | Yes | Yes | | -| 34 | jniDebugFlag | Yes | Yes | | -| 35 | qDebugFlag | Yes | Yes | | -| 36 | cDebugFlag | Yes | Yes | | -| 37 | dDebugFlag | Yes | Yes | | -| 38 | vDebugFlag | Yes | Yes | | -| 39 | mDebugFlag | Yes | Yes | | -| 40 | wDebugFlag | Yes | Yes | | -| 41 | sDebugFlag | Yes | Yes | | -| 42 | tsdbDebugFlag | Yes | Yes | | -| 43 | tqDebugFlag | No | Yes | | -| 44 | fsDebugFlag | Yes | Yes | | -| 45 | udfDebugFlag | No | Yes | | -| 46 | smaDebugFlag | No | Yes | | -| 47 | idxDebugFlag | No | Yes | | -| 48 | tdbDebugFlag | No | Yes | | -| 49 | metaDebugFlag | No | Yes | | -| 50 | timezone | Yes | Yes | | -| 51 | locale | Yes | Yes | | -| 52 | charset | Yes | Yes | | -| 53 | udf | Yes | Yes | | -| 54 | enableCoreFile | Yes | Yes | | +| # | **Parameter** | **Applicable to 2.x ** | **Applicable to 3.0 ** | Current behavior in 3.0 | +| --- | :--------------------: | ---------------------- | ---------------------------- | ----------------------- | +| 1 | firstEp | Yes | Yes | | +| 2 | secondEp | Yes | Yes | | +| 3 | fqdn | Yes | Yes | | +| 4 | serverPort | Yes | Yes | | +| 5 | maxShellConns | Yes | Yes | | +| 6 | monitor | Yes | Yes | | +| 7 | monitorFqdn | No | Yes | | +| 8 | monitorPort | No | Yes | | +| 9 | monitorInterval | Yes | Yes | | +| 10 | queryPolicy | No | Yes | | +| 11 | querySmaOptimize | No | Yes | | +| 12 | maxNumOfDistinctRes | Yes | Yes | | +| 15 | countAlwaysReturnValue | Yes | Yes | | +| 16 | dataDir | Yes | Yes | | +| 17 | minimalDataDirGB | Yes | Yes | | +| 18 | supportVnodes | No | Yes | | +| 19 | tempDir | Yes | Yes | | +| 20 | minimalTmpDirGB | Yes | Yes | | +| 21 | smlChildTableName | Yes | Yes | | +| 22 | smlTagName | Yes | Yes | | +| 23 | smlDataFormat | No | Yes(discarded since 3.0.3.0) | | +| 24 | statusInterval | Yes | Yes | | +| 25 | logDir | Yes | Yes | | +| 26 | minimalLogDirGB | Yes | Yes | | +| 27 | numOfLogLines | Yes | Yes | | +| 28 | asyncLog | Yes | Yes | | +| 29 | logKeepDays | Yes | Yes | | +| 30 | debugFlag | Yes | Yes | | +| 31 | tmrDebugFlag | Yes | Yes | | +| 32 | uDebugFlag | Yes | Yes | | +| 33 | rpcDebugFlag | Yes | Yes | | +| 34 | jniDebugFlag | Yes | Yes | | +| 35 | qDebugFlag | Yes | Yes | | +| 36 | cDebugFlag | Yes | Yes | | +| 37 | dDebugFlag | Yes | Yes | | +| 38 | vDebugFlag | Yes | Yes | | +| 39 | mDebugFlag | Yes | Yes | | +| 40 | wDebugFlag | Yes | Yes | | +| 41 | sDebugFlag | Yes | Yes | | +| 42 | tsdbDebugFlag | Yes | Yes | | +| 43 | tqDebugFlag | No | Yes | | +| 44 | fsDebugFlag | Yes | Yes | | +| 45 | udfDebugFlag | No | Yes | | +| 46 | smaDebugFlag | No | Yes | | +| 47 | idxDebugFlag | No | Yes | | +| 48 | tdbDebugFlag | No | Yes | | +| 49 | metaDebugFlag | No | Yes | | +| 50 | timezone | Yes | Yes | | +| 51 | locale | Yes | Yes | | +| 52 | charset | Yes | Yes | | +| 53 | udf | Yes | Yes | | +| 54 | enableCoreFile | Yes | Yes | | +| 55 | ttlChangeOnWrite | No | Yes | | +| 56 | keepTimeOffset | Yes | Yes | | diff --git a/docs/en/14-reference/13-schemaless/13-schemaless.md b/docs/en/14-reference/13-schemaless/13-schemaless.md index aad0e63a4228ca303302d4a3970182355f750d53..54be18eea32d7e5c08e406ce2a7809f20b4bb0a4 100644 --- a/docs/en/14-reference/13-schemaless/13-schemaless.md +++ b/docs/en/14-reference/13-schemaless/13-schemaless.md @@ -34,7 +34,27 @@ In the schemaless writing data line protocol, each data item in the field_set ne - If there are English double quotes on both sides, it indicates the BINARY(32) type. For example, `"abc"`. - If there are double quotes on both sides and an L prefix, it means NCHAR(32) type. For example, `L"error message"`. -- Spaces, equal signs (=), commas (,), and double quotes (") need to be escaped with a backslash (\\) in front. (All refer to the ASCII character) +- Spaces, equals sign (=), comma (,), double quote ("), and backslash (\\) need to be escaped with a backslash (\\) in front. (All refer to the ASCII character). The rules are as follows: + +| **Serial number** | **Element** | **Escape characters** | +| -------- | ----------- | ----------------------------- | +| 1 | Measurement | Comma, Space | +| 2 | Tag key | Comma, Equals Sign, Space | +| 3 | Tag value | Comma, Equals Sign, Space | +| 4 | Field key | Comma, Equals Sign, Space | +| 5 | Field value | Double quote, Backslash | + +With two contiguous backslashes, the first is interpreted as an escape character. Examples of backslash escape rules are as follows: + +| **Serial number** | **Backslashes** | **Interpreted as** | +| -------- | ----------- | ----------------------------- | +| 1 | \ | \ | +| 2 | \\\\ | \ | +| 3 | \\\\\\ | \\\\ | +| 4 | \\\\\\\\ | \\\\ | +| 5 | \\\\\\\\\\ | \\\\\\ | +| 6 | \\\\\\\\\\\\ | \\\\\\ | + - Numeric types will be distinguished from data types by the suffix. | **Serial number** | **Postfix** | **Mapping type** | **Size (bytes)** | @@ -88,9 +108,11 @@ You can configure smlChildTableName in taos.cfg to specify table names, for exam 8. It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur. Note: TDengine 3.0.3.0 and later automatically detect whether order is consistent. This parameter is no longer used. +9. Due to the fact that SQL table names do not support period (.), schemaless has also processed period (.). If there is a period (.) in the table name automatically created by schemaless, it will be automatically replaced with an underscore (\_). If you manually specify a sub table name, if there is a dot (.) in the sub table name, it will also be converted to an underscore (\_) +10. Taos.cfg adds the configuration of smlTsDefaultName (with a string value), which only works on the client side. After configuration, the time column name of the schemaless automatic table creation can be set through this configuration. If not configured, defaults to _ts. :::tip -All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed 48 KB and the total length of a tag value cannot exceed 16 KB. See [TDengine SQL Boundary Limits](/taos-sql/limit) for specific constraints in this area. +All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed 48 KB(64 KB since version 3.0.5.0) and the total length of a tag value cannot exceed 16 KB. See [TDengine SQL Boundary Limits](/taos-sql/limit) for specific constraints in this area. ::: ## Time resolution recognition diff --git a/docs/en/20-third-party/11-kafka.md b/docs/en/20-third-party/11-kafka.md index 3b0de6c3499c20a6c2cca19623101009493216fd..a98c3e3a6ba10747675ddcedfc5a2697aae040c4 100644 --- a/docs/en/20-third-party/11-kafka.md +++ b/docs/en/20-third-party/11-kafka.md @@ -16,165 +16,79 @@ TDengine Source Connector is used to read data from TDengine in real-time and se ![TDengine Database Kafka Connector -- streaming integration with kafka connect](kafka/streaming-integration-with-kafka-connect.webp) -## What is Confluent? - -[Confluent](https://www.confluent.io/) adds many extensions to Kafka. include: - -1. Schema Registry -2. REST Proxy -3. Non-Java Clients -4. Many packaged Kafka Connect plugins -5. GUI for managing and monitoring Kafka - Confluent Control Center - -Some of these extensions are available in the community version of Confluent. Some are only available in the enterprise version. -![TDengine Database Kafka Connector -- Confluent platform](kafka/confluentPlatform.webp) - -Confluent Enterprise Edition provides the `confluent` command-line tool to manage various components. - ## Prerequisites 1. Linux operating system 2. Java 8 and Maven installed -3. Git is installed +3. Git/curl/vi is installed 4. TDengine is installed and started. If not, please refer to [Installation and Uninstallation](/operation/pkg-install) -## Install Confluent - -Confluent provides two installation methods: Docker and binary packages. This article only introduces binary package installation. +## Install Kafka Execute in any directory: -```` -curl -O http://packages.confluent.io/archive/7.1/confluent-7.1.1.tar.gz -tar xzf confluent-7.1.1.tar.gz -C /opt/ -```` +```shell +curl -O https://downloads.apache.org/kafka/3.4.0/kafka_2.13-3.4.0.tgz +tar xzf kafka_2.13-3.4.0.tgz -C /opt/ +ln -s /opt/kafka_2.13-3.4.0 /opt/kafka +``` -Then you need to add the `$CONFLUENT_HOME/bin` directory to the PATH. +Then you need to add the `$KAFKA_HOME/bin` directory to the PATH. ```title=".profile" -export CONFLUENT_HOME=/opt/confluent-7.1.1 -export PATH=$CONFLUENT_HOME/bin:$PATH +export KAFKA_HOME=/opt/kafka +export PATH=$PATH:$KAFKA_HOME/bin ``` Users can append the above script to the current user's profile file (~/.profile or ~/.bash_profile) -After the installation is complete, you can enter `confluent version` for simple verification: - -``` -# confluent version -confluent - Confluent CLI - -Version: v2.6.1 -Git Ref: 6d920590 -Build Date: 2022-02-18T06:14:21Z -Go Version: go1.17.6 (linux/amd64) -Development: false -``` - ## Install TDengine Connector plugin ### Install from source code -``` +```shell git clone --branch 3.0 https://github.com/taosdata/kafka-connect-tdengine.git cd kafka-connect-tdengine -mvn clean package -unzip -d $CONFLUENT_HOME/share/java/ target/components/packages/taosdata-kafka-connect-tdengine-*.zip +mvn clean package -Dmaven.test.skip=true +unzip -d $KAFKA_HOME/components/ target/components/packages/taosdata-kafka-connect-tdengine-*.zip ``` -The above script first clones the project source code and then compiles and packages it with Maven. After the package is complete, the zip package of the plugin is generated in the `target/components/packages/` directory. Unzip this zip package to plugin path. We used `$CONFLUENT_HOME/share/java/` above because it's a build in plugin path. - -### Install with confluent-hub +The above script first clones the project source code and then compiles and packages it with Maven. After the package is complete, the zip package of the plugin is generated in the `target/components/packages/` directory. Unzip this zip package to plugin path. We used `$KAFKA_HOME/components/` above because it's a build in plugin path. -[Confluent Hub](https://www.confluent.io/hub) provides a service to download Kafka Connect plugins. After TDengine Kafka Connector is published to Confluent Hub, it can be installed using the command tool `confluent-hub`. -**TDengine Kafka Connector is currently not officially released and cannot be installed in this way**. +### Add configuration file -## Start Confluent +add kafka-connect-tdengine plugin path to `plugin.path` in `$KAFKA_HOME/config/connect-distributed.properties`. +```properties +plugin.path=/usr/share/java,/opt/kafka/components ``` -confluent local services start -``` - -:::note -Be sure to install the plugin before starting Confluent. Otherwise, Kafka Connect will fail to discover the plugins. -::: -:::tip -If a component fails to start, try clearing the data and restarting. The data directory will be printed to the console at startup, e.g.: - -```title="Console output log" {1} -Using CONFLUENT_CURRENT: /tmp/confluent.106668 -Starting ZooKeeper -ZooKeeper is [UP] -Starting Kafka -Kafka is [UP] -Starting Schema Registry -Schema Registry is [UP] -Starting Kafka REST -Kafka REST is [UP] -Starting Connect -Connect is [UP] -Starting ksqlDB Server -ksqlDB Server is [UP] -Starting Control Center -Control Center is [UP] -``` +## Start Kafka Services -To clear data, execute `rm -rf /tmp/confluent.106668`. -::: +Use command bellow to start all services: -### Check Confluent Services Status +```shell +zookeeper-server-start.sh -daemon $KAFKA_HOME/config/zookeeper.properties -Use command bellow to check the status of all service: +kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties -``` -confluent local services status -``` +connect-distributed.sh -daemon $KAFKA_HOME/config/connect-distributed.properties -The expected output is: -``` -Connect is [UP] -Control Center is [UP] -Kafka is [UP] -Kafka REST is [UP] -ksqlDB Server is [UP] -Schema Registry is [UP] -ZooKeeper is [UP] ``` ### Check Successfully Loaded Plugin After Kafka Connect was completely started, you can use bellow command to check if our plugins are installed successfully: -``` -confluent local services connect plugin list -``` - -The output should contains `TDengineSinkConnector` and `TDengineSourceConnector` as bellow: -``` -Available Connect Plugins: -[ - { - "class": "com.taosdata.kafka.connect.sink.TDengineSinkConnector", - "type": "sink", - "version": "1.0.0" - }, - { - "class": "com.taosdata.kafka.connect.source.TDengineSourceConnector", - "type": "source", - "version": "1.0.0" - }, -...... +```shell +curl http://localhost:8083/connectors ``` -If not, please check the log file of Kafka Connect. To view the log file path, please execute: +The output as bellow: +```txt +[] ``` -echo `cat /tmp/confluent.current`/connect/connect.stdout -``` -It should produce a path like:`/tmp/confluent.104086/connect/connect.stdout` - -Besides log file `connect.stdout` there is a file named `connect.properties`. At the end of this file you can see the effective `plugin.path` which is a series of paths joined by comma. If Kafka Connect not found our plugins, it's probably because the installed path is not included in `plugin.path`. ## The use of TDengine Sink Connector @@ -184,40 +98,47 @@ TDengine Sink Connector internally uses TDengine [modeless write interface](/ref The following example synchronizes the data of the topic meters to the target database power. The data format is the InfluxDB Line protocol format. -### Add configuration file +### Add Sink Connector configuration file -``` +```shell mkdir ~/test cd ~/test -vi sink-demo.properties +vi sink-demo.json ``` -sink-demo.properties' content is following: - -```ini title="sink-demo.properties" -name=TDengineSinkConnector -connector.class=com.taosdata.kafka.connect.sink.TDengineSinkConnector -tasks.max=1 -topics=meters -connection.url=jdbc:TAOS://127.0.0.1:6030 -connection.user=root -connection.password=taosdata -connection.database=power -db.schemaless=line -data.precision=ns -key.converter=org.apache.kafka.connect.storage.StringConverter -value.converter=org.apache.kafka.connect.storage.StringConverter +sink-demo.json' content is following: + +```json title="sink-demo.json" +{ + "name": "TDengineSinkConnector", + "config": { + "connector.class":"com.taosdata.kafka.connect.sink.TDengineSinkConnector", + "tasks.max": "1", + "topics": "meters", + "connection.url": "jdbc:TAOS://127.0.0.1:6030", + "connection.user": "root", + "connection.password": "taosdata", + "connection.database": "power", + "db.schemaless": "line", + "data.precision": "ns", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.storage.StringConverter", + "errors.tolerance": "all", + "errors.deadletterqueue.topic.name": "dead_letter_topic", + "errors.deadletterqueue.topic.replication.factor": 1 + } +} ``` Key configuration instructions: -1. `topics=meters` and `connection.database=power` means to subscribe to the data of the topic meters and write to the database power. -2. `db.schemaless=line` means the data in the InfluxDB Line protocol format. +1. `"topics": "meters"` and `"connection.database": "power"` means to subscribe to the data of the topic meters and write to the database power. +2. `"db.schemaless": "line"` means the data in the InfluxDB Line protocol format. -### Create Connector instance +### Create Sink Connector instance -```` -confluent local services connect connector load TDengineSinkConnector --config ./sink-demo.properties +````shell +curl -X POST -d @sink-demo.json http://localhost:8083/connectors -H "Content-Type: application/json" ```` If the above command is executed successfully, the output is as follows: @@ -237,7 +158,10 @@ If the above command is executed successfully, the output is as follows: "tasks.max": "1", "topics": "meters", "value.converter": "org.apache.kafka.connect.storage.StringConverter", - "name": "TDengineSinkConnector" + "name": "TDengineSinkConnector", + "errors.tolerance": "all", + "errors.deadletterqueue.topic.name": "dead_letter_topic", + "errors.deadletterqueue.topic.replication.factor": "1", }, "tasks": [], "type": "sink" @@ -257,8 +181,8 @@ meters,location=California.LoSangeles,groupid=3 current=11.3,voltage=221,phase=0 Use kafka-console-producer to write test data to the topic `meters`. -``` -cat test-data.txt | kafka-console-producer --broker-list localhost:9092 --topic meters +```shell +cat test-data.txt | kafka-console-producer.sh --broker-list localhost:9092 --topic meters ``` :::note @@ -269,12 +193,12 @@ TDengine Sink Connector will automatically create the database if the target dat Use the TDengine CLI to verify that the sync was successful. -``` +```sql taos> use power; Database changed. taos> select * from meters; - ts | current | voltage | phase | groupid | location | + _ts | current | voltage | phase | groupid | location | =============================================================================================================================================================== 2022-03-28 09:56:51.249000000 | 11.800000000 | 221.000000000 | 0.280000000 | 2 | California.LosAngeles | 2022-03-28 09:56:51.250000000 | 13.400000000 | 223.000000000 | 0.290000000 | 2 | California.LosAngeles | @@ -291,32 +215,39 @@ The role of the TDengine Source Connector is to push all the data of a specific TDengine Source Connector will convert the data in TDengine data table into [InfluxDB Line protocol format](/develop/insert-data/influxdb-line/) or [OpenTSDB JSON protocol format](/develop/insert-data/opentsdb-json ) and then write to Kafka. -The following sample program synchronizes the data in the database test to the topic tdengine-source-test. +The following sample program synchronizes the data in the database test to the topic tdengine-test-meters. -### Add configuration file +### Add Source Connector configuration file -``` -vi source-demo.properties +```shell +vi source-demo.json ``` Input following content: -```ini title="source-demo.properties" -name=TDengineSourceConnector -connector.class=com.taosdata.kafka.connect.source.TDengineSourceConnector -tasks.max=1 -connection.url=jdbc:TAOS://127.0.0.1:6030 -connection.username=root -connection.password=taosdata -connection.database=test -connection.attempts=3 -connection.backoff.ms=5000 -topic.prefix=tdengine-source- -poll.interval.ms=1000 -fetch.max.rows=100 -out.format=line -key.converter=org.apache.kafka.connect.storage.StringConverter -value.converter=org.apache.kafka.connect.storage.StringConverter +```json title="source-demo.json" +{ + "name":"TDengineSourceConnector", + "config":{ + "connector.class": "com.taosdata.kafka.connect.source.TDengineSourceConnector", + "tasks.max": 1, + "connection.url": "jdbc:TAOS://127.0.0.1:6030", + "connection.username": "root", + "connection.password": "taosdata", + "connection.database": "test", + "connection.attempts": 3, + "connection.backoff.ms": 5000, + "topic.prefix": "tdengine", + "topic.delimiter": "-", + "poll.interval.ms": 1000, + "fetch.max.rows": 100, + "topic.per.stable": true, + "topic.ignore.db": false, + "out.format": "line", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.storage.StringConverter" + } +} ``` ### Prepare test data @@ -341,40 +272,40 @@ INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES('2018-1 Use TDengine CLI to execute SQL script -``` +```shell taos -f prepare-source-data.sql ``` ### Create Connector instance -```` -confluent local services connect connector load TDengineSourceConnector --config source-demo.properties -```` +```shell +curl -X POST -d @source-demo.json http://localhost:8083/connectors -H "Content-Type: application/json" +``` ### View topic data -Use the kafka-console-consumer command-line tool to monitor data in the topic tdengine-source-test. In the beginning, all historical data will be output. After inserting two new data into TDengine, kafka-console-consumer immediately outputs the two new data. +Use the kafka-console-consumer command-line tool to monitor data in the topic tdengine-test-meters. In the beginning, all historical data will be output. After inserting two new data into TDengine, kafka-console-consumer immediately outputs the two new data. The output is in InfluxDB line protocol format. -```` -kafka-console-consumer --bootstrap-server localhost:9092 --from-beginning --topic tdengine-source-test +````shell +kafka-console-consumer.sh --bootstrap-server localhost:9092 --from-beginning --topic tdengine-test-meters ```` output: -```` +```txt ...... meters,location="California.SanFrancisco",groupid=2i32 current=10.3f32,voltage=219i32,phase=0.31f32 1538548685000000000 meters,location="California.SanFrancisco",groupid=2i32 current=12.6f32,voltage=218i32,phase=0.33f32 1538548695000000000 ...... -```` +``` All historical data is displayed. Switch to the TDengine CLI and insert two new pieces of data: -```` +```sql USE test; INSERT INTO d1001 VALUES (now, 13.3, 229, 0.38); INSERT INTO d1002 VALUES (now, 16.3, 233, 0.22); -```` +``` Switch back to kafka-console-consumer, and the command line window has printed out the two pieces of data just inserted. @@ -384,16 +315,16 @@ After testing, use the unload command to stop the loaded connector. View currently active connectors: -```` -confluent local services connect connector status -```` +```shell +curl http://localhost:8083/connectors +``` You should now have two active connectors if you followed the previous steps. Use the following command to unload: -```` -confluent local services connect connector unload TDengineSinkConnector -confluent local services connect connector unload TDengineSourceConnector -```` +```shell +curl -X DELETE http://localhost:8083/connectors/TDengineSinkConnector +curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector +``` ## Configuration reference @@ -424,24 +355,27 @@ The following configuration items apply to TDengine Sink Connector and TDengine ### TDengine Source Connector specific configuration 1. `connection.database`: source database name, no default value. -2. `topic.prefix`: topic name prefix after data is imported into kafka. Use `topic.prefix` + `connection.database` name as the full topic name. Defaults to the empty string "". -3. `timestamp.initial`: Data synchronization start time. The format is 'yyyy-MM-dd HH:mm:ss'. Default "1970-01-01 00:00:00". -4. `poll.interval.ms`: Pull data interval, the unit is ms. Default is 1000. -5. `fetch.max.rows`: The maximum number of rows retrieved when retrieving the database. Default is 100. -6. `out.format`: The data format. The value could be line or json. The line represents the InfluxDB Line protocol format, and json represents the OpenTSDB JSON format. Default is `line`. - +2. `topic.prefix`: topic name prefix used when importing data into kafka. Its defaults value is empty string "". +3. `timestamp.initial`: Data synchronization start time. The format is 'yyyy-MM-dd HH:mm:ss'. If it is not set, the data importing to Kafka will be started from the first/oldest row in the database. +4. `poll.interval.ms`: The time interval for checking newly created tables or removed tables, default value is 1000. +5. `fetch.max.rows`: The maximum number of rows retrieved when retrieving the database, default is 100. +6. `query.interval.ms`: The time range of reading data from TDengine each time, its unit is millisecond. It should be adjusted according to the data flow in rate, the default value is 0, this means to get all the data to the latest time. +7. `out.format`: Result output format. `line` indicates that the output format is InfluxDB line protocol format, `json` indicates that the output format is json. The default is line. +8. `topic.per.stable`: If it's set to true, it means one super table in TDengine corresponds to a topic in Kafka, the topic naming rule is ``; if it's set to false, it means the whole DB corresponds to a topic in Kafka, the topic naming rule is ``. +9. `topic.ignore.db`: Whether the topic naming rule contains the database name: true indicates that the rule is ``, false indicates that the rule is ``, and the default is false. Does not take effect when `topic.per.stable` is set to false. +10. `topic.delimiter`: topic name delimiter，default is `-`. +11. `read.method`: read method for query TDengine data, query or subscription. default is subscription. +12. `subscription.group.id`: subscription group id for subscription data from TDengine, this field is required when `read.method` is subscription. +13. `subscription.from`: subscription from latest or earliest. default is latest。 ## Other notes -1. To install plugin to a customized location, refer to https://docs.confluent.io/home/connect/self-managed/install.html#install-connector-manually. -2. To use Kafka Connect without confluent, refer to https://kafka.apache.org/documentation/#connect. +1. To use Kafka Connect, refer to . ## Feedback -https://github.com/taosdata/kafka-connect-tdengine/issues + ## Reference -1. https://www.confluent.io/what-is-apache-kafka -2. https://developer.confluent.io/learn-kafka/kafka-connect/intro -3. https://docs.confluent.io/platform/current/platform.html +1. For more information, see diff --git a/docs/en/20-third-party/14-dbeaver.md b/docs/en/20-third-party/14-dbeaver.md new file mode 100644 index 0000000000000000000000000000000000000000..fd0a0672f25b2b20ef890c30dae6c53e8bd497d5 --- /dev/null +++ b/docs/en/20-third-party/14-dbeaver.md @@ -0,0 +1,36 @@ +--- +sidebar_label: DBeaver +title: DBeaver +description: You can use DBeaver to access your data stored in TDengine and TDengine Cloud. +--- + +[DBeaver](https://dbeaver.io/) is a popular cross-platform database management tool that facilitates data management for developers, database administrators, data analysts, and other users. Starting from version 23.1.1, DBeaver natively supports TDengine and can be used to manage TDengine Cloud as well as TDengine clusters deployed on-premises. + +## Prerequisites + +To use DBeaver to manage TDengine, you need to prepare the following: + +- Install DBeaver. DBeaver supports mainstream operating systems including Windows, macOS, and Linux. Please make sure you download and install the correct version (23.1.1+) and platform package. Please refer to the [official DBeaver documentation](https://github.com/dbeaver/dbeaver/wiki/Installation) for detailed installation steps. +- If you use an on-premises TDengine cluster, please make sure that TDengine and taosAdapter are deployed and running properly. For detailed information, please refer to the taosAdapter User Manual. + +## Use DBeaver to access on-premises TDengine cluster + +1. Start the DBeaver application, click the button or menu item to choose **New Database Connection**, and then select **TDengine** in the **Timeseries** category. + + ![Connect TDengine with DBeaver](./dbeaver/dbeaver-connect-tdengine-en.webp) + +2. Configure the TDengine connection by filling in the host address, port number, username, and password. If TDengine is deployed on the local machine, you are only required to fill in the username and password. The default username is root and the default password is taosdata. Click **Test Connection** to check whether the connection is workable. If you do not have the TDengine Java connector installed on the local machine, DBeaver will prompt you to download and install it. + + ![Configure the TDengine connection](./dbeaver/dbeaver-config-tdengine-en.webp)) + +3. If the connection is successful, it will be displayed as shown in the following figure. If the connection fails, please check whether the TDengine service and taosAdapter are running correctly and whether the host address, port number, username, and password are correct. + + ![Connection successful](./dbeaver/dbeaver-connect-tdengine-test-en.webp) + +4. Use DBeaver to select databases and tables and browse your data stored in TDengine. + + ![Browse TDengine data with DBeaver](./dbeaver/dbeaver-browse-data-en.webp) + +5. You can also manipulate TDengine data by executing SQL commands. + + ![Use SQL commands to manipulate TDengine data in DBeaver](./dbeaver/dbeaver-sql-execution-en.webp) diff --git a/docs/en/20-third-party/50-qstudio.md b/docs/en/20-third-party/50-qstudio.md new file mode 100644 index 0000000000000000000000000000000000000000..8339678a0e18da6fc38cc048ac152f3cd7bee04b --- /dev/null +++ b/docs/en/20-third-party/50-qstudio.md @@ -0,0 +1,40 @@ +--- +sidebar_label: qStudio +title: qStudio +description: Step-by-Step Guide to Accessing TDengine Data with qStudio +--- + +qStudio is a free cross-platform SQL data analysis tool that allows easy browsing of tables, variables, functions, and configuration settings in a database. The latest version of qStudio includes built-in support for TDengine. + +## Prerequisites + +To connect TDengine using qStudio, you need to complete the following preparations: + +- Install qStudio: qStudio supports major operating systems, including Windows, macOS, and Linux. Please ensure you download the correct installation package for your platform from the [download page](https://www.timestored.com/qstudio/download/). +- Set up TDengine instance: Make sure TDengine is installed and running correctly, and the taosAdapter is installed and running. For detailed information, refer to the taosAdapter User Manual. + +## Connecting to TDengine with qStudio + +1. Launch the qStudio application and select "Server" and then "Add Server..." from the menu. Choose TDengine from the Server Type dropdown. + + ![qConnecting TDengine with qStudio](./qstudio/qstudio-connect-tdengine.webp) + +2. Configure the TDengine connection by entering the host address, port number, username, and password. If TDengine is deployed on the local machine, you can fill in the username and password only. The default username is "root," and the default password is "taosdata." Click "Test" to test the connection's availability. If the TDengine Java connector is not installed on the local machine, qStudio will prompt you to download and install it. + + ![Download Java Connector](./qstudio/qstudio-jdbc-connector-download.webp) + +3. Once connected successfully, the screen will display as shown below. If the connection fails, check that the TDengine service and taosAdapter are running correctly, and ensure that the host address, port number, username, and password are correct. + + ![Successful Connection](./qstudio/qstudio-connect-tdengine-test.webp) + +4. Use qStudio to select databases and tables to browse data from the TDengine server. + + ![Browsing TDengine Data with qStudio](./qstudio/qstudio-browse-data.webp) + +5. You can also perform operations on TDengine data by executing SQL commands. + + ![qStudio SQL Commands](./qstudio/qstudio-sql-execution.webp) + +6. qStudio supports charting functions based on the data. For more information, please refer to the [qStudio documentation](https://www.timestored.com/qstudio/help). + + ![qStudio Chart](./qstudio/qstudio-chart.webp) diff --git a/docs/en/20-third-party/dbeaver/dbeaver-browse-data-cloud-en.webp b/docs/en/20-third-party/dbeaver/dbeaver-browse-data-cloud-en.webp new file mode 100644 index 0000000000000000000000000000000000000000..84424284119ed688e8a5657e2d1be373fc12485e Binary files /dev/null and b/docs/en/20-third-party/dbeaver/dbeaver-browse-data-cloud-en.webp differ diff --git a/docs/en/20-third-party/dbeaver/dbeaver-browse-data-en.webp b/docs/en/20-third-party/dbeaver/dbeaver-browse-data-en.webp new file mode 100644 index 0000000000000000000000000000000000000000..bb2f05a9a1306794601e9d3a537bbb27e7e88823 Binary files /dev/null and b/docs/en/20-third-party/dbeaver/dbeaver-browse-data-en.webp differ diff --git a/docs/en/20-third-party/dbeaver/dbeaver-config-tdengine-en.webp b/docs/en/20-third-party/dbeaver/dbeaver-config-tdengine-en.webp new file mode 100644 index 0000000000000000000000000000000000000000..ff59e96029854b5a8c89460829dda62db8cc1a0a Binary files /dev/null and b/docs/en/20-third-party/dbeaver/dbeaver-config-tdengine-en.webp differ diff --git a/docs/en/20-third-party/dbeaver/dbeaver-connect-tdengine-cloud-en.webp b/docs/en/20-third-party/dbeaver/dbeaver-connect-tdengine-cloud-en.webp new file mode 100644 index 0000000000000000000000000000000000000000..56b2938c9b9cf44f5f03cc0ea2dbeab5843d9086 Binary files /dev/null and b/docs/en/20-third-party/dbeaver/dbeaver-connect-tdengine-cloud-en.webp differ diff --git a/docs/en/20-third-party/dbeaver/dbeaver-connect-tdengine-cloud-test-en.webp b/docs/en/20-third-party/dbeaver/dbeaver-connect-tdengine-cloud-test-en.webp new file mode 100644 index 0000000000000000000000000000000000000000..c15228c8d34426879f8c7198bf8603b7483fff1d Binary files /dev/null and b/docs/en/20-third-party/dbeaver/dbeaver-connect-tdengine-cloud-test-en.webp differ diff --git a/docs/en/20-third-party/dbeaver/dbeaver-connect-tdengine-en.webp b/docs/en/20-third-party/dbeaver/dbeaver-connect-tdengine-en.webp new file mode 100644 index 0000000000000000000000000000000000000000..9dc8286773e5dc0fa8f2d92763dbbcb4e85eecac Binary files /dev/null and b/docs/en/20-third-party/dbeaver/dbeaver-connect-tdengine-en.webp differ diff --git a/docs/en/20-third-party/dbeaver/dbeaver-connect-tdengine-test-en.webp b/docs/en/20-third-party/dbeaver/dbeaver-connect-tdengine-test-en.webp new file mode 100644 index 0000000000000000000000000000000000000000..d22d02e82047d97801a3669fedb22d84191b77ca Binary files /dev/null and b/docs/en/20-third-party/dbeaver/dbeaver-connect-tdengine-test-en.webp differ diff --git a/docs/en/20-third-party/dbeaver/dbeaver-sql-execution-cloud-en.webp b/docs/en/20-third-party/dbeaver/dbeaver-sql-execution-cloud-en.webp new file mode 100644 index 0000000000000000000000000000000000000000..6938c970ac97577798ce7dfe6443f144df54ce9e Binary files /dev/null and b/docs/en/20-third-party/dbeaver/dbeaver-sql-execution-cloud-en.webp differ diff --git a/docs/en/20-third-party/dbeaver/dbeaver-sql-execution-en.webp b/docs/en/20-third-party/dbeaver/dbeaver-sql-execution-en.webp new file mode 100644 index 0000000000000000000000000000000000000000..1f3fc19571f242db918da525ed7da9120847cab3 Binary files /dev/null and b/docs/en/20-third-party/dbeaver/dbeaver-sql-execution-en.webp differ diff --git a/docs/en/20-third-party/dbeaver/tdengine-cloud-jdbc-dsn-en.webp b/docs/en/20-third-party/dbeaver/tdengine-cloud-jdbc-dsn-en.webp new file mode 100644 index 0000000000000000000000000000000000000000..7a8ed8f6d3ab1de3051137c7880f641640746c2f Binary files /dev/null and b/docs/en/20-third-party/dbeaver/tdengine-cloud-jdbc-dsn-en.webp differ diff --git a/docs/en/20-third-party/qstudio/qstudio-browse-data.webp b/docs/en/20-third-party/qstudio/qstudio-browse-data.webp new file mode 100644 index 0000000000000000000000000000000000000000..0de3a569f74615db5a9220d810244382d280fb61 Binary files /dev/null and b/docs/en/20-third-party/qstudio/qstudio-browse-data.webp differ diff --git a/docs/en/20-third-party/qstudio/qstudio-chart.webp b/docs/en/20-third-party/qstudio/qstudio-chart.webp new file mode 100644 index 0000000000000000000000000000000000000000..397d525cd292c5fecf7f90ea1de6c01d82a788e9 Binary files /dev/null and b/docs/en/20-third-party/qstudio/qstudio-chart.webp differ diff --git a/docs/en/20-third-party/qstudio/qstudio-connect-tdengine-test.webp b/docs/en/20-third-party/qstudio/qstudio-connect-tdengine-test.webp new file mode 100644 index 0000000000000000000000000000000000000000..c488dfba5083c9be363387ad6a83c0bcdf590eb7 Binary files /dev/null and b/docs/en/20-third-party/qstudio/qstudio-connect-tdengine-test.webp differ diff --git a/docs/en/20-third-party/qstudio/qstudio-connect-tdengine.webp b/docs/en/20-third-party/qstudio/qstudio-connect-tdengine.webp new file mode 100644 index 0000000000000000000000000000000000000000..c9805d3699c716b2e0a6efa4a17aeee124e63a01 Binary files /dev/null and b/docs/en/20-third-party/qstudio/qstudio-connect-tdengine.webp differ diff --git a/docs/en/20-third-party/qstudio/qstudio-jdbc-connector-download.webp b/docs/en/20-third-party/qstudio/qstudio-jdbc-connector-download.webp new file mode 100644 index 0000000000000000000000000000000000000000..1ae7c73963aad68d2d0ebc86072d50e013100a50 Binary files /dev/null and b/docs/en/20-third-party/qstudio/qstudio-jdbc-connector-download.webp differ diff --git a/docs/en/20-third-party/qstudio/qstudio-sql-execution.webp b/docs/en/20-third-party/qstudio/qstudio-sql-execution.webp new file mode 100644 index 0000000000000000000000000000000000000000..2fb0c0f135af962e637aef45611c469ea720c94d Binary files /dev/null and b/docs/en/20-third-party/qstudio/qstudio-sql-execution.webp differ diff --git a/docs/en/25-application/_03-immigrate.md b/docs/en/25-application/_03-immigrate.md index f78042353249a29f7ee634cfc544c6c0914e3251..457a40614e836e8735195b6b7e6c50268cf66662 100644 --- a/docs/en/25-application/_03-immigrate.md +++ b/docs/en/25-application/_03-immigrate.md @@ -338,7 +338,7 @@ Remark: Equivalent function: sum ```sql -Select max(value) from (select first(val) value from table_name interval(10s) fill(linear)) interval(10s) +Select sum(value) from (select first(val) value from table_name interval(10s) fill(linear)) interval(10s) ``` Note: This function has no interpolation requirements, so it can be directly calculated. diff --git a/docs/en/27-train-faq/01-faq.md b/docs/en/27-train-faq/01-faq.md index aa28303f5d10e69a8446a2511d288c67ecc8ac02..715704a0c322c7ec22926e95896cafef5677aa1a 100644 --- a/docs/en/27-train-faq/01-faq.md +++ b/docs/en/27-train-faq/01-faq.md @@ -56,7 +56,7 @@ This error indicates that the client could not connect to the server. Perform th 7. If you are using the Python, Java, Go, Rust, C#, or Node.js connector on Linux to connect to the server, verify that `libtaos.so` is in the `/usr/local/taos/driver` directory and `/usr/local/taos/driver` is in the `LD_LIBRARY_PATH` environment variable. -8. If you are using macOS, verify that `libtaos.dylib` is in the `/usr/local/lib` directory and `/usr/local/lib` is in the `LD_LIBRARY_PATH` environment variable.. +8. If you are using macOS, verify that `libtaos.dylib` is in the `/usr/local/lib` directory and `/usr/local/lib` is in the `DYLD_LIBRARY_PATH` environment variable.. 9. If you are using Windows, verify that `C:\TDengine\driver\taos.dll` is in the `PATH` environment variable. If possible, move `taos.dll` to the `C:\Windows\System32` directory. diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index fb31ad67c0f89dfd4428359495239b0bc96f4613..31484dc1c5515548fad40e7072a2e62e92c80e9a 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -6,10 +6,43 @@ description: This document provides download links for all released versions of TDengine 3.x installation packages can be downloaded at the following links: -For TDengine 2.x installation packages by version, please visit [here](https://www.taosdata.com/all-downloads). +For TDengine 2.x installation packages by version, please visit [here](https://tdengine.com/downloads/historical/). import Release from "/components/ReleaseV3"; +## 3.1.0.0 + +:::note IMPORTANT +- Once you upgrade to TDengine 3.1.0.0, you cannot roll back to any previous version of TDengine. Upgrading to 3.1.0.0 will alter your data such that it cannot be read by previous versions. +- You must remove all streams before upgrading to TDengine 3.1.0.0. If you upgrade a deployment that contains streams, the upgrade will fail and your deployment will become nonoperational. +::: + + + +## 3.0.7.1 + + + +## 3.0.7.0 + + + +## 3.0.6.0 + + + +## 3.0.5.1 + + + +## 3.0.5.0 + + + +## 3.0.4.2 + + + ## 3.0.4.1 diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index 9f8dbfee7e8dbce75c62e0978a9be6b13a5b5f35..f3099b13b461548d50228e3bdd2ee5366d214b22 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -10,6 +10,14 @@ For other historical version installers, please visit [here](https://www.taosdat import Release from "/components/ReleaseV3"; +## 2.5.2 + + + +## 2.5.1 + + + ## 2.5.0 diff --git a/docs/examples/R/connect_native.r b/docs/examples/R/connect_native.r index 3c5c9e199b61263b785e86238d277bef70070b28..bd764c3ec49f076d7fabf8104ffa829c47ffe70b 100644 --- a/docs/examples/R/connect_native.r +++ b/docs/examples/R/connect_native.r @@ -8,9 +8,13 @@ library("rJava") library("RJDBC") args<- commandArgs(trailingOnly = TRUE) -driver_path = args[1] # path to jdbc-driver for example: "/root/taos-jdbcdriver-3.0.0-dist.jar" +driver_path = args[1] # path to jdbc-driver for example: "/root/taos-jdbcdriver-3.2.4-dist.jar" driver = JDBC("com.taosdata.jdbc.TSDBDriver", driver_path) conn = dbConnect(driver, "jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata") dbGetQuery(conn, "SELECT server_version()") +dbSendUpdate(conn, "create database if not exists rtest") +dbSendUpdate(conn, "create table if not exists rtest.test (ts timestamp, current float, voltage int, devname varchar(20))") +dbSendUpdate(conn, "insert into rtest.test values (now, 1.2, 220, 'test')") +dbGetQuery(conn, "select * from rtest.test") dbDisconnect(conn) # ANCHOR_END: demo diff --git a/docs/examples/R/connect_rest.r b/docs/examples/R/connect_rest.r index 5ceec572fc26575dfc597983eeac3233bc4488ab..a5221d2c3b2091880e4c87b1ed39df6db0b62d75 100644 --- a/docs/examples/R/connect_rest.r +++ b/docs/examples/R/connect_rest.r @@ -2,11 +2,19 @@ if (! "RJDBC" %in% installed.packages()[, "Package"]) { install.packages('RJDBC', repos='http://cran.us.r-project.org') } +# ANCHOR: demo library("DBI") library("rJava") library("RJDBC") -driver_path = "/home/debug/build/lib/taos-jdbcdriver-2.0.38-dist.jar" + +args<- commandArgs(trailingOnly = TRUE) +driver_path = args[1] # path to jdbc-driver for example: "/root/taos-jdbcdriver-3.2.4-dist.jar" driver = JDBC("com.taosdata.jdbc.rs.RestfulDriver", driver_path) conn = dbConnect(driver, "jdbc:TAOS-RS://localhost:6041?user=root&password=taosdata") dbGetQuery(conn, "SELECT server_version()") -dbDisconnect(conn) \ No newline at end of file +dbSendUpdate(conn, "create database if not exists rtest") +dbSendUpdate(conn, "create table if not exists rtest.test (ts timestamp, current float, voltage int, devname varchar(20))") +dbSendUpdate(conn, "insert into rtest.test values (now, 1.2, 220, 'test')") +dbGetQuery(conn, "select * from rtest.test") +dbDisconnect(conn) +# ANCHOR_END: demo diff --git a/docs/examples/R/readme.txt b/docs/examples/R/readme.txt new file mode 100644 index 0000000000000000000000000000000000000000..131a324aa4ebf20eaef268d295c06640e0f98026 --- /dev/null +++ b/docs/examples/R/readme.txt @@ -0,0 +1 @@ +apt install -y libbz2-dev libpcre2-dev libicu-dev diff --git a/docs/examples/c/async_query_example.c b/docs/examples/c/async_query_example.c index 3807c4bfd7e3d4eac449ec9c18acc45bfa3a0389..0618c09f36d093a6e74863171d01cc5dffcf0c52 100644 --- a/docs/examples/c/async_query_example.c +++ b/docs/examples/c/async_query_example.c @@ -78,7 +78,8 @@ int printRow(char *str, TAOS_ROW row, TAOS_FIELD *fields, int numFields) { } break; case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: { int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); memcpy(str + len, row[i], charLen); len += charLen; diff --git a/docs/examples/c/query_example.c b/docs/examples/c/query_example.c index c7d52115b53cccaf7a29dfd1d80c2a2445a6ce82..88c031abc68c08c001d7cdc593fa2fc2206444a5 100644 --- a/docs/examples/c/query_example.c +++ b/docs/examples/c/query_example.c @@ -76,7 +76,8 @@ int printRow(char *str, TAOS_ROW row, TAOS_FIELD *fields, int numFields) { } break; case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: { int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); memcpy(str + len, row[i], charLen); len += charLen; diff --git a/docs/examples/java/pom.xml b/docs/examples/java/pom.xml index 4bdb2062b279ed16367d5b7f6ea5f9404dc361bd..69a0179759ea7bc16ec3189692041abec379e89f 100644 --- a/docs/examples/java/pom.xml +++ b/docs/examples/java/pom.xml @@ -22,7 +22,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.2.1 + 3.2.4 @@ -33,4 +33,4 @@ - \ No newline at end of file + diff --git a/docs/examples/java/src/main/java/com/taos/example/StmtInsertExample.java b/docs/examples/java/src/main/java/com/taos/example/StmtInsertExample.java index bbcc92b22f67c31384b0fb7a082975eaac2ff2bc..72d4ecc725aa4c7cfd6543dd0c41d70563252c1d 100644 --- a/docs/examples/java/src/main/java/com/taos/example/StmtInsertExample.java +++ b/docs/examples/java/src/main/java/com/taos/example/StmtInsertExample.java @@ -6,39 +6,32 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.text.SimpleDateFormat; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Comparator; import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; public class StmtInsertExample { - private static ArrayList tsToLongArray(String ts) { - ArrayList result = new ArrayList<>(); - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); - LocalDateTime localDateTime = LocalDateTime.parse(ts, formatter); - result.add(localDateTime.toInstant(ZoneOffset.of("+8")).toEpochMilli()); - return result; - } - - private static ArrayList toArray(T v) { - ArrayList result = new ArrayList<>(); - result.add(v); - return result; - } + private static String datePattern = "yyyy-MM-dd HH:mm:ss.SSS"; + private static DateTimeFormatter formatter = DateTimeFormatter.ofPattern(datePattern); - private static List getRawData() { - return Arrays.asList( - "d1001,2018-10-03 14:38:05.000,10.30000,219,0.31000,California.SanFrancisco,2", - "d1001,2018-10-03 14:38:15.000,12.60000,218,0.33000,California.SanFrancisco,2", - "d1001,2018-10-03 14:38:16.800,12.30000,221,0.31000,California.SanFrancisco,2", - "d1002,2018-10-03 14:38:16.650,10.30000,218,0.25000,California.SanFrancisco,3", - "d1003,2018-10-03 14:38:05.500,11.80000,221,0.28000,California.LosAngeles,2", - "d1003,2018-10-03 14:38:16.600,13.40000,223,0.29000,California.LosAngeles,2", - "d1004,2018-10-03 14:38:05.000,10.80000,223,0.29000,California.LosAngeles,3", - "d1004,2018-10-03 14:38:06.500,11.50000,221,0.35000,California.LosAngeles,3" - ); + private static List getRawData(int size) { + SimpleDateFormat format = new SimpleDateFormat(datePattern); + List result = new ArrayList<>(); + long current = System.currentTimeMillis(); + Random random = new Random(); + for (int i = 0; i < size; i++) { + String time = format.format(current + i); + int id = random.nextInt(10); + result.add("d" + id + "," + time + ",10.30000,219,0.31000,California.SanFrancisco,2"); + } + return result.stream() + .sorted(Comparator.comparing(s -> s.split(",")[0])).collect(Collectors.toList()); } private static Connection getConnection() throws SQLException { @@ -48,9 +41,9 @@ public class StmtInsertExample { private static void createTable(Connection conn) throws SQLException { try (Statement stmt = conn.createStatement()) { - stmt.execute("CREATE DATABASE power KEEP 3650"); - stmt.executeUpdate("USE power"); - stmt.execute("CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " + + stmt.execute("CREATE DATABASE if not exists power KEEP 3650"); + stmt.executeUpdate("use power"); + stmt.execute("CREATE STABLE if not exists meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " + "TAGS (location BINARY(64), groupId INT)"); } } @@ -58,21 +51,54 @@ public class StmtInsertExample { private static void insertData() throws SQLException { try (Connection conn = getConnection()) { createTable(conn); - String psql = "INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)"; + String psql = "INSERT INTO ? USING power.meters TAGS(?, ?) VALUES(?, ?, ?, ?)"; try (TSDBPreparedStatement pst = (TSDBPreparedStatement) conn.prepareStatement(psql)) { - for (String line : getRawData()) { + String tableName = null; + ArrayList ts = new ArrayList<>(); + ArrayList current = new ArrayList<>(); + ArrayList voltage = new ArrayList<>(); + ArrayList phase = new ArrayList<>(); + for (String line : getRawData(100000)) { String[] ps = line.split(","); - // bind table name and tags - pst.setTableName(ps[0]); - pst.setTagString(0, ps[5]); - pst.setTagInt(1, Integer.valueOf(ps[6])); + if (tableName == null) { + // bind table name and tags + tableName = "power." + ps[0]; + pst.setTableName(ps[0]); + pst.setTagString(0, ps[5]); + pst.setTagInt(1, Integer.valueOf(ps[6])); + } else { + if (!tableName.equals(ps[0])) { + pst.setTimestamp(0, ts); + pst.setFloat(1, current); + pst.setInt(2, voltage); + pst.setFloat(3, phase); + pst.columnDataAddBatch(); + pst.columnDataExecuteBatch(); + + // bind table name and tags + tableName = ps[0]; + pst.setTableName(ps[0]); + pst.setTagString(0, ps[5]); + pst.setTagInt(1, Integer.valueOf(ps[6])); + ts.clear(); + current.clear(); + voltage.clear(); + phase.clear(); + } + } // bind values - pst.setTimestamp(0, tsToLongArray(ps[1])); //ps[1] looks like: 2018-10-03 14:38:05.000 - pst.setFloat(1, toArray(Float.valueOf(ps[2]))); - pst.setInt(2, toArray(Integer.valueOf(ps[3]))); - pst.setFloat(3, toArray(Float.valueOf(ps[4]))); - pst.columnDataAddBatch(); + // ps[1] looks like: 2018-10-03 14:38:05.000 + LocalDateTime localDateTime = LocalDateTime.parse(ps[1], formatter); + ts.add(localDateTime.toInstant(ZoneOffset.of("+8")).toEpochMilli()); + current.add(Float.valueOf(ps[2])); + voltage.add(Integer.valueOf(ps[3])); + phase.add(Float.valueOf(ps[4])); } + pst.setTimestamp(0, ts); + pst.setFloat(1, current); + pst.setInt(2, voltage); + pst.setFloat(3, phase); + pst.columnDataAddBatch(); pst.columnDataExecuteBatch(); } } diff --git a/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java b/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java index b5cdedc34f00dc2914f62176e4bd8b1d80a01bf6..3c5d2867e230658536cabfa36252662656cb7f02 100644 --- a/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java @@ -53,20 +53,28 @@ public class SubscribeDemo { // create consumer Properties properties = new Properties(); + properties.getProperty(TMQConstants.CONNECT_TYPE, "jni"); properties.setProperty(TMQConstants.BOOTSTRAP_SERVERS, "127.0.0.1:6030"); + properties.setProperty(TMQConstants.CONNECT_USER, "root"); + properties.setProperty(TMQConstants.CONNECT_PASS, "taosdata"); properties.setProperty(TMQConstants.MSG_WITH_TABLE_NAME, "true"); properties.setProperty(TMQConstants.ENABLE_AUTO_COMMIT, "true"); - properties.setProperty(TMQConstants.GROUP_ID, "test"); + properties.setProperty(TMQConstants.AUTO_COMMIT_INTERVAL, "1000"); + properties.setProperty(TMQConstants.GROUP_ID, "test1"); + properties.setProperty(TMQConstants.CLIENT_ID, "1"); + properties.setProperty(TMQConstants.AUTO_OFFSET_RESET, "earliest"); properties.setProperty(TMQConstants.VALUE_DESERIALIZER, "com.taos.example.MetersDeserializer"); + properties.setProperty(TMQConstants.VALUE_DESERIALIZER_ENCODING, "UTF-8"); + properties.setProperty(TMQConstants.EXPERIMENTAL_SNAPSHOT_ENABLE, "true"); // poll data try (TaosConsumer consumer = new TaosConsumer<>(properties)) { consumer.subscribe(Collections.singletonList(TOPIC)); while (!shutdown.get()) { ConsumerRecords meters = consumer.poll(Duration.ofMillis(100)); - for (ConsumerRecord recode : meters) { - Meters meter = recode.value(); + for (ConsumerRecord r : meters) { + Meters meter = r.value(); System.out.println(meter); } } diff --git a/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java b/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java index 83cb04f55244aacf860573a42ddc952e1e522803..03f7e3a11e8ae64ed641097746f9d44da73bf3d6 100644 --- a/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java @@ -1,5 +1,6 @@ package com.taos.example; +import com.taosdata.jdbc.tmq.ConsumerRecord; import com.taosdata.jdbc.tmq.ConsumerRecords; import com.taosdata.jdbc.tmq.TMQConstants; import com.taosdata.jdbc.tmq.TaosConsumer; @@ -54,18 +55,26 @@ public class WebsocketSubscribeDemo { Properties properties = new Properties(); properties.setProperty(TMQConstants.BOOTSTRAP_SERVERS, "127.0.0.1:6041"); properties.setProperty(TMQConstants.CONNECT_TYPE, "ws"); + properties.setProperty(TMQConstants.CONNECT_USER, "root"); + properties.setProperty(TMQConstants.CONNECT_PASS, "taosdata"); + properties.setProperty(TMQConstants.AUTO_OFFSET_RESET, "earliest"); properties.setProperty(TMQConstants.MSG_WITH_TABLE_NAME, "true"); properties.setProperty(TMQConstants.ENABLE_AUTO_COMMIT, "true"); - properties.setProperty(TMQConstants.GROUP_ID, "test"); + properties.setProperty(TMQConstants.AUTO_COMMIT_INTERVAL, "1000"); + properties.setProperty(TMQConstants.GROUP_ID, "test2"); + properties.setProperty(TMQConstants.CLIENT_ID, "1"); properties.setProperty(TMQConstants.VALUE_DESERIALIZER, "com.taos.example.MetersDeserializer"); + properties.setProperty(TMQConstants.VALUE_DESERIALIZER_ENCODING, "UTF-8"); + properties.setProperty(TMQConstants.EXPERIMENTAL_SNAPSHOT_ENABLE, "true"); // poll data try (TaosConsumer consumer = new TaosConsumer<>(properties)) { consumer.subscribe(Collections.singletonList(TOPIC)); while (!shutdown.get()) { ConsumerRecords meters = consumer.poll(Duration.ofMillis(100)); - for (Meters meter : meters) { + for (ConsumerRecord r : meters) { + Meters meter = (Meters) r.value(); System.out.println(meter); } } diff --git a/docs/examples/python/stmt_example.py b/docs/examples/python/stmt_example.py new file mode 100644 index 0000000000000000000000000000000000000000..83197a777ab6419962212e7f1106fba7b78e884e --- /dev/null +++ b/docs/examples/python/stmt_example.py @@ -0,0 +1,82 @@ +#! + +import taosws + +import taos + +db_name = 'test_ws_stmt' + + +def before(): + taos_conn = taos.connect() + taos_conn.execute("drop database if exists %s" % db_name) + taos_conn.execute("create database %s" % db_name) + taos_conn.select_db(db_name) + taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))") + taos_conn.execute( + "create table stb1 (ts timestamp, a int, b float, c varchar(10)) tags (t1 int, t2 binary(10))") + taos_conn.close() + + +def stmt_insert(): + before() + + conn = taosws.connect('taosws://root:taosdata@localhost:6041/%s' % db_name) + + while True: + try: + stmt = conn.statement() + stmt.prepare("insert into t1 values (?, ?, ?, ?)") + + stmt.bind_param([ + taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]), + taosws.ints_to_column([1, 2, 3, 4]), + taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]), + taosws.varchar_to_column(['a', 'b', 'c', 'd']), + ]) + + stmt.add_batch() + rows = stmt.execute() + print(rows) + stmt.close() + except Exception as e: + if 'Retry needed' in e.args[0]: # deal with [0x0125] Retry needed + continue + else: + raise e + + break + + +def stmt_insert_into_stable(): + before() + + conn = taosws.connect("taosws://root:taosdata@localhost:6041/%s" % db_name) + + while True: + try: + stmt = conn.statement() + stmt.prepare("insert into ? using stb1 tags (?, ?) values (?, ?, ?, ?)") + stmt.set_tbname('stb1_1') + stmt.set_tags([ + taosws.int_to_tag(1), + taosws.varchar_to_tag('aaa'), + ]) + stmt.bind_param([ + taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]), + taosws.ints_to_column([1, 2, 3, 4]), + taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]), + taosws.varchar_to_column(['a', 'b', 'c', 'd']), + ]) + + stmt.add_batch() + rows = stmt.execute() + print(rows) + stmt.close() + except Exception as e: + if 'Retry needed' in e.args[0]: # deal with [0x0125] Retry needed + continue + else: + raise e + + break diff --git a/docs/examples/python/stmt_websocket_example.py b/docs/examples/python/stmt_websocket_example.py new file mode 100644 index 0000000000000000000000000000000000000000..d0824cfa9f29320a6613eb49ee58d108fc61cfd7 --- /dev/null +++ b/docs/examples/python/stmt_websocket_example.py @@ -0,0 +1,78 @@ +#! +import time + +import taosws + +import taos + + +def before_test(db_name): + taos_conn = taos.connect() + taos_conn.execute("drop database if exists %s" % db_name) + taos_conn.execute("create database %s" % db_name) + taos_conn.select_db(db_name) + taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))") + taos_conn.execute( + "create table stb1 (ts timestamp, a int, b float, c varchar(10)) tags (t1 int, t2 binary(10))") + taos_conn.close() + + +def after_test(db_name): + taos_conn = taos.connect() + taos_conn.execute("drop database if exists %s" % db_name) + taos_conn.close() + + +def stmt_insert(): + db_name = 'test_ws_stmt_{}'.format(int(time.time())) + before_test(db_name) + + conn = taosws.connect('taosws://root:taosdata@localhost:6041/%s' % db_name) + + stmt = conn.statement() + stmt.prepare("insert into t1 values (?, ?, ?, ?)") + + stmt.bind_param([ + taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]), + taosws.ints_to_column([1, 2, 3, 4]), + taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]), + taosws.varchar_to_column(['a', 'b', 'c', 'd']), + ]) + + stmt.add_batch() + rows = stmt.execute() + assert rows == 4 + stmt.close() + after_test(db_name) + + +def stmt_insert_into_stable(): + db_name = 'test_ws_stmt_{}'.format(int(time.time())) + before_test(db_name) + + conn = taosws.connect("taosws://root:taosdata@localhost:6041/%s" % db_name) + + stmt = conn.statement() + stmt.prepare("insert into ? using stb1 tags (?, ?) values (?, ?, ?, ?)") + stmt.set_tbname('stb1_1') + stmt.set_tags([ + taosws.int_to_tag(1), + taosws.varchar_to_tag('aaa'), + ]) + stmt.bind_param([ + taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]), + taosws.ints_to_column([1, 2, 3, 4]), + taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]), + taosws.varchar_to_column(['a', 'b', 'c', 'd']), + ]) + + stmt.add_batch() + rows = stmt.execute() + assert rows == 4 + stmt.close() + after_test(db_name) + + +if __name__ == '__main__': + stmt_insert() + stmt_insert_into_stable() diff --git a/docs/examples/python/tmq_assignment_example.py b/docs/examples/python/tmq_assignment_example.py new file mode 100644 index 0000000000000000000000000000000000000000..41737e3fc498c785b6bd0e23b79f80c62f364476 --- /dev/null +++ b/docs/examples/python/tmq_assignment_example.py @@ -0,0 +1,58 @@ +import taos +from taos.tmq import Consumer +import taosws + + +def prepare(): + conn = taos.connect() + conn.execute("drop topic if exists tmq_assignment_demo_topic") + conn.execute("drop database if exists tmq_assignment_demo_db") + conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600") + conn.select_db("tmq_assignment_demo_db") + conn.execute( + "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)") + conn.execute( + "create topic if not exists tmq_assignment_demo_topic as select ts, c1, c2, c3 from tmq_assignment_demo_table") + conn.execute("insert into d0 using tmq_assignment_demo_table tags (0) values (now-2s, 1, 1.0, 'tmq test')") + conn.execute("insert into d0 using tmq_assignment_demo_table tags (0) values (now-1s, 2, 2.0, 'tmq test')") + conn.execute("insert into d0 using tmq_assignment_demo_table tags (0) values (now, 3, 3.0, 'tmq test')") + + +def taos_get_assignment_and_seek_demo(): + prepare() + consumer = Consumer( + { + "group.id": "0", + # should disable snapshot, + # otherwise it will cause invalid params error + "experimental.snapshot.enable": "false", + } + ) + consumer.subscribe(["tmq_assignment_demo_topic"]) + + # get topic assignment + assignments = consumer.assignment() + for assignment in assignments: + print(assignment) + + # poll + consumer.poll(1) + consumer.poll(1) + + # get topic assignment again + after_pool_assignments = consumer.assignment() + for assignment in after_pool_assignments: + print(assignment) + + # seek to the beginning + for assignment in assignments: + consumer.seek(assignment) + + # now the assignment should be the same as before poll + assignments = consumer.assignment() + for assignment in assignments: + print(assignment) + + +if __name__ == '__main__': + taos_get_assignment_and_seek_demo() diff --git a/docs/examples/python/tmq_websocket_assgnment_example.py b/docs/examples/python/tmq_websocket_assgnment_example.py new file mode 100644 index 0000000000000000000000000000000000000000..0f8e4a28042e484029068816fdfebd3b0b27a587 --- /dev/null +++ b/docs/examples/python/tmq_websocket_assgnment_example.py @@ -0,0 +1,57 @@ +import taos +import taosws + + +def prepare(): + conn = taos.connect() + conn.execute("drop topic if exists tmq_assignment_demo_topic") + conn.execute("drop database if exists tmq_assignment_demo_db") + conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600") + conn.select_db("tmq_assignment_demo_db") + conn.execute( + "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)") + conn.execute( + "create topic if not exists tmq_assignment_demo_topic as select ts, c1, c2, c3 from tmq_assignment_demo_table") + conn.execute("insert into d0 using tmq_assignment_demo_table tags (0) values (now-2s, 1, 1.0, 'tmq test')") + conn.execute("insert into d0 using tmq_assignment_demo_table tags (0) values (now-1s, 2, 2.0, 'tmq test')") + conn.execute("insert into d0 using tmq_assignment_demo_table tags (0) values (now, 3, 3.0, 'tmq test')") + + +def taosws_get_assignment_and_seek_demo(): + prepare() + consumer = taosws.Consumer(conf={ + "td.connect.websocket.scheme": "ws", + # should disable snapshot, + # otherwise it will cause invalid params error + "experimental.snapshot.enable": "false", + "group.id": "0", + }) + consumer.subscribe(["tmq_assignment_demo_topic"]) + + # get topic assignment + assignments = consumer.assignment() + for assignment in assignments: + print(assignment.to_string()) + + # poll + consumer.poll(1) + consumer.poll(1) + + # get topic assignment again + after_poll_assignments = consumer.assignment() + for assignment in after_poll_assignments: + print(assignment.to_string()) + + # seek to the beginning + for assignment in assignments: + for a in assignment.assignments(): + consumer.seek(assignment.topic(), a.vg_id(), a.offset()) + + # now the assignment should be the same as before poll + assignments = consumer.assignment() + for assignment in assignments: + print(assignment.to_string()) + + +if __name__ == '__main__': + taosws_get_assignment_and_seek_demo() diff --git a/docs/zh/02-intro.md b/docs/zh/02-intro.md index 525c8aa5c9764409381928ecddab49779346c206..888f779966155c19fe90d2376b053937405eb3c9 100644 --- a/docs/zh/02-intro.md +++ b/docs/zh/02-intro.md @@ -92,7 +92,7 @@ TDengine 的主要功能如下： ## 典型适用场景 -作为一个高性能、分布式、支持 SQL 的时序数据库（Database），TDengine 的典型适用场景包括但不限于 IoT、工业互联网、车联网、IT 运维、能源、金融证券等领域。需要指出的是，TDengine 是针对时序数据场景设计的专用数据库和专用大数据处理工具，因其充分利用了时序大数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM 等通用型数据。下面本文将对适用场景做更多详细的分析。 +作为一个高性能、分布式、支持 SQL 的时序数据库（Time-series Database），TDengine 的典型适用场景包括但不限于 IoT、工业互联网、车联网、IT 运维、能源、金融证券等领域。需要指出的是，TDengine 是针对时序数据场景设计的专用数据库和专用大数据处理工具，因其充分利用了时序大数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM 等通用型数据。下面本文将对适用场景做更多详细的分析。 ### 数据源特点和需求 diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md index e772447db087c119fae5571b574b419950437b9d..efc21d32964025b436aa1e7efd17115333c69418 100644 --- a/docs/zh/05-get-started/01-docker.md +++ b/docs/zh/05-get-started/01-docker.md @@ -28,6 +28,21 @@ docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043 注意：TDengine 3.0 服务端仅使用 6030 TCP 端口。6041 为 taosAdapter 所使用提供 REST 服务端口。6043-6049 为 taosAdapter 提供第三方应用接入所使用端口，可根据需要选择是否打开。 +如果需要将数据持久化到本机的某一个文件夹，则执行下边的命令： + +```shell +docker run -d -v ~/data/taos/dnode/data:/var/lib/taos \ + -v ~/data/taos/dnode/log:/var/log/taos \ + -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine +``` + +:::note + +- /var/lib/taos: TDengine 默认数据文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode/data为你自己的数据目录 +- /var/log/taos: TDengine 默认日志文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode/log为你自己的日志目录 + +::: + 确定该容器已经启动并且在正常运行。 ```shell @@ -108,4 +123,4 @@ SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL( ## 其它 -更多关于在 Docker 环境下使用 TDengine 的细节，请参考 [在 Docker 下使用 TDengine](../../reference/docker)。 +更多关于在 Docker 环境下使用 TDengine 的细节，请参考 [用 Docker 部署 TDengine](../../deployment/docker)。 diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 1cd0076ba592e5055b7018f0149b5f6f53f18626..621effa6fd52f17fd9ae36994dbfe2f0d4dfc52e 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -16,6 +16,20 @@ TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc） 在 Linux 系统上，TDengine 社区版提供 Deb 和 RPM 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 Deb 支持 Debian/Ubuntu 及其衍生系统，RPM 支持 CentOS/RHEL/SUSE 及其衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，RPM 和 Deb 包不含 `taosdump` 和 TDinsight 安装脚本，这些工具需要通过安装 taosTools 包获得。TDengine 也提供 Windows x64 平台和 macOS x64/m1 平台的安装包。 +## 运行环境要求 +在linux系统中，运行环境最低要求如下: + +linux 内核版本 - 3.10.0-1160.83.1.el7.x86_64; + +glibc 版本 - 2.17; + +如果通过clone源码进行编译安装，还需要满足: + +cmake版本 - 3.26.4或以上; + +gcc 版本 - 9.3.1或以上; + + ## 安装 @@ -187,7 +201,7 @@ Active: inactive (dead) -安装后，可以在拥有管理员权限的 cmd 窗口执行 `sc start taosd` 或在 `C:\TDengine` 目录下，运行 `taosd.exe` 来启动 TDengine 服务进程。 +安装后，可以在拥有管理员权限的 cmd 窗口执行 `sc start taosd` 或在 `C:\TDengine` 目录下，运行 `taosd.exe` 来启动 TDengine 服务进程。如需使用 http/REST 服务，请执行 `sc start taosadapter` 或运行 `taosadapter.exe` 来启动 taosAdapter 服务进程。 **TDengine 命令行（CLI）** @@ -299,7 +313,7 @@ SELECT COUNT(*) FROM test.meters WHERE location = "California.SanFrancisco"; SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10; ``` -对表 `d10` 按 10 每秒进行平均值、最大值和最小值聚合统计： +对表 `d10` 按每 10 秒进行平均值、最大值和最小值聚合统计： ```sql SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s); diff --git a/docs/zh/07-develop/01-connect/index.md b/docs/zh/07-develop/01-connect/index.md index da31dec4c1583cdda56b1bbeab52eed91a7b07f7..83032067d98e5506cae03e6072f2297c4a8fa6ed 100644 --- a/docs/zh/07-develop/01-connect/index.md +++ b/docs/zh/07-develop/01-connect/index.md @@ -82,7 +82,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 com.taosdata.jdbc taos-jdbcdriver - 3.0.0 + 3.2.4 ``` diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index 11aef1f7e8e9c7e84a66aef229b44f87dc582260..38b91d7cea5a5aa0e18a43044a15136e8d66628e 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -81,10 +81,6 @@ Set subscription() throws SQLException; ConsumerRecords poll(Duration timeout) throws SQLException; -void commitAsync(); - -void commitAsync(OffsetCommitCallback callback); - void commitSync() throws SQLException; void close() throws SQLException; @@ -105,6 +101,12 @@ class Consumer: def poll(self, timeout: float = 1.0): pass + def assignment(self): + pass + + def seek(self, partition): + pass + def close(self): pass @@ -237,6 +239,7 @@ TDengine 使用 SQL 创建一个 topic： ```sql CREATE TOPIC topic_name AS SELECT ts, c1, c2, c3 FROM tmqdb.stb WHERE c1 > 1; ``` +- topic创建个数有上限，通过参数 tmqMaxTopicNum 控制，默认 20 个 TMQ 支持多种订阅类型： @@ -259,14 +262,15 @@ CREATE TOPIC topic_name as subquery 语法： ```sql -CREATE TOPIC topic_name AS STABLE stb_name +CREATE TOPIC topic_name [with meta] AS STABLE stb_name [where_condition] ``` 与 `SELECT * from stbName` 订阅的区别是： - 不会限制用户的表结构变更。 - 返回的是非结构化的数据：返回数据的结构会随之超级表的表结构变化而变化。 -- 用户对于要处理的每一个数据块都可能有不同的表结构。 +- with meta 参数可选，选择时将返回创建超级表，子表等语句，主要用于taosx做超级表迁移 +- where_condition 参数可选，选择时将用来过滤符合条件的子表，订阅这些子表。where 条件里不能有普通列，只能是tag或tbname，where条件里可以用函数，用来过滤tag，但是不能是聚合函数，因为子表tag值无法做聚合。也可以是常量表达式，比如 2 > 1（订阅全部子表），或者 false（订阅0个子表） - 返回数据不包含标签。 ### 数据库订阅 @@ -274,22 +278,24 @@ CREATE TOPIC topic_name AS STABLE stb_name 语法： ```sql -CREATE TOPIC topic_name AS DATABASE db_name; +CREATE TOPIC topic_name [with meta] AS DATABASE db_name; ``` 通过该语句可创建一个包含数据库所有表数据的订阅 +- with meta 参数可选，选择时将返回创建数据库里所有超级表，子表的语句，主要用于taosx做数据库迁移 + ## 创建消费者 *consumer* 消费者需要通过一系列配置选项创建，基础配置项如下表所示： | 参数名称 | 类型 | 参数说明 | 备注 | | :----------------------------: | :-----: | -------------------------------------------------------- | ------------------------------------------- | -| `td.connect.ip` | string | 用于创建连接，同 `taos_connect` | 仅用于建立原生连接 | -| `td.connect.user` | string | 用于创建连接，同 `taos_connect` | 仅用于建立原生连接 | -| `td.connect.pass` | string | 用于创建连接，同 `taos_connect` | 仅用于建立原生连接 | -| `td.connect.port` | integer | 用于创建连接，同 `taos_connect` | 仅用于建立原生连接 | -| `group.id` | string | 消费组 ID，同一消费组共享消费进度 | **必填项**。最大长度：192。 | +| `td.connect.ip` | string | 服务端的 IP 地址 | | +| `td.connect.user` | string | 用户名 | | +| `td.connect.pass` | string | 密码 | | +| `td.connect.port` | integer | 服务端的端口号 | | +| `group.id` | string | 消费组 ID，同一消费组共享消费进度 |
**必填项**。最大长度：192。
每个topic最多可建立100个 consumer group | | `client.id` | string | 客户端 ID | 最大长度：192。 | | `auto.offset.reset` | enum | 消费组订阅的初始位置 |
`earliest`: default;从头开始订阅;
`latest`: 仅从最新数据开始订阅;
`none`: 没有提交的 offset 无法订阅 | | `enable.auto.commit` | boolean | 是否启用消费位点自动提交，true: 自动提交，客户端应用无需commit；false：客户端应用需要自行commit | 默认值为 true | @@ -321,10 +327,11 @@ tmq_conf_destroy(conf); -对于 Java 程序，使用如下配置项： +对于 Java 程序，还可以使用如下配置项： | 参数名称 | 类型 | 参数说明 | | ----------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------- | +| `td.connect.type` | string | 连接类型，"jni" 指原生连接，"ws" 指 websocket 连接，默认值为 "jni" | | `bootstrap.servers` | string | 连接地址，如 `localhost:6030` | | `value.deserializer` | string | 值解析方法，使用此方法应实现 `com.taosdata.jdbc.tmq.Deserializer` 接口或继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer` 类 | | `value.deserializer.encoding` | string | 指定字符串解析的字符集 | | @@ -401,21 +408,6 @@ from taos.tmq import Consumer consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) ``` -其中，`configs` 为 dict 类型，传递创建 Consumer 的参数。可以配置的参数有： - -| 参数名称 | 类型 | 参数说明 | 备注 | -|:------:|:----:|:-------:|:---:| -| `td.connect.ip` | string | 用于创建连接|| -| `td.connect.user` | string | 用于创建连接|| -| `td.connect.pass` | string | 用于创建连接|| -| `td.connect.port` | string | 用于创建连接|| -| `group.id` | string | 消费组 ID，同一消费组共享消费进度 | **必填项**。最大长度：192 | -| `client.id` | string | 客户端 ID | 最大长度：192 | -| `msg.with.table.name` | string | 是否允许从消息中解析表名,不适用于列订阅 | 合法值：`true`, `false` | -| `enable.auto.commit` | string | 启用自动提交 | 合法值：`true`, `false` | -| `auto.commit.interval.ms` | string | 以毫秒为单位的自动提交时间间隔 | 默认值：5000 ms | -| `auto.offset.reset` | string | 消费组订阅的初始位置 | 可选：`earliest`(default), `latest`, `none` | - diff --git a/docs/zh/07-develop/08-cache.md b/docs/zh/07-develop/08-cache.md index 29e28e3dde0816d9e5a08f74abd2382854d336da..07479b9a27e12745101898e9a8e046b5f169adc1 100644 --- a/docs/zh/07-develop/08-cache.md +++ b/docs/zh/07-develop/08-cache.md @@ -10,10 +10,10 @@ description: "TDengine 内部的缓存设计" TDengine 采用时间驱动缓存管理策略（First-In-First-Out，FIFO），又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式（Least-Recent-Used，LRU），直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候，将最早的数据批量写入磁盘。一般意义上来说，对于物联网数据的使用，用户最为关心最近产生的数据，即当前状态。TDengine 充分利用了这一特性，将最近到达的（当前状态）数据保存在缓存中。 -每个 vnode 的写入缓存大小在创建数据库时决定，创建数据库时的两个关键参数 vgroups 和 buffer 分别决定了该数据库中的数据由多少个 vgroup 处理，以及向其中的每个 vnode 分配多少写入缓存。 +每个 vnode 的写入缓存大小在创建数据库时决定，创建数据库时的两个关键参数 vgroups 和 buffer 分别决定了该数据库中的数据由多少个 vgroup 处理，以及向其中的每个 vnode 分配多少写入缓存。buffer 的单位是MB。 ```sql -create database db0 vgroups 100 buffer 16MB +create database db0 vgroups 100 buffer 16 ``` 理论上缓存越大越好，但超过一定阈值后再增加缓存对写入性能提升并无帮助，一般情况下使用默认值即可。 @@ -28,10 +28,10 @@ create database db0 vgroups 100 buffer 16MB ## 元数据缓存 -为了更高效地处理查询和写入，每个 vnode 都会缓存自己曾经获取到的元数据。元数据缓存由创建数据库时的两个参数 pages 和 pagesize 决定。 +为了更高效地处理查询和写入，每个 vnode 都会缓存自己曾经获取到的元数据。元数据缓存由创建数据库时的两个参数 pages 和 pagesize 决定。pagesize 的单位是 kb。 ```sql -create database db0 pages 128 pagesize 16kb +create database db0 pages 128 pagesize 16 ``` 上述语句会为数据库 db0 的每个 vnode 创建 128 个 page，每个 page 16kb 的元数据缓存。 diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 99ecd903b40a1a67a447415d1392ec65a5ecc18b..bb6a575ccde7166ae84ece2d0da9f438578767ba 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -17,7 +17,7 @@ TDengine 支持通过 C/Python 语言进行 UDF 定义。接下来结合示例 - 聚合函数需要实现聚合接口函数 aggfn_start ， aggfn ， aggfn_finish。 - 如果需要初始化，实现 udf_init；如果需要清理工作，实现udf_destroy。 -接口函数的名称是 UDF 名称，或者是 UDF 名称和特定后缀（_start, _finish, _init, _destroy)的连接。列表中的scalarfn，aggfn, udf需要替换成udf函数名。 +接口函数的名称是 UDF 名称，或者是 UDF 名称和特定后缀（`_start`, `_finish`, `_init`, `_destroy`)的连接。列表中的scalarfn，aggfn, udf需要替换成udf函数名。 ### 用 C 语言实现标量函数 标量函数实现模板如下 @@ -271,26 +271,90 @@ select max_vol(vol1,vol2,vol3,deviceid) from battery; ## 用 Python 语言实现 UDF +### 准备环境 + +1. 准备好 Python 运行环境 + +2. 安装 Python 包 `taospyudf` + +```shell +pip3 install taospyudf +``` + +安装过程中会编译 C++ 源码，因此系统上要有 cmake 和 gcc。编译生成的 libtaospyudf.so 文件自动会被复制到 /usr/local/lib/ 目录，因此如果是非 root 用户，安装时需加 sudo。安装完可以检查这个目录是否有了这个文件: + +```shell +root@slave11 ~/udf $ ls -l /usr/local/lib/libtaos* +-rw-r--r-- 1 root root 671344 May 24 22:54 /usr/local/lib/libtaospyudf.so +``` + +然后执行命令 +```shell +ldconfig +``` + +3. 如果 Python UDF 程序执行时，通过 PYTHONPATH 引用其它的包，可以设置 taos.cfg 的 UdfdLdLibPath 变量为PYTHONPATH的内容 + +4. 启动 `taosd` 服务 +细节请参考 [快速开始](../../get-started) + +### 接口定义 + +#### 接口概述 + 使用 Python 语言实现 UDF 时，需要实现规定的接口函数 - 标量函数需要实现标量接口函数 process 。 - 聚合函数需要实现聚合接口函数 start ，reduce ，finish。 - 如果需要初始化，实现 init；如果需要清理工作，实现 destroy。 -### 用 Python 实现标量函数 +#### 标量函数接口 +```Python +def process(input: datablock) -> tuple[output_type]: +``` + +说明： + - input:datablock 类似二维矩阵，通过成员方法 data(row,col)返回位于 row 行，col 列的 python 对象 + - 返回值是一个 Python 对象元组，每个元素类型为输出类型。 + +#### 聚合函数接口 +```Python +def start() -> bytes: +def reduce(inputs: datablock, buf: bytes) -> bytes +def finish(buf: bytes) -> output_type: +``` + +说明： + - 首先调用 start 生成最初结果 buffer + - 然后输入数据会被分为多个行数据块，对每个数据块 inputs 和当前中间结果 buf 调用 reduce，得到新的中间结果 + - 最后再调用 finish 从中间结果 buf 产生最终输出，最终输出只能含 0 或 1 条数据。 + +#### 初始化和销毁接口 +```Python +def init() +def destroy() +``` + +说明： + - init 完成初始化工作 + - destroy 完成清理工作 + +### Python UDF 函数模板 + +#### 标量函数实现模板 标量函数实现模版如下 + ```Python def init(): # initialization def destroy(): # destroy -def process(input: datablock) -> tuple[output_type]: - # process input datablock, - # datablock.data(row, col) is to access the python object in location(row,col) - # return tuple object consisted of object of type outputtype +def process(input: datablock) -> tuple[output_type]: ``` -### 用 Python 实现聚合函数 +注意：定义标题函数最重要是要实现 process 函数，同时必须定义 init 和 destroy 函数即使什么都不做 + +#### 聚合函数实现模板 聚合函数实现模版如下 ```Python @@ -303,41 +367,16 @@ def start() -> bytes: def reduce(inputs: datablock, buf: bytes) -> bytes # deserialize buf to state # reduce the inputs and state into new_state. - # use inputs.data(i,j) to access python ojbect of location(i,j) + # use inputs.data(i,j) to access python object of location(i,j) # serialize new_state into new_state_bytes return new_state_bytes def finish(buf: bytes) -> output_type: #return obj of type outputtype ``` -### Python UDF 接口函数定义 - -#### 标量函数接口 -```Python -def process(input: datablock) -> tuple[output_type]: -``` -- input:datablock 类似二维矩阵，通过成员方法 data(row,col)返回位于 row 行，col 列的 python 对象 -- 返回值是一个 Python 对象元组，每个元素类型为输出类型。 - -#### 聚合函数接口 -```Python -def start() -> bytes: -def reduce(inputs: datablock, buf: bytes) -> bytes -def finish(buf: bytes) -> output_type: -``` - -首先调用 start 生成最初结果 buffer，然后输入数据会被分为多个行数据块，对每个数据块 inputs 和当前中间结果 buf 调用 reduce，得到新的中间结果，最后再调用 finish 从中间结果 buf 产生最终输出，最终输出只能含 0 或 1 条数据。 - - -#### 初始化和销毁接口 -```Python -def init() -def destroy() -``` +注意：定义聚合函数最重要是要实现 start, reduce 和 finish，且必须定义 init 和 destroy 函数。start 生成最初结果 buffer，然后输入数据会被分为多个行数据块，对每个数据块 inputs 和当前中间结果 buf 调用 reduce，得到新的中间结果，最后再调用 finish 从中间结果 buf 产生最终输出。 -其中 init 完成初始化工作。 destroy 完成清理工作。如果没有初始化工作，无需定义 init 函数。如果没有清理工作，无需定义 destroy 函数。 - -### Python 和 TDengine之间的数据类型映射 +### 数据类型映射 下表描述了TDengine SQL数据类型和Python数据类型的映射。任何类型的NULL值都映射成Python的None值。 @@ -351,15 +390,461 @@ def destroy() |TIMESTAMP | int | |JSON and other types | 不支持 | -### Python UDF 环境的安装 -1. 安装 taospyudf 包。此包执行Python UDF程序。 -```bash -sudo pip install taospyudf -ldconfig +### 开发指南 + +本文内容由浅入深包括 4 个示例程序： +1. 定义一个只接收一个整数的标量函数： 输入 n， 输出 ln(n^2 + 1)。 +2. 定义一个接收 n 个整数的标量函数， 输入 （x1, x2, ..., xn）, 输出每个值和它们的序号的乘积的和： x1 + 2 * x2 + ... + n * xn。 +3. 定义一个标量函数，输入一个时间戳，输出距离这个时间最近的下一个周日。完成这个函数要用到第三方库 moment。我们在这个示例中讲解使用第三方库的注意事项。 +4. 定义一个聚合函数，计算某一列最大值和最小值的差, 也就是实现 TDengien 内置的 spread 函数。 +同时也包含大量实用的 debug 技巧。 +本文假设你用的是 Linux 系统，且已安装好了 TDengine 3.0.4.0+ 和 Python 3.7+。 + +注意：**UDF 内无法通过 print 函数输出日志，需要自己写文件或用 python 内置的 logging 库写文件**。 + +#### 最简单的 UDF + +编写一个只接收一个整数的 UDF 函数： 输入 n， 输出 ln(n^2 + 1)。 +首先编写一个 Python 文件，存在系统某个目录，比如 /root/udf/myfun.py 内容如下 + +```python +from math import log + +def init(): + pass + +def destroy(): + pass + +def process(block): + rows, _ = block.shape() + return [log(block.data(i, 0) ** 2 + 1) for i in range(rows)] ``` -2. 如果 Python UDF 程序执行时，通过 PYTHONPATH 引用其它的包，可以设置 taos.cfg 的 UdfdLdLibPath 变量为PYTHONPATH的内容 + +这个文件包含 3 个函数， init 和 destroy 都是空函数，它们是 UDF 的生命周期函数，即使什么都不做也要定义。最关键的是 process 函数， 它接受一个数据块，这个数据块对象有两个方法： +1. shape() 返回数据块的行数和列数 +2. data(i, j) 返回 i 行 j 列的数据 +标量函数的 process 方法传人的数据块有多少行，就需要返回多少个数据。上述代码中我们忽略的列数，因为我们只想对每行的第一个数做计算。 +接下来我们创建对应的 UDF 函数，在 TDengine CLI 中执行下面语句： + +```sql +create function myfun as '/root/udf/myfun.py' outputtype double language 'Python' +``` +其输出如下 + +```shell + taos> create function myfun as '/root/udf/myfun.py' outputtype double language 'Python'; +Create OK, 0 row(s) affected (0.005202s) +``` + +看起来很顺利，接下来 show 一下系统中所有的自定义函数，确认创建成功： + +```text +taos> show functions; + name | +================================= + myfun | +Query OK, 1 row(s) in set (0.005767s) +``` + +接下来就来测试一下这个函数，测试之前先执行下面的 SQL 命令，制造些测试数据，在 TDengine CLI 中执行下述命令 + +```sql +create database test; +create table t(ts timestamp, v1 int, v2 int, v3 int); +insert into t values('2023-05-01 12:13:14', 1, 2, 3); +insert into t values('2023-05-03 08:09:10', 2, 3, 4); +insert into t values('2023-05-10 07:06:05', 3, 4, 5); +``` + +测试 myfun 函数： + +```sql +taos> select myfun(v1, v2) from t; + +DB error: udf function execution failure (0.011088s) +``` + +不幸的是执行失败了，什么原因呢？ +查看 udfd 进程的日志 + +```shell +tail -10 /var/log/taos/udfd.log +``` + +发现以下错误信息： + +```text +05/24 22:46:28.733545 01665799 UDF ERROR can not load library libtaospyudf.so. error: operation not permitted +05/24 22:46:28.733561 01665799 UDF ERROR can not load python plugin. lib path libtaospyudf.so +``` + +错误很明确：没有加载到 Python 插件 libtaospyudf.so，如果遇到此错误，请参考前面的准备环境一节。 + +修复环境错误后再次执行，如下： + +```sql +taos> select myfun(v1) from t; + myfun(v1) | +============================ + 0.693147181 | + 1.609437912 | + 2.302585093 | +``` + +至此，我们完成了第一个 UDF 😊，并学会了简单的 debug 方法。 + +#### 示例二：异常处理 + +上面的 myfun 虽然测试测试通过了，但是有两个缺点： + +1. 这个标量函数只接受 1 列数据作为输入，如果用户传入了多列也不会抛异常。 + +```sql +taos> select myfun(v1, v2) from t; + myfun(v1, v2) | +============================ + 0.693147181 | + 1.609437912 | + 2.302585093 | +``` + +2. 没有处理 null 值。我们期望如果输入有 null，则会抛异常终止执行。 +因此 process 函数改进如下： + +```python +def process(block): + rows, cols = block.shape() + if cols > 1: + raise Exception(f"require 1 parameter but given {cols}") + return [ None if block.data(i, 0) is None else log(block.data(i, 0) ** 2 + 1) for i in range(rows)] +``` + +然后执行下面的语句更新已有的 UDF： + +```sql +create or replace function myfun as '/root/udf/myfun.py' outputtype double language 'Python'; +``` + +再传入 myfun 两个参数，就会执行失败了 + +```sql +taos> select myfun(v1, v2) from t; + +DB error: udf function execution failure (0.014643s) +``` + +但遗憾的是我们自定义的异常信息没有展示给用户，而是在插件的日志文件 /var/log/taos/taospyudf.log 中： + +```text +2023-05-24 23:21:06.790 ERROR [1666188] [doPyUdfScalarProc@507] call pyUdfScalar proc function. context 0x7faade26d180. error: Exception: require 1 parameter but given 2 + +At: + /var/lib/taos//.udf/myfun_3_1884e1281d9.py(12): process + +``` + +至此，我们学会了如何更新 UDF，并查看 UDF 输出的错误日志。 +（注：如果 UDF 更新后未生效，在 TDengine 3.0.5.0 以前（不含）的版本中需要重启 taosd，在 3.0.5.0 及之后的版本中不需要重启 taosd 即可生效。） + +#### 示例三： 接收 n 个参数的 UDF + +编写一个 UDF：输入（x1, x2, ..., xn）, 输出每个值和它们的序号的乘积的和： 1 * x1 + 2 * x2 + ... + n * xn。如果 x1 至 xn 中包含 null，则结果为 null。 +这个示例与示例一的区别是，可以接受任意多列作为输入，且要处理每一列的值。编写 UDF 文件 /root/udf/nsum.py： + +```python +def init(): + pass + + +def destroy(): + pass + + +def process(block): + rows, cols = block.shape() + result = [] + for i in range(rows): + total = 0 + for j in range(cols): + v = block.data(i, j) + if v is None: + total = None + break + total += (j + 1) * block.data(i, j) + result.append(total) + return result +``` + +创建 UDF： + +```sql +create function nsum as '/root/udf/nsum.py' outputtype double language 'Python'; +``` + +测试 UDF： + +```sql +taos> insert into t values('2023-05-25 09:09:15', 6, null, 8); +Insert OK, 1 row(s) affected (0.003675s) + +taos> select ts, v1, v2, v3, nsum(v1, v2, v3) from t; + ts | v1 | v2 | v3 | nsum(v1, v2, v3) | +================================================================================================ + 2023-05-01 12:13:14.000 | 1 | 2 | 3 | 14.000000000 | + 2023-05-03 08:09:10.000 | 2 | 3 | 4 | 20.000000000 | + 2023-05-10 07:06:05.000 | 3 | 4 | 5 | 26.000000000 | + 2023-05-25 09:09:15.000 | 6 | NULL | 8 | NULL | +Query OK, 4 row(s) in set (0.010653s) +``` + +#### 示例四：使用第三方库 + +编写一个 UDF，输入一个时间戳，输出距离这个时间最近的下一个周日。比如今天是 2023-05-25， 则下一个周日是 2023-05-28。 +完成这个函数要用到第三方库 momen。先安装这个库： + +```shell +pip3 install moment +``` + +然后编写 UDF 文件 /root/udf/nextsunday.py + +```python +import moment + + +def init(): + pass + + +def destroy(): + pass + + +def process(block): + rows, cols = block.shape() + if cols > 1: + raise Exception("require only 1 parameter") + if not type(block.data(0, 0)) is int: + raise Exception("type error") + return [moment.unix(block.data(i, 0)).replace(weekday=7).format('YYYY-MM-DD') + for i in range(rows)] +``` + +UDF 框架会将 TDengine 的 timestamp 类型映射为 Python 的 int 类型，所以这个函数只接受一个表示毫秒数的整数。process 方法先做参数检查，然后用 moment 包替换时间的星期为星期日，最后格式化输出。输出的字符串长度是固定的10个字符长，因此可以这样创建 UDF 函数： + +```sql +create function nextsunday as '/root/udf/nextsunday.py' outputtype binary(10) language 'Python'; +``` + +此时测试函数，如果你是用 systemctl 启动的 taosd，肯定会遇到错误： + +```sql +taos> select ts, nextsunday(ts) from t; + +DB error: udf function execution failure (1.123615s) +``` + +```shell + tail -20 taospyudf.log +2023-05-25 11:42:34.541 ERROR [1679419] [PyUdf::PyUdf@217] py udf load module failure. error ModuleNotFoundError: No module named 'moment' +``` + +这是因为 “moment” 所在位置不在 python udf 插件默认的库搜索路径中。怎么确认这一点呢？通过以下命令搜索 taospyudf.log: + +```shell +grep 'sys path' taospyudf.log | tail -1 +``` + +输出如下 + +```text +2023-05-25 10:58:48.554 INFO [1679419] [doPyOpen@592] python sys path: ['', '/lib/python38.zip', '/lib/python3.8', '/lib/python3.8/lib-dynload', '/lib/python3/dist-packages', '/var/lib/taos//.udf'] +``` + +发现 python udf 插件默认搜索的第三方库安装路径是： /lib/python3/dist-packages，而 moment 默认安装到了 /usr/local/lib/python3.8/dist-packages。下面我们修改 python udf 插件默认的库搜索路径。 +先打开 python3 命令行，查看当前的 sys.path + +```python +>>> import sys +>>> ":".join(sys.path) +'/usr/lib/python3.8:/usr/lib/python3.8/lib-dynload:/usr/local/lib/python3.8/dist-packages:/usr/lib/python3/dist-packages' +``` + +复制上面脚本的输出的字符串，然后编辑 /var/taos/taos.cfg 加入以下配置： + +```shell +UdfdLdLibPath /usr/lib/python3.8:/usr/lib/python3.8/lib-dynload:/usr/local/lib/python3.8/dist-packages:/usr/lib/python3/dist-packages +``` + +保存后执行 systemctl restart taosd, 再测试就不报错了： + +```sql +taos> select ts, nextsunday(ts) from t; + ts | nextsunday(ts) | +=========================================== + 2023-05-01 12:13:14.000 | 2023-05-07 | + 2023-05-03 08:09:10.000 | 2023-05-07 | + 2023-05-10 07:06:05.000 | 2023-05-14 | + 2023-05-25 09:09:15.000 | 2023-05-28 | +Query OK, 4 row(s) in set (1.011474s) +``` + +#### 示例五：聚合函数 + +编写一个聚合函数，计算某一列最大值和最小值的差。 +聚合函数与标量函数的区别是：标量函数是多行输入对应多个输出，聚合函数是多行输入对应一个输出。聚合函数的执行过程有点像经典的 map-reduce 框架的执行过程，框架把数据分成若干块，每个 mapper 处理一个块，reducer 再把 mapper 的结果做聚合。不一样的地方在于，对于 TDengine Python UDF 中的 reduce 函数既有 map 的功能又有 reduce 的功能。reduce 函数接受两个参数：一个是自己要处理的数据，一个是别的任务执行 reduce 函数的处理结果。如下面的示例 /root/udf/myspread.py: + +```python +import io +import math +import pickle + +LOG_FILE: io.TextIOBase = None + + +def init(): + global LOG_FILE + LOG_FILE = open("/var/log/taos/spread.log", "wt") + log("init function myspead success") + + +def log(o): + LOG_FILE.write(str(o) + '\n') + + +def destroy(): + log("close log file: spread.log") + LOG_FILE.close() + + +def start(): + return pickle.dumps((-math.inf, math.inf)) + + +def reduce(block, buf): + max_number, min_number = pickle.loads(buf) + log(f"initial max_number={max_number}, min_number={min_number}") + rows, _ = block.shape() + for i in range(rows): + v = block.data(i, 0) + if v > max_number: + log(f"max_number={v}") + max_number = v + if v < min_number: + log(f"min_number={v}") + min_number = v + return pickle.dumps((max_number, min_number)) + + +def finish(buf): + max_number, min_number = pickle.loads(buf) + return max_number - min_number +``` + +在这个示例中我们不光定义了一个聚合函数，还添加记录执行日志的功能，讲解如下： +1. init 函数不再是空函数，而是打开了一个文件用于写执行日志 +2. log 函数是记录日志的工具，自动将传入的对象转成字符串，加换行符输出 +3. destroy 函数用来在执行结束关闭文件 +4. start 返回了初始的 buffer，用来存聚合函数的中间结果，我们把最大值初始化为负无穷大，最小值初始化为正无穷大 +5. reduce 处理每个数据块并聚合结果 +6. finish 函数将最终的 buffer 转换成最终的输出 +执行下面的 SQL语句创建对应的 UDF： + +```sql +create or replace aggregate function myspread as '/root/udf/myspread.py' outputtype double bufsize 128 language 'Python'; +``` + +这个 SQL 语句与创建标量函数的 SQL 语句有两个重要区别： +1. 增加了 aggregate 关键字 +2. 增加了 bufsize 关键字，用来指定存储中间结果的内存大小，这个数值可以大于实际使用的数值。本例中间结果是两个浮点数组成的 tuple，序列化后实际占用大小只有 32 个字节，但指定的 bufsize 是128，可以用 python 命令行打印实际占用的字节数 + +```python +>>> len(pickle.dumps((12345.6789, 23456789.9877))) +32 +``` + +测试这个函数，可以看到 myspread 的输出结果和内置的 spread 函数的输出结果是一致的。 + +```sql +taos> select myspread(v1) from t; + myspread(v1) | +============================ + 5.000000000 | +Query OK, 1 row(s) in set (0.013486s) + +taos> select spread(v1) from t; + spread(v1) | +============================ + 5.000000000 | +Query OK, 1 row(s) in set (0.005501s) +``` + +最后，查看我们自己打印的执行日志，从日志可以看出，reduce 函数被执行了 3 次。执行过程中 max 值被更新了 4 次， min 值只被更新 1 次。 + +```shell +root@slave11 /var/log/taos $ cat spread.log +init function myspead success +initial max_number=-inf, min_number=inf +max_number=1 +min_number=1 +initial max_number=1, min_number=1 +max_number=2 +max_number=3 +initial max_number=3, min_number=1 +max_number=6 +close log file: spread.log +``` + +通过这个示例，我们学会了如何定义聚合函数，并打印自定义的日志信息。 + +### SQL 命令 + +1. 创建标量函数的语法 + +```sql +CREATE FUNCTION function_name AS library_path OUTPUTTYPE output_type LANGUAGE 'Python'; +``` + +2. 创建聚合函数的语法 + +```sql +CREATE AGGREGATE FUNCTION function_name library_path OUTPUTTYPE output_type LANGUAGE 'Python'; +``` + +3. 更新标量函数 + +```sql +CREATE OR REPLACE FUNCTION function_name AS OUTPUTTYPE int LANGUAGE 'Python'; +``` + +4. 更新聚合函数 + +```sql +CREATE OR REPLACE AGGREGATE FUNCTION function_name AS OUTPUTTYPE BUFSIZE buf_size int LANGUAGE 'Python'; +``` + +注意：如果加了 “AGGREGATE” 关键字，更新之后函数将被当作聚合函数，无论之前是什么类型的函数。相反，如果没有加 “AGGREGATE” 关键字，更新之后的函数将被当作标量函数，无论之前是什么类型的函数。 + +5. 查看函数信息 + + 同名的 UDF 每更新一次，版本号会增加 1。 + +```sql +select * from ins_functions \G; +``` + +6. 查看和删除已有的 UDF + +```sql +SHOW functions; +DROP FUNCTION function_name; +``` + + +上面的命令可以查看 UDF 的完整信息 -### Python UDF 示例代码 +### 更多 Python UDF 示例代码 #### 标量函数示例 [pybitand](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pybitand.py) pybitand 实现多列的按位与功能。如果只有一列，返回这一列。pybitand 忽略空值。 @@ -386,6 +871,17 @@ pyl2norm 实现了输入列的所有数据的二阶范数，即对每个数据 +#### 聚合函数示例 [pycumsum](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pycumsum.py) + +pycumsum 使用 numpy 计算输入列所有数据的累积和。 +
+pycumsum.py + +```c +{{#include tests/script/sh/pycumsum.py}} +``` + +
## 管理和使用 UDF 在使用 UDF 之前需要先将其加入到 TDengine 系统中。关于如何管理和使用 UDF，请参考[管理和使用 UDF](../12-taos-sql/26-udf.md) diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 35332a96022ceca87901708985d15a73e6e6f96d..36eacd26a4ee220b798a490a25b5570bc7497e67 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -32,25 +32,24 @@ TDengine 的 JDBC 驱动实现尽可能与关系型数据库驱动保持一致 原生连接支持的平台和 TDengine 客户端驱动支持的平台一致。 REST 连接支持所有能运行 Java 的平台。 -## 版本支持 - -请参考[版本支持列表](../#版本支持) - -## 最近更新记录 - -| taos-jdbcdriver 版本 | 主要变化 | -| :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | -| 3.2.1 | 新增功能：WebSocket 连接支持 schemaless 与 prepareStatement 写入。变更：consumer poll 返回结果集为 ConsumerRecord，可通过 value() 获取指定结果集数据。 | -| 3.2.0 | 存在连接问题，不推荐使用 | -| 3.1.0 | WebSocket 连接支持订阅功能 | -| 3.0.1 - 3.0.4 | 修复一些情况下结果集数据解析错误的问题。3.0.1 在 JDK 11 环境编译，JDK 8 环境下建议使用其他版本 | -| 3.0.0 | 支持 TDengine 3.0 | -| 2.0.42 | 修在 WebSocket 连接中 wasNull 接口返回值 | -| 2.0.41 | 修正 REST 连接中用户名和密码转码方式 | -| 2.0.39 - 2.0.40 | 增加 REST 连接/请求 超时设置 | -| 2.0.38 | JDBC REST 连接增加批量拉取功能 | -| 2.0.37 | 增加对 json tag 支持 | -| 2.0.36 | 增加对 schemaless 写入支持 | +## 版本历史 + +| taos-jdbcdriver 版本 | 主要变化 | TDengine 版本 | +| :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------: | +| 3.2.4 | 数据订阅在 WebSocket 连接下增加 enable.auto.commit 参数，以及 unsubscribe() 方法。 | - | +| 3.2.3 | 修复 ResultSet 在一些情况数据解析失败 | - | +| 3.2.2 | 新增功能：数据订阅支持 seek 功能。 | 3.0.5.0 及更高版本 | +| 3.2.1 | 新增功能：WebSocket 连接支持 schemaless 与 prepareStatement 写入。变更：consumer poll 返回结果集为 ConsumerRecord，可通过 value() 获取指定结果集数据。 | 3.0.3.0 及更高版本 | +| 3.2.0 | 存在连接问题，不推荐使用 | - | +| 3.1.0 | WebSocket 连接支持订阅功能 | - | +| 3.0.1 - 3.0.4 | 修复一些情况下结果集数据解析错误的问题。3.0.1 在 JDK 11 环境编译，JDK 8 环境下建议使用其他版本 | - | +| 3.0.0 | 支持 TDengine 3.0 | 3.0.0.0 及更高版本 | +| 2.0.42 | 修复 WebSocket 连接中 wasNull 接口返回值 | - | +| 2.0.41 | 修复 REST 连接中用户名和密码转码方式 | - | +| 2.0.39 - 2.0.40 | 增加 REST 连接/请求 超时设置 | - | +| 2.0.38 | JDBC REST 连接增加批量拉取功能 | - | +| 2.0.37 | 增加对 json tag 支持 | - | +| 2.0.36 | 增加对 schemaless 写入支持 | - | **注**：REST 连接中增加 `batchfetch` 参数并设置为 true，将开启 WebSocket 连接。 @@ -80,45 +79,47 @@ JDBC 连接器可能报错的错误码包括 4 种： 具体的错误码请参考： -| Error Code | Description | Suggested Actions | -| ---------- | --------------------------------------------------------------- | --------------------------------------------------------------------------------------- | -| 0x2301 | connection already closed | 连接已经关闭，检查连接情况，或重新创建连接去执行相关指令。 | -| 0x2302 | this operation is NOT supported currently! | 当前使用接口不支持，可以更换其他连接方式。 | -| 0x2303 | invalid variables | 参数不合法，请检查相应接口规范，调整参数类型及大小。 | -| 0x2304 | statement is closed | statement 已经关闭，请检查 statement 是否关闭后再次使用，或是连接是否正常。 | -| 0x2305 | resultSet is closed | resultSet 结果集已经释放，请检查 resultSet 是否释放后再次使用。 | -| 0x2306 | Batch is empty! | prepareStatement 添加参数后再执行 executeBatch。 | -| 0x2307 | Can not issue data manipulation statements with executeQuery() | 更新操作应该使用 executeUpdate()，而不是 executeQuery()。 | -| 0x2308 | Can not issue SELECT via executeUpdate() | 查询操作应该使用 executeQuery()，而不是 executeUpdate()。 | -| 0x230d | parameter index out of range | 参数越界，请检查参数的合理范围。 | -| 0x230e | connection already closed | 连接已经关闭，请检查 Connection 是否关闭后再次使用，或是连接是否正常。 | -| 0x230f | unknown sql type in tdengine | 请检查 TDengine 支持的 Data Type 类型。 | -| 0x2310 | can't register JDBC-JNI driver | 不能注册 JNI 驱动，请检查 url 是否填写正确。 | -| 0x2312 | url is not set | 请检查 REST 连接 url 是否填写正确。 | -| 0x2314 | numeric value out of range | 请检查获取结果集中数值类型是否使用了正确的接口。 | -| 0x2315 | unknown taos type in tdengine | 在 TDengine 数据类型与 JDBC 数据类型转换时，是否指定了正确的 TDengine 数据类型。 | -| 0x2317 | | REST 连接中使用了错误的请求类型。 | -| 0x2318 | | REST 连接中出现了数据传输异常，请检查网络情况并重试。 | -| 0x2319 | user is required | 创建连接时缺少用户名信息 | -| 0x231a | password is required | 创建连接时缺少密码信息 | -| 0x231c | httpEntity is null, sql: | REST 连接中执行出现异常 | -| 0x2350 | unknown error | 未知异常，请在 github 反馈给开发人员。 | -| 0x2352 | Unsupported encoding | 本地连接下指定了不支持的字符编码集 | -| 0x2353 | internal error of database, please see taoslog for more details | 本地连接执行 prepareStatement 时出现错误，请检查 taos log 进行问题定位。 | -| 0x2354 | JNI connection is NULL | 本地连接执行命令时，Connection 已经关闭。请检查与 TDengine 的连接情况。 | -| 0x2355 | JNI result set is NULL | 本地连接获取结果集，结果集异常，请检查连接情况，并重试。 | -| 0x2356 | invalid num of fields | 本地连接获取结果集的 meta 信息不匹配。 | -| 0x2357 | empty sql string | 填写正确的 SQL 进行执行。 | -| 0x2359 | JNI alloc memory failed, please see taoslog for more details | 本地连接分配内存错误，请检查 taos log 进行问题定位。 | -| 0x2371 | consumer properties must not be null! | 创建订阅时参数为空，请填写正确的参数。 | -| 0x2372 | configs contain empty key, failed to set consumer property | 参数 key 中包含空值，请填写正确的参数。 | -| 0x2373 | failed to set consumer property, | 参数 value 中包含空值，请填写正确的参数。 | -| 0x2375 | topic reference has been destroyed | 创建数据订阅过程中，topic 引用被释放。请检查与 TDengine 的连接情况。 | -| 0x2376 | failed to set consumer topic, topic name is empty | 创建数据订阅过程中，订阅 topic 名称为空。请检查指定的 topic 名称是否填写正确。 | -| 0x2377 | consumer reference has been destroyed | 订阅数据传输通道已经关闭，请检查与 TDengine 的连接情况。 | -| 0x2378 | consumer create error | 创建数据订阅失败，请根据错误信息检查 taos log 进行问题定位。 | -| - | can't create connection with server within | 通过增加参数 httpConnectTimeout 增加连接耗时，或是请检查与 taosAdapter 之间的连接情况。 | -| - | failed to complete the task within the specified time | 通过增加参数 messageWaitTimeout 增加执行耗时，或是请检查与 taosAdapter 之间的连接情况。 | +| Error Code | Description | Suggested Actions | +| ---------- | --------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | +| 0x2301 | connection already closed | 连接已经关闭，检查连接情况，或重新创建连接去执行相关指令。 | +| 0x2302 | this operation is NOT supported currently! | 当前使用接口不支持，可以更换其他连接方式。 | +| 0x2303 | invalid variables | 参数不合法，请检查相应接口规范，调整参数类型及大小。 | +| 0x2304 | statement is closed | statement 已经关闭，请检查 statement 是否关闭后再次使用，或是连接是否正常。 | +| 0x2305 | resultSet is closed | resultSet 结果集已经释放，请检查 resultSet 是否释放后再次使用。 | +| 0x2306 | Batch is empty! | prepareStatement 添加参数后再执行 executeBatch。 | +| 0x2307 | Can not issue data manipulation statements with executeQuery() | 更新操作应该使用 executeUpdate()，而不是 executeQuery()。 | +| 0x2308 | Can not issue SELECT via executeUpdate() | 查询操作应该使用 executeQuery()，而不是 executeUpdate()。 | +| 0x230d | parameter index out of range | 参数越界，请检查参数的合理范围。 | +| 0x230e | connection already closed | 连接已经关闭，请检查 Connection 是否关闭后再次使用，或是连接是否正常。 | +| 0x230f | unknown sql type in tdengine | 请检查 TDengine 支持的 Data Type 类型。 | +| 0x2310 | can't register JDBC-JNI driver | 不能注册 JNI 驱动，请检查 url 是否填写正确。 | +| 0x2312 | url is not set | 请检查 REST 连接 url 是否填写正确。 | +| 0x2314 | numeric value out of range | 请检查获取结果集中数值类型是否使用了正确的接口。 | +| 0x2315 | unknown taos type in tdengine | 在 TDengine 数据类型与 JDBC 数据类型转换时，是否指定了正确的 TDengine 数据类型。 | +| 0x2317 | | REST 连接中使用了错误的请求类型。 | +| 0x2318 | | REST 连接中出现了数据传输异常，请检查网络情况并重试。 | +| 0x2319 | user is required | 创建连接时缺少用户名信息 | +| 0x231a | password is required | 创建连接时缺少密码信息 | +| 0x231c | httpEntity is null, sql: | REST 连接中执行出现异常 | +| 0x231d | can't create connection with server within | 通过增加参数 httpConnectTimeout 增加连接耗时，或是请检查与 taosAdapter 之间的连接情况。 | +| 0x231e | failed to complete the task within the specified time | 通过增加参数 messageWaitTimeout 增加执行耗时，或是请检查与 taosAdapter 之间的连接情况。 | +| 0x2350 | unknown error | 未知异常，请在 github 反馈给开发人员。 | +| 0x2352 | Unsupported encoding | 本地连接下指定了不支持的字符编码集 | +| 0x2353 | internal error of database, please see taoslog for more details | 本地连接执行 prepareStatement 时出现错误，请检查 taos log 进行问题定位。 | +| 0x2354 | JNI connection is NULL | 本地连接执行命令时，Connection 已经关闭。请检查与 TDengine 的连接情况。 | +| 0x2355 | JNI result set is NULL | 本地连接获取结果集，结果集异常，请检查连接情况，并重试。 | +| 0x2356 | invalid num of fields | 本地连接获取结果集的 meta 信息不匹配。 | +| 0x2357 | empty sql string | 填写正确的 SQL 进行执行。 | +| 0x2359 | JNI alloc memory failed, please see taoslog for more details | 本地连接分配内存错误，请检查 taos log 进行问题定位。 | +| 0x2371 | consumer properties must not be null! | 创建订阅时参数为空，请填写正确的参数。 | +| 0x2372 | configs contain empty key, failed to set consumer property | 参数 key 中包含空值，请填写正确的参数。 | +| 0x2373 | failed to set consumer property, | 参数 value 中包含空值，请填写正确的参数。 | +| 0x2375 | topic reference has been destroyed | 创建数据订阅过程中，topic 引用被释放。请检查与 TDengine 的连接情况。 | +| 0x2376 | failed to set consumer topic, topic name is empty | 创建数据订阅过程中，订阅 topic 名称为空。请检查指定的 topic 名称是否填写正确。 | +| 0x2377 | consumer reference has been destroyed | 订阅数据传输通道已经关闭，请检查与 TDengine 的连接情况。 | +| 0x2378 | consumer create error | 创建数据订阅失败，请根据错误信息检查 taos log 进行问题定位。 | +| 0x2379 | seek offset must not be a negative number | seek 接口参数不能为负值，请使用正确的参数 | +| 0x237a | vGroup not found in result set | VGroup 没有分配给当前 consumer，由于 Rebalance 机制导致 Consumer 与 VGroup 不是绑定的关系 | - [TDengine Java Connector](https://github.com/taosdata/taos-connector-jdbc/blob/main/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java) @@ -169,7 +170,7 @@ Maven 项目中，在 pom.xml 中添加以下依赖： com.taosdata.jdbc taos-jdbcdriver - 3.2.1 + 3.2.2 ``` @@ -288,10 +289,11 @@ url 中的配置参数如下： - batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。JDBC REST 连接支持批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 - charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 -- httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。 -- httpSocketTimeout: socket 超时时间，单位 ms，默认值为 5000。仅在 batchfetch 设置为 false 时生效。 -- messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。 +- httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 60000。 +- httpSocketTimeout: socket 超时时间，单位 ms，默认值为 60000。仅在 batchfetch 设置为 false 时生效。 +- messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 60000。 仅在 batchfetch 设置为 true 时生效。 - useSSL: 连接中是否使用 SSL。 +- httpPoolSize: REST 并发请求大小，默认 20。 **注意**：部分配置项（比如：locale、timezone）在 REST 连接中不生效。 @@ -355,10 +357,11 @@ properties 中的配置参数如下： - TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。 - TSDBDriver.PROPERTY_KEY_LOCALE：仅在使用 JDBC 原生连接时生效。 客户端语言环境，默认值系统当前 locale。 - TSDBDriver.PROPERTY_KEY_TIME_ZONE：仅在使用 JDBC 原生连接时生效。 客户端使用的时区，默认值为系统当前时区。 -- TSDBDriver.HTTP_CONNECT_TIMEOUT: 连接超时时间，单位 ms， 默认值为 5000。仅在 REST 连接时生效。 -- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间，单位 ms，默认值为 5000。仅在 REST 连接且 batchfetch 设置为 false 时生效。 -- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。 +- TSDBDriver.HTTP_CONNECT_TIMEOUT: 连接超时时间，单位 ms， 默认值为 60000。仅在 REST 连接时生效。 +- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间，单位 ms，默认值为 60000。仅在 REST 连接且 batchfetch 设置为 false 时生效。 +- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 60000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。 - TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 REST 连接时生效。 +- TSDBDriver.HTTP_POOL_SIZE: REST 并发请求大小，默认 20。 此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](/reference/config/#仅客户端适用)。 ### 配置参数的优先级 @@ -423,6 +426,19 @@ while(resultSet.next()){ > 查询和操作关系型数据库一致，使用下标获取返回字段内容时从 1 开始，建议使用字段名称获取。 +### 执行带有 reqId 的 SQL + +此 reqId 可用于请求链路追踪。 + +```java +AbstractStatement aStmt = (AbstractStatement) connection.createStatement(); +aStmt.execute("create database if not exists db", 1L); +aStmt.executeUpdate("use db", 2L); +try (ResultSet rs = aStmt.executeQuery("select * from tb", 3L)) { + Timestamp ts = rs.getTimestamp(1); +} +``` + ### 通过参数绑定写入数据 TDengine 的 JDBC 原生连接实现大幅改进了参数绑定方式对数据写入（INSERT）场景的支持。采用这种方式写入数据时，能避免 SQL 语法解析的资源消耗，从而在很多情况下显著提升写入性能。 @@ -430,7 +446,7 @@ TDengine 的 JDBC 原生连接实现大幅改进了参数绑定方式对数据 **注意**： - JDBC REST 连接目前不支持参数绑定 -- 以下示例代码基于 taos-jdbcdriver-3.2.1 +- 以下示例代码基于 taos-jdbcdriver-3.2.4 - binary 类型数据需要调用 setString 方法，nchar 类型数据需要调用 setNString 方法 - 预处理语句中指定数据库与子表名称不要使用 `db.?`，应直接使用 `?`，然后在 setTableName 中指定数据库，如：`prepareStatement.setTableName("db.t1")`。 @@ -916,14 +932,15 @@ public class SchemalessWsTest { public static void main(String[] args) throws SQLException { final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata&batchfetch=true"; - Connection connection = DriverManager.getConnection(url); - init(connection); - - SchemalessWriter writer = new SchemalessWriter(connection, "test_ws_schemaless"); - writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS); - writer.write(telnetDemo, SchemalessProtocolType.TELNET, SchemalessTimestampType.MILLI_SECONDS); - writer.write(jsonDemo, SchemalessProtocolType.JSON, SchemalessTimestampType.SECONDS); - System.exit(0); + try(Connection connection = DriverManager.getConnection(url)){ + init(connection); + + try(SchemalessWriter writer = new SchemalessWriter(connection, "test_ws_schemaless")){ + writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS); + writer.write(telnetDemo, SchemalessProtocolType.TELNET, SchemalessTimestampType.MILLI_SECONDS); + writer.write(jsonDemo, SchemalessProtocolType.JSON, SchemalessTimestampType.SECONDS); + } + } } private static void init(Connection connection) throws SQLException { @@ -939,6 +956,14 @@ public class SchemalessWsTest { +### 执行带有 reqId 的无模式写入 + +此 reqId 可用于请求链路追踪。 + +```java +writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS, 1L); +``` + ### 数据订阅 TDengine Java 连接器支持订阅功能，应用 API 如下： @@ -962,6 +987,7 @@ statement.executeUpdate("create topic if not exists topic_speed as select ts, sp ```java Properties config = new Properties(); +config.setProperty("bootstrap.servers", "localhost:6030"); config.setProperty("enable.auto.commit", "true"); config.setProperty("group.id", "group1"); config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ResultDeserializer"); @@ -969,12 +995,14 @@ config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.Res TaosConsumer consumer = new TaosConsumer<>(config); ``` +- bootstrap.servers: TDengine 服务端所在的`ip:port`，如果使用 WebSocket 连接，则为 taosAdapter 所在的`ip:port`。 - enable.auto.commit: 是否允许自动提交。 - group.id: consumer: 所在的 group。 - value.deserializer: 结果集反序列化方法，可以继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer`，并指定结果集 bean，实现反序列化。也可以继承 `com.taosdata.jdbc.tmq.Deserializer`，根据 SQL 的 resultSet 自定义反序列化方式。 - td.connect.type: 连接方式。jni：表示使用动态库连接的方式，ws/WebSocket：表示使用 WebSocket 进行数据通信。默认为 jni 方式。 -- httpConnectTimeout：创建连接超时参数，单位 ms，默认为 5000 ms。仅在 WebSocket 连接下有效。 -- messageWaitTimeout：数据传输超时参数，单位 ms，默认为 10000 ms。仅在 WebSocket 连接下有效。 +- httpConnectTimeout: 创建连接超时参数，单位 ms，默认为 5000 ms。仅在 WebSocket 连接下有效。 +- messageWaitTimeout: 数据传输超时参数，单位 ms，默认为 10000 ms。仅在 WebSocket 连接下有效。 +- httpPoolSize: 同一个连接下最大并行请求数。仅在 WebSocket 连接下有效。 其他参数请参考：[Consumer 参数列表](../../../develop/tmq#创建-consumer-以及consumer-group) #### 订阅消费数据 @@ -991,6 +1019,42 @@ while(true) { `poll` 每次调用获取一个消息。 +#### 指定订阅 Offset + +```java +// 获取 offset +long position(TopicPartition partition) throws SQLException; +Map position(String topic) throws SQLException; +Map beginningOffsets(String topic) throws SQLException; +Map endOffsets(String topic) throws SQLException; + +// 指定下一次 poll 中使用的 offset +void seek(TopicPartition partition, long offset) throws SQLException; +``` + +示例代码： + +```java +String topic = "offset_seek_test"; +Map offset = null; +try (TaosConsumer consumer = new TaosConsumer<>(properties)) { + consumer.subscribe(Collections.singletonList(topic)); + for (int i = 0; i < 10; i++) { + if (i == 3) { + // Saving consumption position + offset = consumer.position(topic); + } + if (i == 5) { + // reset consumption to the previously saved position + for (Map.Entry entry : offset.entrySet()) { + consumer.seek(entry.getKey(), entry.getValue()); + } + } + ConsumerRecords records = consumer.poll(Duration.ofMillis(500)); + } +} +``` + #### 关闭订阅 ```java @@ -1016,10 +1080,19 @@ public abstract class ConsumerLoop { public ConsumerLoop() throws SQLException { Properties config = new Properties(); + config.setProperty("td.connect.type", "jni"); + config.setProperty("bootstrap.servers", "localhost:6030"); + config.setProperty("td.connect.user", "root"); + config.setProperty("td.connect.pass", "taosdata"); + config.setProperty("auto.offset.reset", "earliest"); config.setProperty("msg.with.table.name", "true"); config.setProperty("enable.auto.commit", "true"); + config.setProperty("auto.commit.interval.ms", "1000"); config.setProperty("group.id", "group1"); + config.setProperty("client.id", "1"); config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer"); + config.setProperty("value.deserializer.encoding", "UTF-8"); + config.setProperty("experimental.snapshot.enable", "true"); this.consumer = new TaosConsumer<>(config); this.topics = Collections.singletonList("topic_speed"); @@ -1093,12 +1166,19 @@ public abstract class ConsumerLoop { public ConsumerLoop() throws SQLException { Properties config = new Properties(); - config.setProperty("bootstrap.servers", "localhost:6041"); config.setProperty("td.connect.type", "ws"); + config.setProperty("bootstrap.servers", "localhost:6041"); + config.setProperty("td.connect.user", "root"); + config.setProperty("td.connect.pass", "taosdata"); + config.setProperty("auto.offset.reset", "earliest"); config.setProperty("msg.with.table.name", "true"); config.setProperty("enable.auto.commit", "true"); + config.setProperty("auto.commit.interval.ms", "1000"); config.setProperty("group.id", "group2"); + config.setProperty("client.id", "1"); config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer"); + config.setProperty("value.deserializer.encoding", "UTF-8"); + config.setProperty("experimental.snapshot.enable", "true"); this.consumer = new TaosConsumer<>(config); this.topics = Collections.singletonList("topic_speed"); @@ -1239,6 +1319,7 @@ public static void main(String[] args) throws Exception { - connectionPools：HikariCP, Druid, dbcp, c3p0 等连接池中使用 taos-jdbcdriver。 - SpringJdbcTemplate：Spring JdbcTemplate 中使用 taos-jdbcdriver。 - mybatisplus-demo：Springboot + Mybatis 中使用 taos-jdbcdriver。 +- consumer-demo：Consumer 消费 TDengine 数据示例，可通过参数控制消费速度。 请参考：[JDBC example](https://github.com/taosdata/TDengine/tree/3.0/examples/JDBC) diff --git a/docs/zh/08-connector/20-go.mdx b/docs/zh/08-connector/20-go.mdx index fd6df992b530fdce4427e34ef8bc227f449eaac7..90ef4d83cafa604fbe531a9f9ea0ece5b11b2df4 100644 --- a/docs/zh/08-connector/20-go.mdx +++ b/docs/zh/08-connector/20-go.mdx @@ -30,26 +30,46 @@ REST 连接支持所有能运行 Go 的平台。 ## 版本支持 -请参考[版本支持列表](../#版本支持) +请参考[版本支持列表](https://github.com/taosdata/driver-go#remind) -## 支持的功能特性 +## 处理异常 -### 原生连接 +如果是 TDengine 错误可以通过以下方式获取错误码和错误信息。 -“原生连接”指连接器通过 TDengine 客户端驱动（taosc）直接与 TDengine 运行实例建立的连接。支持的功能特性有： - -* 普通查询 -* 连续查询 -* 订阅 -* schemaless 接口 -* 参数绑定接口 - -### REST 连接 - -"REST 连接"指连接器通过 taosAdapter 组件提供的 REST API 与 TDengine 运行实例建立的连接。支持的功能特性有： +```go +// import "github.com/taosdata/driver-go/v3/errors" + if err != nil { + tError, is := err.(*errors.TaosError) + if is { + fmt.Println("errorCode:", int(tError.Code)) + fmt.Println("errorMessage:", tError.ErrStr) + } else { + fmt.Println(err.Error()) + } + } +``` -* 普通查询 -* 连续查询 +## TDengine DataType 和 Go DataType + +| TDengine DataType | Go Type | +|-------------------|-----------| +| TIMESTAMP | time.Time | +| TINYINT | int8 | +| SMALLINT | int16 | +| INT | int32 | +| BIGINT | int64 | +| TINYINT UNSIGNED | uint8 | +| SMALLINT UNSIGNED | uint16 | +| INT UNSIGNED | uint32 | +| BIGINT UNSIGNED | uint64 | +| FLOAT | float32 | +| DOUBLE | float64 | +| BOOL | bool | +| BINARY | string | +| NCHAR | string | +| JSON | []byte | + +**注意**：JSON 类型仅在 tag 中支持。 ## 安装步骤 @@ -63,32 +83,28 @@ REST 连接支持所有能运行 Go 的平台。 * ```go env``` * ```gcc -v``` -### 使用 go get 安装 - -`go get -u github.com/taosdata/driver-go/v3@latest` - -### 使用 go mod 管理 +### 安装连接器 1. 使用 `go mod` 命令初始化项目： - ```text - go mod init taos-demo - ``` + ```text + go mod init taos-demo + ``` 2. 引入 taosSql ： - ```go - import ( - "database/sql" - _ "github.com/taosdata/driver-go/v3/taosSql" - ) - ``` + ```go + import ( + "database/sql" + _ "github.com/taosdata/driver-go/v3/taosSql" + ) + ``` 3. 使用 `go mod tidy` 更新依赖包： - ```text - go mod tidy - ``` + ```text + go mod tidy + ``` 4. 使用 `go run taos-demo` 运行程序或使用 `go build` 命令编译出二进制文件。 @@ -99,8 +115,6 @@ REST 连接支持所有能运行 Go 的平台。 ## 建立连接 -### 数据源名称（DSN） - 数据源名称具有通用格式，例如 [PEAR DB](http://pear.php.net/manual/en/package.database.db.intro-dsn.php)，但没有类型前缀（方括号表示可选）： ``` text @@ -113,9 +127,7 @@ REST 连接支持所有能运行 Go 的平台。 username:password@protocol(address)/dbname?param=value ``` -### 使用连接器进行连接 - - + _taosSql_ 通过 cgo 实现了 Go 的 `database/sql/driver` 接口。只需要引入驱动就可以使用 [`database/sql`](https://golang.org/pkg/database/sql/) 的接口。 @@ -213,314 +225,900 @@ func main() { -## 使用示例 - -### 写入数据 +### 指定 URL 和 Properties 获取连接 -#### SQL 写入 +Go 连接器不支持此功能 - +### 配置参数的优先级 -#### InfluxDB 行协议写入 +Go 连接器不支持此功能 - +## 使用示例 -#### OpenTSDB Telnet 行协议写入 +### 创建数据库和表 - +```go +var taosDSN = "root:taosdata@tcp(localhost:6030)/" +taos, err := sql.Open("taosSql", taosDSN) +if err != nil { + log.Fatalln("failed to connect TDengine, err:", err) +} +defer taos.Close() +_, err := taos.Exec("CREATE DATABASE power") +if err != nil { + log.Fatalln("failed to create database, err:", err) +} +_, err = taos.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") +if err != nil { + log.Fatalln("failed to create stable, err:", err) +} +``` -#### OpenTSDB JSON 行协议写入 +### 插入数据 - + ### 查询数据 -### 更多示例程序 +### 执行带有 reqId 的 SQL -* [示例程序](https://github.com/taosdata/driver-go/tree/3.0/examples) -* [视频教程](https://www.taosdata.com/blog/2020/11/11/1951.html)。 +此 reqId 可用于请求链路追踪。 -## 使用限制 - -由于 REST 接口无状态所以 `use db` 语法不会生效，需要将 db 名称放到 SQL 语句中，如：`create table if not exists tb1 (ts timestamp, a int)`改为`create table if not exists test.tb1 (ts timestamp, a int)`否则将报错`[0x217] Database not specified or available`。 +```go +db, err := sql.Open("taosSql", "root:taosdata@tcp(localhost:6030)/") +if err != nil { + panic(err) +} +defer db.Close() +ctx := context.WithValue(context.Background(), common.ReqIDKey, common.GetReqID()) +_, err = db.ExecContext(ctx, "create database if not exists example_taos_sql") +if err != nil { + panic(err) +} +``` -也可以将 db 名称放到 DSN 中，将 `root:taosdata@http(localhost:6041)/` 改为 `root:taosdata@http(localhost:6041)/test`。当指定的 db 不存在时执行 `create database` 语句不会报错，而执行针对该 db 的其他查询或写入操作会报错。 +### 通过参数绑定写入数据 -完整示例如下： + + ```go package main import ( - "database/sql" - "fmt" "time" - _ "github.com/taosdata/driver-go/v3/taosRestful" + "github.com/taosdata/driver-go/v3/af" + "github.com/taosdata/driver-go/v3/common" + "github.com/taosdata/driver-go/v3/common/param" ) func main() { - var taosDSN = "root:taosdata@http(localhost:6041)/test" - taos, err := sql.Open("taosRestful", taosDSN) + db, err := af.Open("", "root", "taosdata", "", 0) if err != nil { - fmt.Println("failed to connect TDengine, err:", err) - return + panic(err) } - defer taos.Close() - taos.Exec("create database if not exists test") - taos.Exec("create table if not exists tb1 (ts timestamp, a int)") - _, err = taos.Exec("insert into tb1 values(now, 0)(now+1s,1)(now+2s,2)(now+3s,3)") + defer db.Close() + _, err = db.Exec("create database if not exists example_stmt") if err != nil { - fmt.Println("failed to insert, err:", err) - return + panic(err) } - rows, err := taos.Query("select * from tb1") + _, err = db.Exec("create table if not exists example_stmt.tb1(ts timestamp," + + "c1 bool," + + "c2 tinyint," + + "c3 smallint," + + "c4 int," + + "c5 bigint," + + "c6 tinyint unsigned," + + "c7 smallint unsigned," + + "c8 int unsigned," + + "c9 bigint unsigned," + + "c10 float," + + "c11 double," + + "c12 binary(20)," + + "c13 nchar(20)" + + ")") if err != nil { - fmt.Println("failed to select from table, err:", err) - return + panic(err) } - - defer rows.Close() - for rows.Next() { - var r struct { - ts time.Time - a int - } - err := rows.Scan(&r.ts, &r.a) - if err != nil { - fmt.Println("scan error:\n", err) - return - } - fmt.Println(r.ts, r.a) + stmt := db.InsertStmt() + err = stmt.Prepare("insert into example_stmt.tb1 values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + if err != nil { + panic(err) } + now := time.Now() + params := make([]*param.Param, 14) + params[0] = param.NewParam(2). + AddTimestamp(now, common.PrecisionMilliSecond). + AddTimestamp(now.Add(time.Second), common.PrecisionMilliSecond) + params[1] = param.NewParam(2).AddBool(true).AddNull() + params[2] = param.NewParam(2).AddTinyint(2).AddNull() + params[3] = param.NewParam(2).AddSmallint(3).AddNull() + params[4] = param.NewParam(2).AddInt(4).AddNull() + params[5] = param.NewParam(2).AddBigint(5).AddNull() + params[6] = param.NewParam(2).AddUTinyint(6).AddNull() + params[7] = param.NewParam(2).AddUSmallint(7).AddNull() + params[8] = param.NewParam(2).AddUInt(8).AddNull() + params[9] = param.NewParam(2).AddUBigint(9).AddNull() + params[10] = param.NewParam(2).AddFloat(10).AddNull() + params[11] = param.NewParam(2).AddDouble(11).AddNull() + params[12] = param.NewParam(2).AddBinary([]byte("binary")).AddNull() + params[13] = param.NewParam(2).AddNchar("nchar").AddNull() + + paramTypes := param.NewColumnType(14). + AddTimestamp(). + AddBool(). + AddTinyint(). + AddSmallint(). + AddInt(). + AddBigint(). + AddUTinyint(). + AddUSmallint(). + AddUInt(). + AddUBigint(). + AddFloat(). + AddDouble(). + AddBinary(6). + AddNchar(5) + err = stmt.BindParam(params, paramTypes) + if err != nil { + panic(err) + } + err = stmt.AddBatch() + if err != nil { + panic(err) + } + err = stmt.Execute() + if err != nil { + panic(err) + } + err = stmt.Close() + if err != nil { + panic(err) + } + // select * from example_stmt.tb1 } ``` -## 常见问题 - -1. database/sql 中 stmt（参数绑定）相关接口崩溃 - - REST 不支持参数绑定相关接口，建议使用`db.Exec`和`db.Query`。 - -2. 使用 `use db` 语句后执行其他语句报错 `[0x217] Database not specified or available` - - 在 REST 接口中 SQL 语句的执行无上下文关联，使用 `use db` 语句不会生效，解决办法见上方使用限制章节。 - -3. 使用 taosSql 不报错使用 taosRestful 报错 `[0x217] Database not specified or available` - - 因为 REST 接口无状态，使用 `use db` 语句不会生效，解决办法见上方使用限制章节。 - -4. `readBufferSize` 参数调大后无明显效果 - - `readBufferSize` 调大后会减少获取结果时 `syscall` 的调用。如果查询结果的数据量不大，修改该参数不会带来明显提升，如果该参数修改过大，瓶颈会在解析 JSON 数据。如果需要优化查询速度，需要根据实际情况调整该值来达到查询效果最优。 - -5. `disableCompression` 参数设置为 `false` 时查询效率降低 - - 当 `disableCompression` 参数设置为 `false` 时查询结果会使用 `gzip` 压缩后传输，拿到数据后要先进行 `gzip` 解压。 - -6. `go get` 命令无法获取包，或者获取包超时 - - 设置 Go 代理 `go env -w GOPROXY=https://goproxy.cn,direct`。 - -## 常用 API - -### database/sql API - -* `sql.Open(DRIVER_NAME string, dataSourceName string) *DB` - - 该 API 用来打开 DB，返回一个类型为 \*DB 的对象。 - -:::info -该 API 成功创建的时候，并没有做权限等检查，只有在真正执行 Query 或者 Exec 的时候才能真正的去创建连接，并同时检查 user/password/host/port 是不是合法。 -::: - -* `func (db *DB) Exec(query string, args ...interface{}) (Result, error)` - - `sql.Open` 内置的方法，用来执行非查询相关 SQL。 - -* `func (db *DB) Query(query string, args ...interface{}) (*Rows, error)` - - `sql.Open` 内置的方法，用来执行查询语句。 - -### 高级功能（af）API - -`af` 包封装了连接管理、订阅、schemaless、参数绑定等 TDengine 高级功能。 - -#### 连接管理 - -* `af.Open(host, user, pass, db string, port int) (*Connector, error)` - - 该 API 通过 cgo 创建与 taosd 的连接。 - -* `func (conn *Connector) Close() error` - - 关闭与 taosd 的连接。 - -#### 订阅 - -* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` - - 创建消费者。 - -* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` -注意：出于兼容目的保留 `rebalanceCb` 参数，当前未使用 - - 订阅单个主题。 - -* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` -注意：出于兼容目的保留 `rebalanceCb` 参数，当前未使用 - - 订阅主题。 - -* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` - - 轮询消息。 - -* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` -注意：出于兼容目的保留 `tmq.TopicPartition` 参数，当前未使用 - - 提交消息。 - -* `func (c *Consumer) Close() error` + + - 关闭连接。 +```go +package main -#### schemaless +import ( + "database/sql" + "fmt" + "time" -* `func (conn *Connector) InfluxDBInsertLines(lines []string, precision string) error` + "github.com/taosdata/driver-go/v3/common" + "github.com/taosdata/driver-go/v3/common/param" + _ "github.com/taosdata/driver-go/v3/taosRestful" + "github.com/taosdata/driver-go/v3/ws/stmt" +) - 写入 InfluxDB 行协议。 +func main() { + db, err := sql.Open("taosRestful", "root:taosdata@http(localhost:6041)/") + if err != nil { + panic(err) + } + defer db.Close() + prepareEnv(db) + + config := stmt.NewConfig("ws://127.0.0.1:6041/rest/stmt", 0) + config.SetConnectUser("root") + config.SetConnectPass("taosdata") + config.SetConnectDB("example_ws_stmt") + config.SetMessageTimeout(common.DefaultMessageTimeout) + config.SetWriteWait(common.DefaultWriteWait) + config.SetErrorHandler(func(connector *stmt.Connector, err error) { + panic(err) + }) + config.SetCloseHandler(func() { + fmt.Println("stmt connector closed") + }) + + connector, err := stmt.NewConnector(config) + if err != nil { + panic(err) + } + now := time.Now() + { + stmt, err := connector.Init() + if err != nil { + panic(err) + } + err = stmt.Prepare("insert into ? using all_json tags(?) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + if err != nil { + panic(err) + } + err = stmt.SetTableName("tb1") + if err != nil { + panic(err) + } + err = stmt.SetTags(param.NewParam(1).AddJson([]byte(`{"tb":1}`)), param.NewColumnType(1).AddJson(0)) + if err != nil { + panic(err) + } + params := []*param.Param{ + param.NewParam(3).AddTimestamp(now, 0).AddTimestamp(now.Add(time.Second), 0).AddTimestamp(now.Add(time.Second*2), 0), + param.NewParam(3).AddBool(true).AddNull().AddBool(true), + param.NewParam(3).AddTinyint(1).AddNull().AddTinyint(1), + param.NewParam(3).AddSmallint(1).AddNull().AddSmallint(1), + param.NewParam(3).AddInt(1).AddNull().AddInt(1), + param.NewParam(3).AddBigint(1).AddNull().AddBigint(1), + param.NewParam(3).AddUTinyint(1).AddNull().AddUTinyint(1), + param.NewParam(3).AddUSmallint(1).AddNull().AddUSmallint(1), + param.NewParam(3).AddUInt(1).AddNull().AddUInt(1), + param.NewParam(3).AddUBigint(1).AddNull().AddUBigint(1), + param.NewParam(3).AddFloat(1).AddNull().AddFloat(1), + param.NewParam(3).AddDouble(1).AddNull().AddDouble(1), + param.NewParam(3).AddBinary([]byte("test_binary")).AddNull().AddBinary([]byte("test_binary")), + param.NewParam(3).AddNchar("test_nchar").AddNull().AddNchar("test_nchar"), + } + paramTypes := param.NewColumnType(14). + AddTimestamp(). + AddBool(). + AddTinyint(). + AddSmallint(). + AddInt(). + AddBigint(). + AddUTinyint(). + AddUSmallint(). + AddUInt(). + AddUBigint(). + AddFloat(). + AddDouble(). + AddBinary(0). + AddNchar(0) + err = stmt.BindParam(params, paramTypes) + if err != nil { + panic(err) + } + err = stmt.AddBatch() + if err != nil { + panic(err) + } + err = stmt.Exec() + if err != nil { + panic(err) + } + affected := stmt.GetAffectedRows() + fmt.Println("all_json affected rows:", affected) + err = stmt.Close() + if err != nil { + panic(err) + } + } + { + stmt, err := connector.Init() + if err != nil { + panic(err) + } + err = stmt.Prepare("insert into ? using all_all tags(?,?,?,?,?,?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + err = stmt.SetTableName("tb1") + if err != nil { + panic(err) + } -* `func (conn *Connector) OpenTSDBInsertTelnetLines(lines []string) error` + err = stmt.SetTableName("tb2") + if err != nil { + panic(err) + } + err = stmt.SetTags( + param.NewParam(14). + AddTimestamp(now, 0). + AddBool(true). + AddTinyint(2). + AddSmallint(2). + AddInt(2). + AddBigint(2). + AddUTinyint(2). + AddUSmallint(2). + AddUInt(2). + AddUBigint(2). + AddFloat(2). + AddDouble(2). + AddBinary([]byte("tb2")). + AddNchar("tb2"), + param.NewColumnType(14). + AddTimestamp(). + AddBool(). + AddTinyint(). + AddSmallint(). + AddInt(). + AddBigint(). + AddUTinyint(). + AddUSmallint(). + AddUInt(). + AddUBigint(). + AddFloat(). + AddDouble(). + AddBinary(0). + AddNchar(0), + ) + if err != nil { + panic(err) + } + params := []*param.Param{ + param.NewParam(3).AddTimestamp(now, 0).AddTimestamp(now.Add(time.Second), 0).AddTimestamp(now.Add(time.Second*2), 0), + param.NewParam(3).AddBool(true).AddNull().AddBool(true), + param.NewParam(3).AddTinyint(1).AddNull().AddTinyint(1), + param.NewParam(3).AddSmallint(1).AddNull().AddSmallint(1), + param.NewParam(3).AddInt(1).AddNull().AddInt(1), + param.NewParam(3).AddBigint(1).AddNull().AddBigint(1), + param.NewParam(3).AddUTinyint(1).AddNull().AddUTinyint(1), + param.NewParam(3).AddUSmallint(1).AddNull().AddUSmallint(1), + param.NewParam(3).AddUInt(1).AddNull().AddUInt(1), + param.NewParam(3).AddUBigint(1).AddNull().AddUBigint(1), + param.NewParam(3).AddFloat(1).AddNull().AddFloat(1), + param.NewParam(3).AddDouble(1).AddNull().AddDouble(1), + param.NewParam(3).AddBinary([]byte("test_binary")).AddNull().AddBinary([]byte("test_binary")), + param.NewParam(3).AddNchar("test_nchar").AddNull().AddNchar("test_nchar"), + } + paramTypes := param.NewColumnType(14). + AddTimestamp(). + AddBool(). + AddTinyint(). + AddSmallint(). + AddInt(). + AddBigint(). + AddUTinyint(). + AddUSmallint(). + AddUInt(). + AddUBigint(). + AddFloat(). + AddDouble(). + AddBinary(0). + AddNchar(0) + err = stmt.BindParam(params, paramTypes) + if err != nil { + panic(err) + } + err = stmt.AddBatch() + if err != nil { + panic(err) + } + err = stmt.Exec() + if err != nil { + panic(err) + } + affected := stmt.GetAffectedRows() + fmt.Println("all_all affected rows:", affected) + err = stmt.Close() + if err != nil { + panic(err) + } - 写入 OpenTDSB telnet 协议数据。 + } +} -* `func (conn *Connector) OpenTSDBInsertJsonPayload(payload string) error` +func prepareEnv(db *sql.DB) { + steps := []string{ + "create database example_ws_stmt", + "create table example_ws_stmt.all_json(ts timestamp," + + "c1 bool," + + "c2 tinyint," + + "c3 smallint," + + "c4 int," + + "c5 bigint," + + "c6 tinyint unsigned," + + "c7 smallint unsigned," + + "c8 int unsigned," + + "c9 bigint unsigned," + + "c10 float," + + "c11 double," + + "c12 binary(20)," + + "c13 nchar(20)" + + ")" + + "tags(t json)", + "create table example_ws_stmt.all_all(" + + "ts timestamp," + + "c1 bool," + + "c2 tinyint," + + "c3 smallint," + + "c4 int," + + "c5 bigint," + + "c6 tinyint unsigned," + + "c7 smallint unsigned," + + "c8 int unsigned," + + "c9 bigint unsigned," + + "c10 float," + + "c11 double," + + "c12 binary(20)," + + "c13 nchar(20)" + + ")" + + "tags(" + + "tts timestamp," + + "tc1 bool," + + "tc2 tinyint," + + "tc3 smallint," + + "tc4 int," + + "tc5 bigint," + + "tc6 tinyint unsigned," + + "tc7 smallint unsigned," + + "tc8 int unsigned," + + "tc9 bigint unsigned," + + "tc10 float," + + "tc11 double," + + "tc12 binary(20)," + + "tc13 nchar(20))", + } + for _, step := range steps { + _, err := db.Exec(step) + if err != nil { + panic(err) + } + } +} - 写入 OpenTSDB JSON 协议数据。 +``` -#### 参数绑定 + + -* `func (conn *Connector) StmtExecute(sql string, params *param.Param) (res driver.Result, err error)` +### 无模式写入 - 参数绑定单行插入。 + + -* `func (conn *Connector) InsertStmt() *insertstmt.InsertStmt` +```go +import ( + "fmt" - 初始化参数。 + "github.com/taosdata/driver-go/v3/af" +) -* `func (stmt *InsertStmt) Prepare(sql string) error` +func main() { + conn, err := af.Open("localhost", "root", "taosdata", "", 6030) + if err != nil { + fmt.Println("fail to connect, err:", err) + } + defer conn.Close() + _, err = conn.Exec("create database if not exists example") + if err != nil { + panic(err) + } + _, err = conn.Exec("use example") + if err != nil { + panic(err) + } + influxdbData := "st,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000" + err = conn.InfluxDBInsertLines([]string{influxdbData}, "ns") + if err != nil { + panic(err) + } + telnetData := "stb0_0 1626006833 4 host=host0 interface=eth0" + err = conn.OpenTSDBInsertTelnetLines([]string{telnetData}) + if err != nil { + panic(err) + } + jsonData := "{\"metric\": \"meter_current\",\"timestamp\": 1626846400,\"value\": 10.3, \"tags\": {\"groupid\": 2, \"location\": \"California.SanFrancisco\", \"id\": \"d1001\"}}" + err = conn.OpenTSDBInsertJsonPayload(jsonData) + if err != nil { + panic(err) + } +} +``` - 参数绑定预处理 SQL 语句。 + + -* `func (stmt *InsertStmt) SetTableName(name string) error` +```go +import ( + "database/sql" + "log" + "time" - 参数绑定设置表名。 + "github.com/taosdata/driver-go/v3/common" + _ "github.com/taosdata/driver-go/v3/taosWS" + "github.com/taosdata/driver-go/v3/ws/schemaless" +) -* `func (stmt *InsertStmt) SetSubTableName(name string) error` +func main() { + db, err := sql.Open("taosWS", "root:taosdata@ws(localhost:6041)/") + if err != nil { + log.Fatal(err) + } + defer db.Close() + _, err = db.Exec("create database if not exists schemaless_ws") + if err != nil { + log.Fatal(err) + } + s, err := schemaless.NewSchemaless(schemaless.NewConfig("ws://localhost:6041/rest/schemaless", 1, + schemaless.SetDb("schemaless_ws"), + schemaless.SetReadTimeout(10*time.Second), + schemaless.SetWriteTimeout(10*time.Second), + schemaless.SetUser("root"), + schemaless.SetPassword("taosdata"), + schemaless.SetErrorHandler(func(err error) { + log.Fatal(err) + }), + )) + if err != nil { + panic(err) + } + influxdbData := "st,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000" + telnetData := "stb0_0 1626006833 4 host=host0 interface=eth0" + jsonData := "{\"metric\": \"meter_current\",\"timestamp\": 1626846400,\"value\": 10.3, \"tags\": {\"groupid\": 2, \"location\": \"California.SanFrancisco\", \"id\": \"d1001\"}}" - 参数绑定设置子表名。 + err = s.Insert(influxdbData, schemaless.InfluxDBLineProtocol, "ns", 0, common.GetReqID()) + if err != nil { + panic(err) + } + err = s.Insert(telnetData, schemaless.OpenTSDBTelnetLineProtocol, "ms", 0, common.GetReqID()) + if err != nil { + panic(err) + } + err = s.Insert(jsonData, schemaless.OpenTSDBJsonFormatProtocol, "ms", 0, common.GetReqID()) + if err != nil { + panic(err) + } +} +``` -* `func (stmt *InsertStmt) BindParam(params []*param.Param, bindType *param.ColumnType) error` + + - 参数绑定多行数据。 +### 执行带有 reqId 的无模式写入 -* `func (stmt *InsertStmt) AddBatch() error` +```go +func (s *Schemaless) Insert(lines string, protocol int, precision string, ttl int, reqID int64) error +``` - 添加到参数绑定批处理。 +可以通过 `common.GetReqID()` 获取唯一 id。 -* `func (stmt *InsertStmt) Execute() error` +### 数据订阅 - 执行参数绑定。 +TDengine Go 连接器支持订阅功能，应用 API 如下： -* `func (stmt *InsertStmt) GetAffectedRows() int` +#### 创建 Topic - 获取参数绑定插入受影响行数。 +```go + db, err := af.Open("", "root", "taosdata", "", 0) + if err != nil { + panic(err) + } + defer db.Close() + _, err = db.Exec("create database if not exists example_tmq WAL_RETENTION_PERIOD 86400") + if err != nil { + panic(err) + } + _, err = db.Exec("create topic if not exists example_tmq_topic as DATABASE example_tmq") + if err != nil { + panic(err) + } +``` -* `func (stmt *InsertStmt) Close() error` +#### 创建 Consumer - 结束参数绑定。 +```go + consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ + "group.id": "test", + "auto.offset.reset": "earliest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_client", + "enable.auto.commit": "false", + "msg.with.table.name": "true", + }) + if err != nil { + panic(err) + } +``` -### 通过 WebSocket 订阅 +#### 订阅消费数据 -* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` +```go + err = consumer.Subscribe("example_tmq_topic", nil) + if err != nil { + panic(err) + } + for i := 0; i < 5; i++ { + ev := consumer.Poll(500) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Printf("get message:%v\n", e) + case tmqcommon.Error: + fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() + } + } +``` - 创建消费者。 +#### 指定订阅 Offset -* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` -注意：出于兼容目的保留 `rebalanceCb` 参数，当前未使用 +```go + partitions, err := consumer.Assignment() + if err != nil { + panic(err) + } + for i := 0; i < len(partitions); i++ { + fmt.Println(partitions[i]) + err = consumer.Seek(tmqcommon.TopicPartition{ + Topic: partitions[i].Topic, + Partition: partitions[i].Partition, + Offset: 0, + }, 0) + if err != nil { + panic(err) + } + } +``` - 订阅单个主题。 +#### 关闭订阅 -* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` -注意：出于兼容目的保留 `rebalanceCb` 参数，当前未使用 +```go + err = consumer.Close() + if err != nil { + panic(err) + } +``` - 订阅主题。 +#### 完整示例 -* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` + + - 轮询消息。 +```go +package main -* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` -注意：出于兼容目的保留 `tmq.TopicPartition` 参数，当前未使用 +import ( + "fmt" + "os" - 提交消息。 + "github.com/taosdata/driver-go/v3/af" + "github.com/taosdata/driver-go/v3/af/tmq" + tmqcommon "github.com/taosdata/driver-go/v3/common/tmq" +) -* `func (c *Consumer) Close() error` +func main() { + db, err := af.Open("", "root", "taosdata", "", 0) + if err != nil { + panic(err) + } + defer db.Close() + _, err = db.Exec("create database if not exists example_tmq WAL_RETENTION_PERIOD 86400") + if err != nil { + panic(err) + } + _, err = db.Exec("create topic if not exists example_tmq_topic as DATABASE example_tmq") + if err != nil { + panic(err) + } + if err != nil { + panic(err) + } + consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ + "group.id": "test", + "auto.offset.reset": "earliest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_client", + "enable.auto.commit": "false", + "msg.with.table.name": "true", + }) + if err != nil { + panic(err) + } + err = consumer.Subscribe("example_tmq_topic", nil) + if err != nil { + panic(err) + } + _, err = db.Exec("create table example_tmq.t1 (ts timestamp,v int)") + if err != nil { + panic(err) + } + _, err = db.Exec("insert into example_tmq.t1 values(now,1)") + if err != nil { + panic(err) + } + for i := 0; i < 5; i++ { + ev := consumer.Poll(500) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Printf("get message:%v\n", e) + case tmqcommon.Error: + fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() + } + } + partitions, err := consumer.Assignment() + if err != nil { + panic(err) + } + for i := 0; i < len(partitions); i++ { + fmt.Println(partitions[i]) + err = consumer.Seek(tmqcommon.TopicPartition{ + Topic: partitions[i].Topic, + Partition: partitions[i].Partition, + Offset: 0, + }, 0) + if err != nil { + panic(err) + } + } - 关闭连接。 + partitions, err = consumer.Assignment() + if err != nil { + panic(err) + } + for i := 0; i < len(partitions); i++ { + fmt.Println(partitions[i]) + } -完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) + err = consumer.Close() + if err != nil { + panic(err) + } +} +``` -### 通过 WebSocket 进行参数绑定 + + -* `func NewConnector(config *Config) (*Connector, error)` +```go +package main - 创建连接。 +import ( + "database/sql" + "fmt" -* `func (c *Connector) Init() (*Stmt, error)` + "github.com/taosdata/driver-go/v3/common" + tmqcommon "github.com/taosdata/driver-go/v3/common/tmq" + _ "github.com/taosdata/driver-go/v3/taosRestful" + "github.com/taosdata/driver-go/v3/ws/tmq" +) - 初始化参数。 +func main() { + db, err := sql.Open("taosRestful", "root:taosdata@http(localhost:6041)/") + if err != nil { + panic(err) + } + defer db.Close() + prepareEnv(db) + consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ + "ws.url": "ws://127.0.0.1:6041/rest/tmq", + "ws.message.channelLen": uint(0), + "ws.message.timeout": common.DefaultMessageTimeout, + "ws.message.writeWait": common.DefaultWriteWait, + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "group.id": "example", + "client.id": "example_consumer", + "auto.offset.reset": "earliest", + }) + if err != nil { + panic(err) + } + err = consumer.Subscribe("example_ws_tmq_topic", nil) + if err != nil { + panic(err) + } + go func() { + _, err := db.Exec("create table example_ws_tmq.t_all(ts timestamp," + + "c1 bool," + + "c2 tinyint," + + "c3 smallint," + + "c4 int," + + "c5 bigint," + + "c6 tinyint unsigned," + + "c7 smallint unsigned," + + "c8 int unsigned," + + "c9 bigint unsigned," + + "c10 float," + + "c11 double," + + "c12 binary(20)," + + "c13 nchar(20)" + + ")") + if err != nil { + panic(err) + } + _, err = db.Exec("insert into example_ws_tmq.t_all values(now,true,2,3,4,5,6,7,8,9,10.123,11.123,'binary','nchar')") + if err != nil { + panic(err) + } + }() + for i := 0; i < 5; i++ { + ev := consumer.Poll(500) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Printf("get message:%v\n", e) + case tmqcommon.Error: + fmt.Printf("%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() + } + } + partitions, err := consumer.Assignment() + if err != nil { + panic(err) + } + for i := 0; i < len(partitions); i++ { + fmt.Println(partitions[i]) + err = consumer.Seek(tmqcommon.TopicPartition{ + Topic: partitions[i].Topic, + Partition: partitions[i].Partition, + Offset: 0, + }, 0) + if err != nil { + panic(err) + } + } -* `func (c *Connector) Close() error` + partitions, err = consumer.Assignment() + if err != nil { + panic(err) + } + for i := 0; i < len(partitions); i++ { + fmt.Println(partitions[i]) + } - 关闭连接。 + err = consumer.Close() + if err != nil { + panic(err) + } +} -* `func (s *Stmt) Prepare(sql string) error` +func prepareEnv(db *sql.DB) { + _, err := db.Exec("create database example_ws_tmq WAL_RETENTION_PERIOD 86400") + if err != nil { + panic(err) + } + _, err = db.Exec("create topic example_ws_tmq_topic as database example_ws_tmq") + if err != nil { + panic(err) + } +} +``` - 参数绑定预处理 SQL 语句。 + + -* `func (s *Stmt) SetTableName(name string) error` +### 更多示例程序 - 参数绑定设置表名。 +* [示例程序](https://github.com/taosdata/driver-go/tree/3.0/examples) +* [视频教程](https://www.taosdata.com/blog/2020/11/11/1951.html)。 -* `func (s *Stmt) SetTags(tags *param.Param, bindType *param.ColumnType) error` +## 常见问题 - 参数绑定设置标签。 +1. database/sql 中 stmt（参数绑定）相关接口崩溃 -* `func (s *Stmt) BindParam(params []*param.Param, bindType *param.ColumnType) error` + REST 不支持参数绑定相关接口，建议使用`db.Exec`和`db.Query`。 - 参数绑定多行数据。 +2. 使用 `use db` 语句后执行其他语句报错 `[0x217] Database not specified or available` -* `func (s *Stmt) AddBatch() error` + 在 REST 接口中 SQL 语句的执行无上下文关联，使用 `use db` 语句不会生效，解决办法见上方使用限制章节。 - 添加到参数绑定批处理。 +3. 使用 taosSql 不报错使用 taosRestful 报错 `[0x217] Database not specified or available` -* `func (s *Stmt) Exec() error` + 因为 REST 接口无状态，使用 `use db` 语句不会生效，解决办法见上方使用限制章节。 - 执行参数绑定。 +4. `readBufferSize` 参数调大后无明显效果 -* `func (s *Stmt) GetAffectedRows() int` + `readBufferSize` 调大后会减少获取结果时 `syscall` 的调用。如果查询结果的数据量不大，修改该参数不会带来明显提升，如果该参数修改过大，瓶颈会在解析 JSON 数据。如果需要优化查询速度，需要根据实际情况调整该值来达到查询效果最优。 - 获取参数绑定插入受影响行数。 +5. `disableCompression` 参数设置为 `false` 时查询效率降低 -* `func (s *Stmt) Close() error` + 当 `disableCompression` 参数设置为 `false` 时查询结果会使用 `gzip` 压缩后传输，拿到数据后要先进行 `gzip` 解压。 - 结束参数绑定。 +6. `go get` 命令无法获取包，或者获取包超时 -完整参数绑定示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/3.0/examples/stmtoverws/main.go) + 设置 Go 代理 `go env -w GOPROXY=https://goproxy.cn,direct`。 ## API 参考 diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index d4ca25be8109a2aa28d0805edcd0add5d052138b..3e51aa72bb85841e219f89c1b91b4ff4e4f791cc 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -26,20 +26,62 @@ import RustQuery from "../07-develop/04-query-data/_rust.mdx" 原生连接支持的平台和 TDengine 客户端驱动支持的平台一致。 Websocket 连接支持所有能运行 Rust 的平台。 -## 版本支持 +## 版本历史 -请参考[版本支持列表](../#版本支持) +| Rust 连接器版本 | TDengine 版本 | 主要功能 | +| :----------------: | :--------------: | :--------------------------------------------------: | +| v0.9.2 | 3.0.7.0 or later | STMT：ws 下获取 tag_fields、col_fields。 | +| v0.8.12 | 3.0.5.0 | 消息订阅：获取消费进度及按照指定进度开始消费。 | +| v0.8.0 | 3.0.4.0 | 支持无模式写入。 | +| v0.7.6 | 3.0.3.0 | 支持在请求中使用 req_id。 | +| v0.6.0 | 3.0.0.0 | 基础功能。 | Rust 连接器仍然在快速开发中，1.0 之前无法保证其向后兼容。建议使用 3.0 版本以上的 TDengine，以避免已知问题。 -## 安装 +## 处理错误 + +在报错后，可以获取到错误的具体信息： + +```rust +match conn.exec(sql) { + Ok(_) => { + Ok(()) + } + Err(e) => { + eprintln!("ERROR: {:?}", e); + Err(e) + } +} +``` + +## TDengine DataType 和 Rust DataType + +TDengine 目前支持时间戳、数字、字符、布尔类型，与 Rust 对应类型转换如下： + +| TDengine DataType | Rust DataType | +| ----------------- | ----------------- | +| TIMESTAMP | Timestamp | +| INT | i32 | +| BIGINT | i64 | +| FLOAT | f32 | +| DOUBLE | f64 | +| SMALLINT | i16 | +| TINYINT | i8 | +| BOOL | bool | +| BINARY | Vec | +| NCHAR | String | +| JSON | serde_json::Value | + +**注意**：JSON 类型仅在 tag 中支持。 + +## 安装步骤 ### 安装前准备 * 安装 Rust 开发工具链 * 如果使用原生连接，请安装 TDengine 客户端驱动，具体步骤请参考[安装客户端驱动](../#安装客户端驱动) -### 添加 taos 依赖 +### 安装连接器 根据选择的连接方式，按照如下说明在 [Rust](https://rust-lang.org) 项目中添加 [taos][taos] 依赖： @@ -65,6 +107,13 @@ taos = "*" taos = { version = "*", default-features = false, features = ["ws"] } ``` +当仅启用 `ws` 特性时，可同时指定 `r2d2` 使得在同步（blocking/sync）模式下使用 [r2d2] 作为连接池： + +```toml +[dependencies] +taos = { version = "*", default-features = false, features = ["r2d2", "ws"] } +``` + @@ -139,7 +188,8 @@ let builder = TaosBuilder::from_dsn("taos://localhost:6030")?; let conn1 = builder.build(); // use websocket protocol. -let conn2 = TaosBuilder::from_dsn("taos+ws://localhost:6041")?; +let builder2 = TaosBuilder::from_dsn("taos+ws://localhost:6041")?; +let conn2 = builder2.build(); ``` 建立连接后，您可以进行相关数据库操作： @@ -221,62 +271,210 @@ async fn demo(taos: &Taos, db: &str) -> Result<(), Error> { ## 使用示例 -### 写入数据 +### 创建数据库和表 + +```rust +use taos::*; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let dsn = "taos://localhost:6030"; + let builder = TaosBuilder::from_dsn(dsn)?; + + let taos = builder.build()?; + + let db = "query"; + + // create database + taos.exec_many([ + format!("DROP DATABASE IF EXISTS `{db}`"), + format!("CREATE DATABASE `{db}`"), + format!("USE `{db}`"), + ]) + .await?; + + // create table + taos.exec_many([ + // create super table + "CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) \ + TAGS (`groupid` INT, `location` BINARY(16))", + // create child table + "CREATE TABLE `d0` USING `meters` TAGS(0, 'Los Angles')", + ]).await?; +} +``` + +> **注意**：如果不使用 `use db` 指定数据库，则后续对表的操作都需要增加数据库名称作为前缀，如 db.tb。 -#### SQL 写入 +### 插入数据 -#### STMT 写入 +### 查询数据 + + + +### 执行带有 req_id 的 SQL + +此 req_id 可用于请求链路追踪。 + +```rust +let rs = taos.query_with_req_id("select * from stable where tag1 is null", 1)?; +``` + +### 通过参数绑定写入数据 + +TDengine 的 Rust 连接器实现了参数绑定方式对数据写入（INSERT）场景的支持。采用这种方式写入数据时，能避免 SQL 语法解析的资源消耗，从而在很多情况下显著提升写入性能。 + +参数绑定接口详见[API参考](#stmt-api) -#### Schemaless 写入 +### 无模式写入 + +TDengine 支持无模式写入功能。无模式写入兼容 InfluxDB 的 行协议（Line Protocol）、OpenTSDB 的 telnet 行协议和 OpenTSDB 的 JSON 格式协议。详情请参见[无模式写入](../../reference/schemaless/)。 -### 查询数据 +### 执行带有 req_id 的无模式写入 - +此 req_id 可用于请求链路追踪。 -## API 参考 +```rust +let sml_data = SmlDataBuilder::default() + .protocol(SchemalessProtocol::Line) + .data(data) + .req_id(100u64) + .build()?; -### 连接构造器 +client.put(&sml_data)? +``` + +### 数据订阅 + +TDengine 通过消息队列 [TMQ](../../../taos-sql/tmq/) 启动一个订阅。 -通过 DSN 来构建一个连接器构造器。 +#### 创建 Topic ```rust -let cfg = TaosBuilder::default().build()?; +taos.exec_many([ + // create topic for subscription + format!("CREATE TOPIC tmq_meters with META AS DATABASE {db}") +]) +.await?; ``` -使用 `builder` 对象创建多个连接： +#### 创建 Consumer + +从 DSN 开始，构建一个 TMQ 连接器。 ```rust -let conn: Taos = cfg.build(); +let tmq = TmqBuilder::from_dsn("taos://localhost:6030/?group.id=test")?; ``` -### 连接池 +创建消费者： + +```rust +let mut consumer = tmq.build()?; +``` -在复杂应用中，建议启用连接池。[taos] 的连接池使用 [r2d2] 实现。 +#### 订阅消费数据 -如下，可以生成一个默认参数的连接池。 +消费者可订阅一个或多个 `TOPIC`。 ```rust -let pool = TaosBuilder::from_dsn(dsn)?.pool()?; +consumer.subscribe(["tmq_meters"]).await?; ``` -同样可以使用连接池的构造器，对连接池参数进行设置： +TMQ 消息队列是一个 [futures::Stream](https://docs.rs/futures/latest/futures/stream/index.html) 类型，可以使用相应 API 对每个消息进行消费，并通过 `.commit` 进行已消费标记。 + +```rust +{ + let mut stream = consumer.stream(); + + while let Some((offset, message)) = stream.try_next().await? { + // get information from offset + + // the topic + let topic = offset.topic(); + // the vgroup id, like partition id in kafka. + let vgroup_id = offset.vgroup_id(); + println!("* in vgroup id {vgroup_id} of topic {topic}\n"); + + if let Some(data) = message.into_data() { + while let Some(block) = data.fetch_raw_block().await? { + // one block for one table, get table name if needed + let name = block.table_name(); + let records: Vec = block.deserialize().try_collect()?; + println!( + "** table: {}, got {} records: {:#?}\n", + name.unwrap(), + records.len(), + records + ); + } + } + consumer.commit(offset).await?; + } +} +``` + +获取消费进度： + +版本要求 connector-rust >= v0.8.8， TDengine >= 3.0.5.0 + +```rust +let assignments = consumer.assignments().await.unwrap(); +``` + +#### 指定订阅 Offset + +按照指定的进度消费： + +版本要求 connector-rust >= v0.8.8， TDengine >= 3.0.5.0 + +```rust +consumer.offset_seek(topic, vgroup_id, offset).await; +``` + +#### 关闭订阅 + +```rust +consumer.unsubscribe().await; +``` + +对于 TMQ DSN, 有以下配置项可以进行设置，需要注意的是，`group.id` 是必须的。 + +- `group.id`: 同一个消费者组，将以至少消费一次的方式进行消息负载均衡。 +- `client.id`: 可选的订阅客户端识别项。 +- `auto.offset.reset`: 可选初始化订阅起点， *earliest* 为从头开始订阅， *latest* 为仅从最新数据开始订阅，默认为从头订阅。注意，此选项在同一个 `group.id` 中仅生效一次。 +- `enable.auto.commit`: 当设置为 `true` 时，将启用自动标记模式，当对数据一致性不敏感时，可以启用此方式。 +- `auto.commit.interval.ms`: 自动标记的时间间隔。 + +#### 完整示例 + +完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/TDengine/blob/3.0/docs/examples/rust/nativeexample/examples/subscribe_demo.rs). + +### 与连接池使用 + +在复杂应用中，建议启用连接池。[taos] 的连接池默认（异步模式）使用 [deadpool] 实现。 + +如下，可以生成一个默认参数的连接池。 ```rust -let dsn = "taos://localhost:6030"; +let pool: Pool = TaosBuilder::from_dsn("taos:///") + .unwrap() + .pool() + .unwrap(); +``` -let opts = PoolBuilder::new() - .max_size(5000) // max connections - .max_lifetime(Some(Duration::from_secs(60 * 60))) // lifetime of each connection - .min_idle(Some(1000)) // minimal idle connections - .connection_timeout(Duration::from_secs(2)); +同样可以使用连接池的构造器，对连接池参数进行设置： -let pool = TaosBuilder::from_dsn(dsn)?.with_pool_builder(opts)?; +```rust +let pool: Pool = Pool::builder(Manager::from_dsn(self.dsn.clone()).unwrap().0) + .max_size(88) // 最大连接数 + .build() + .unwrap(); ``` 在应用代码中，使用 `pool.get()?` 来获取一个连接对象 [Taos]。 @@ -285,7 +483,17 @@ let pool = TaosBuilder::from_dsn(dsn)?.with_pool_builder(opts)?; let taos = pool.get()?; ``` -### 连接 +### 更多示例程序 + +示例程序源码位于 `TDengine/examples/rust` 下: + +请参考：[rust example](https://github.com/taosdata/TDengine/tree/3.0/examples/rust) + +## 常见问题 + +请参考 [FAQ](../../../train-faq/faq) + +## API 参考 [Taos][struct.Taos] 对象提供了多个数据库操作的 API： @@ -371,9 +579,13 @@ let taos = pool.get()?; - `.create_database(database: &str)`: 执行 `CREATE DATABASE` 语句。 - `.use_database(database: &str)`: 执行 `USE` 语句。 -除此之外，该结构也是 [参数绑定](#参数绑定接口) 和 [行协议接口](#行协议接口) 的入口，使用方法请参考具体的 API 说明。 +除此之外，该结构也是参数绑定和行协议接口的入口，使用方法请参考具体的 API 说明。 -### 参数绑定接口 +

+ +参数绑定接口 + +

与 C 接口类似，Rust 提供参数绑定接口。首先，通过 [Taos][struct.Taos] 对象创建一个 SQL 语句的参数绑定对象 [Stmt]： @@ -384,7 +596,7 @@ stmt.prepare("INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)")?; 参数绑定对象提供了一组接口用于实现参数绑定： -#### `.set_tbname(name)` +`.set_tbname(name)` 用于绑定表名。 @@ -393,7 +605,7 @@ let mut stmt = taos.stmt("insert into ? values(? ,?)")?; stmt.set_tbname("d0")?; ``` -#### `.set_tags(&[tag])` +`.set_tags(&[tag])` 当 SQL 语句使用超级表时，用于绑定子表表名和标签值： @@ -403,7 +615,7 @@ stmt.set_tbname("d0")?; stmt.set_tags(&[Value::VarChar("涛思".to_string())])?; ``` -#### `.bind(&[column])` +`.bind(&[column])` 用于绑定值类型。使用 [ColumnView] 结构体构建需要的类型并绑定： @@ -427,7 +639,7 @@ let params = vec![ let rows = stmt.bind(¶ms)?.add_batch()?.execute()?; ``` -#### `.execute()` +`.execute()` 执行 SQL。[Stmt] 对象可以复用，在执行后可以重新绑定并执行。执行前请确保所有数据已通过 `.add_batch` 加入到执行队列中。 @@ -442,80 +654,11 @@ stmt.execute()?; 一个可运行的示例请见 [GitHub 上的示例](https://github.com/taosdata/taos-connector-rust/blob/main/examples/bind.rs)。 -### 订阅 - -TDengine 通过消息队列 [TMQ](../../../taos-sql/tmq/) 启动一个订阅。 - -从 DSN 开始，构建一个 TMQ 连接器。 - -```rust -let tmq = TmqBuilder::from_dsn("taos://localhost:6030/?group.id=test")?; -``` - -创建消费者： - -```rust -let mut consumer = tmq.build()?; -``` - -消费者可订阅一个或多个 `TOPIC`。 - -```rust -consumer.subscribe(["tmq_meters"]).await?; -``` - -TMQ 消息队列是一个 [futures::Stream](https://docs.rs/futures/latest/futures/stream/index.html) 类型，可以使用相应 API 对每个消息进行消费，并通过 `.commit` 进行已消费标记。 - -```rust -{ - let mut stream = consumer.stream(); - - while let Some((offset, message)) = stream.try_next().await? { - // get information from offset - - // the topic - let topic = offset.topic(); - // the vgroup id, like partition id in kafka. - let vgroup_id = offset.vgroup_id(); - println!("* in vgroup id {vgroup_id} of topic {topic}\n"); - - if let Some(data) = message.into_data() { - while let Some(block) = data.fetch_raw_block().await? { - // one block for one table, get table name if needed - let name = block.table_name(); - let records: Vec = block.deserialize().try_collect()?; - println!( - "** table: {}, got {} records: {:#?}\n", - name.unwrap(), - records.len(), - records - ); - } - } - consumer.commit(offset).await?; - } -} -``` - -停止订阅： - -```rust -consumer.unsubscribe().await; -``` - -对于 TMQ DSN, 有以下配置项可以进行设置，需要注意的是，`group.id` 是必须的。 - -- `group.id`: 同一个消费者组，将以至少消费一次的方式进行消息负载均衡。 -- `client.id`: 可选的订阅客户端识别项。 -- `auto.offset.reset`: 可选初始化订阅起点， *earliest* 为从头开始订阅， *latest* 为仅从最新数据开始订阅，默认为从头订阅。注意，此选项在同一个 `group.id` 中仅生效一次。 -- `enable.auto.commit`: 当设置为 `true` 时，将启用自动标记模式，当对数据一致性不敏感时，可以启用此方式。 -- `auto.commit.interval.ms`: 自动标记的时间间隔。 - -完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/taos-connector-rust/blob/main/examples/subscribe.rs). 其他相关结构体 API 使用说明请移步 Rust 文档托管网页：。 [taos]: https://github.com/taosdata/rust-connector-taos +[deadpool]: https://crates.io/crates/deadpool [r2d2]: https://crates.io/crates/r2d2 [TaosBuilder]: https://docs.rs/taos/latest/taos/struct.TaosBuilder.html [TaosCfg]: https://docs.rs/taos/latest/taos/struct.TaosCfg.html diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index 1cff142e11d39e6afe86fab187697d222f37a9dd..ab98b5b8de5574278167be932398cff00f8ab9e5 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -1,4 +1,5 @@ --- +toc_max_heading_level: 4 sidebar_label: Python title: TDengine Python Connector description: "taospy 是 TDengine 的官方 Python 连接器。taospy 提供了丰富的 API， 使得 Python 应用可以很方便地使用 TDengine。tasopy 对 TDengine 的原生接口和 REST 接口都进行了封装， 分别对应 tasopy 的两个子模块：taos 和 taosrest。除了对原生接口和 REST 接口的封装，taospy 还提供了符合 Python 数据访问规范(PEP 249)的编程接口。这使得 taospy 和很多第三方工具集成变得简单，比如 SQLAlchemy 和 pandas" @@ -21,18 +22,76 @@ Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-con - 原生连接[支持的平台](../#支持的平台)和 TDengine 客户端支持的平台一致。 - REST 连接支持所有能运行 Python 的平台。 -## 版本选择 +### 支持的功能 + +- 原生连接支持 TDengine 的所有核心功能， 包括： 连接管理、执行 SQL、参数绑定、订阅、无模式写入（schemaless）。 +- REST 连接支持的功能包括：连接管理、执行 SQL。 (通过执行 SQL 可以： 管理数据库、管理表和超级表、写入数据、查询数据、创建连续查询等)。 + +## 历史版本 无论使用什么版本的 TDengine 都建议使用最新版本的 `taospy`。 -## 支持的功能 +|Python Connector 版本|主要变化| +|:-------------------:|:----:| +|2.7.9|数据订阅支持获取消费进度和重置消费进度| +|2.7.8|新增 `execute_many`| -- 原生连接支持 TDengine 的所有核心功能， 包括： 连接管理、执行 SQL、参数绑定、订阅、无模式写入（schemaless）。 -- REST 连接支持的功能包括：连接管理、执行 SQL。 (通过执行 SQL 可以： 管理数据库、管理表和超级表、写入数据、查询数据、创建连续查询等)。 +|Python Websocket Connector 版本|主要变化| +|:----------------------------:|:-----:| +|0.2.5|1. 数据订阅支持获取消费进度和重置消费进度
2. 支持 schemaless
3. 支持 STMT| +|0.2.4|数据订阅新增取消订阅方法| -## 安装 +## 处理异常 -### 准备 +Python 连接器可能会产生 4 种异常： + +- Python 连接器本身的异常 +- 原生连接方式的异常 +- websocket 连接方式异常 +- 数据订阅异常 +- TDengine 其他功能模块的异常 + +|Error Type|Description|Suggested Actions| +|:--------:|:---------:|:---------------:| +|InterfaceError|taosc 版本太低，不支持所使用的接口|请检查 TDengine 客户端版本| +|ConnectionError|数据库链接错误|请检查 TDengine 服务端状态和连接参数| +|DatabaseError|数据库错误|请检查 TDengine 服务端版本，并将 Python 连接器升级到最新版| +|OperationalError|操作错误|API 使用错误，请检查代码| +|ProgrammingError||| +|StatementError|stmt 相关异常|| +|ResultError||| +|SchemalessError|schemaless 相关异常|| +|TmqError|tmq 相关异常|| + +Python 中通常通过 try-expect 处理异常，异常处理相关请参考 [Python 错误和异常文档](https://docs.python.org/3/tutorial/errors.html)。 + +Python Connector 的所有数据库操作如果出现异常，都会直接抛出来。由应用程序负责异常处理。比如： + +```python +{{#include docs/examples/python/handle_exception.py}} +``` + +## TDengine DataType 和 Python DataType + +TDengine 目前支持时间戳、数字、字符、布尔类型，与 Python 对应类型转换如下： + +|TDengine DataType|Python DataType| +|:---------------:|:-------------:| +|TIMESTAMP|datetime| +|INT|int| +|BIGINT|int| +|FLOAT|float| +|DOUBLE|int| +|SMALLINT|int| +|TINYINT|int| +|BOOL|bool| +|BINARY|str| +|NCHAR|str| +|JSON|str| + +## 安装步骤 + +### 安装前准备 1. 安装 Python。新近版本 taospy 包要求 Python 3.6.2+。早期版本 taospy 包要求 Python 3.7+。taos-ws-py 包要求 Python 3.7+。如果系统上还没有 Python 可参考 [Python BeginnersGuide](https://wiki.python.org/moin/BeginnersGuide/Download) 安装。 2. 安装 [pip](https://pypi.org/project/pip/)。大部分情况下 Python 的安装包都自带了 pip 工具， 如果没有请参考 [pip documentation](https://pip.pypa.io/en/stable/installation/) 安装。 @@ -218,7 +277,7 @@ Transfer-Encoding: chunked -### 使用连接器建立连接 +### 指定 Host 和 Properties 获取连接 以下示例代码假设 TDengine 安装在本机， 且 FQDN 和 serverPort 都使用了默认配置。 @@ -274,7 +333,68 @@ Transfer-Encoding: chunked -## 示例程序 +### 配置参数的优先级 + +如果配置参数在参数和客户端配置文件中有重复，则参数的优先级由高到低分别如下： + +1. 连接参数 +2. 使用原生连接时，TDengine 客户端驱动的配置文件 taos.cfg + +## 使用示例 + +### 创建数据库和表 + + + + +```python +conn = taos.connect() +# Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. +conn.execute("DROP DATABASE IF EXISTS test") +conn.execute("CREATE DATABASE test") +# change database. same as execute "USE db" +conn.select_db("test") +conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") +``` + + + + + +```python +conn = taosrest.connect(url="http://localhost:6041") +# Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. +conn.execute("DROP DATABASE IF EXISTS test") +conn.execute("CREATE DATABASE test") +conn.execute("USE test") +conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") +``` + + + + + +```python +conn = taosws.connect("taosws://localhost:6041") +# Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. +conn.execute("DROP DATABASE IF EXISTS test") +conn.execute("CREATE DATABASE test") +conn.execute("USE test") +conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") +``` + + + + +### 插入数据 + +```python +conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m, 24.4)") +``` + +::: +now 为系统内部函数，默认为客户端所在计算机当前时间。 now + 1s 代表客户端当前时间往后加 1 秒，数字后面代表时间单位：a(毫秒)，s(秒)，m(分)，h(小时)，d(天)，w(周)，n(月)，y(年)。 +::: ### 基本使用 @@ -314,7 +434,6 @@ Transfer-Encoding: chunked :::note TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线程的场景下，这个游标实例必须保持线程独享，不能跨线程共享使用，否则会导致返回结果出现错误。 - ::: @@ -343,6 +462,10 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 +##### Connection 类的使用 + +`Connection` 类既包含对 PEP249 Connection 接口的实现(如：cursor方法和 close 方法)，也包含很多扩展功能（如： execute、 query、schemaless_insert 和 subscribe 方法。 + ```python {{#include docs/examples/python/connect_websocket_examples.py:basic}} ``` @@ -353,7 +476,47 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 -### 与 req_id 一起使用 +### 查询数据 + + + + +`TaosConnection` 类的 `query` 方法可以用来查询数据，返回 `TaosResult` 类型的结果数据。 + +```python +{{#include docs/examples/python/connection_usage_native_reference.py:query}} +``` + +:::tip +查询结果只能获取一次。比如上面的示例中 `fetch_all()` 和 `fetch_all_into_dict()` 只能用一个。重复获取得到的结果为空列表。 +::: + + + + + +RestClient 类是对于 REST API 的直接封装。它只包含一个 sql() 方法用于执行任意 SQL 语句， 并返回执行结果。 + +```python +{{#include docs/examples/python/rest_client_example.py}} +``` + +对于 `sql()` 方法更详细的介绍， 请参考 [RestClient](https://docs.taosdata.com/api/taospy/taosrest/restclient.html)。 + + + + + +`TaosConnection` 类的 `query` 方法可以用来查询数据，返回 `TaosResult` 类型的结果数据。 + +```python +{{#include docs/examples/python/connect_websocket_examples.py:basic}} +``` + + + + +### 执行带有 reqId 的 SQL 使用可选的 req_id 参数，指定请求 id，可以用于 tracing @@ -362,7 +525,7 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 ##### TaosConnection 类的使用 -`TaosConnection` 类既包含对 PEP249 Connection 接口的实现(如：`cursor`方法和 `close` 方法)，也包含很多扩展功能（如： `execute`、 `query`、`schemaless_insert` 和 `subscribe` 方法。 +类似上文介绍的使用方法，增加 `req_id` 参数。 ```python title="execute 方法" {{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:insert}} @@ -372,13 +535,9 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 {{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:query}} ``` -:::tip -查询结果只能获取一次。比如上面的示例中 `fetch_all()` 和 `fetch_all_into_dict()` 只能用一个。重复获取得到的结果为空列表。 -::: - ##### TaosResult 类的使用 -上面 `TaosConnection` 类的使用示例中，我们已经展示了两种获取查询结果的方法： `fetch_all()` 和 `fetch_all_into_dict()`。除此之外 `TaosResult` 还提供了按行迭代(`rows_iter`)或按数据块迭代(`blocks_iter`)结果集的方法。在查询数据量较大的场景，使用这两个方法会更高效。 +类似上文介绍的使用方法，增加 `req_id` 参数。 ```python title="blocks_iter 方法" {{#include docs/examples/python/result_set_with_req_id_examples.py}} @@ -391,14 +550,11 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 {{#include docs/examples/python/cursor_usage_native_reference_with_req_id.py}} ``` -:::note -TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线程的场景下，这个游标实例必须保持线程独享，不能跨线程共享使用，否则会导致返回结果出现错误。 - -::: - +类似上文介绍的使用方法，增加 `req_id` 参数。 + ##### TaosRestCursor 类的使用 `TaosRestCursor` 类是对 PEP249 Cursor 接口的实现。 @@ -420,8 +576,11 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 对于 `sql()` 方法更详细的介绍， 请参考 [RestClient](https://docs.taosdata.com/api/taospy/taosrest/restclient.html)。 + +类似上文介绍的使用方法，增加 `req_id` 参数。 + ```python {{#include docs/examples/python/connect_websocket_with_req_id_examples.py:basic}} ``` @@ -458,29 +617,138 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 -### 数据订阅 +### 通过参数绑定写入数据 -连接器支持数据订阅功能，数据订阅功能请参考 [数据订阅](../../develop/tmq/)。 +TDengine 的 Python 连接器支持参数绑定风格的 Prepare API 方式写入数据，和大多数数据库类似，目前仅支持用 `?` 来代表待绑定的参数。 - + -`Consumer` 提供了 Python 连接器订阅 TMQ 数据的 API，相关 API 定义请参考 [数据订阅文档](../../develop/tmq/#%E4%B8%BB%E8%A6%81%E6%95%B0%E6%8D%AE%E7%BB%93%E6%9E%84%E5%92%8C-api)。 +##### 创建 stmt -```python -{{#include docs/examples/python/tmq_example.py}} +Python 连接器的 `Connection` 提供了 `statement` 方法用于创建参数绑定对象 stmt，该方法接收 sql 字符串作为参数，sql 字符串目前仅支持用 `?` 来代表绑定的参数。 + +``` +import taos + +conn = taos.connect() +stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") +``` + +##### 参数绑定 + +调用 `new_multi_binds` 函数创建 params 列表，用于参数绑定。 + +``` +params = new_multi_binds(16) +params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) +params[1].bool((True, None, False)) +params[2].tinyint([-128, -128, None]) # -128 is tinyint null +params[3].tinyint([0, 127, None]) +params[4].smallint([3, None, 2]) +params[5].int([3, 4, None]) +params[6].bigint([3, 4, None]) +params[7].tinyint_unsigned([3, 4, None]) +params[8].smallint_unsigned([3, 4, None]) +params[9].int_unsigned([3, 4, None]) +params[10].bigint_unsigned([3, 4, None]) +params[11].float([3, None, 1]) +params[12].double([3, None, 1.2]) +params[13].binary(["abc", "dddafadfadfadfadfa", None]) +params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) +params[15].timestamp([None, None, 1626861392591]) +``` + +调用 stmt 的 `bind_param` 以单行的方式设置 values 或 `bind_param_batch` 以多行的方式设置 values 方法绑定参数。 + +``` +stmt.bind_param_batch(params) +``` + +##### 执行 sql + +调用 stmt 的 `execute` 方法执行 sql + +``` +stmt.execute() +``` + +##### 关闭 stmt + +最后需要关闭 stmt。 + +``` +stmt.close() ``` +##### 示例代码 + +```python +{{#include docs/examples/python/stmt_example.py}} +``` -除了原生的连接方式，Python 连接器还支持通过 websocket 订阅 TMQ 数据。 +##### 创建 stmt + +Python WebSocket 连接器的 `Connection` 提供了 `statement` 方法用于创建参数绑定对象 stmt，该方法接收 sql 字符串作为参数，sql 字符串目前仅支持用 `?` 来代表绑定的参数。 + +``` +import taosws + +conn = taosws.connect('taosws://localhost:6041/test') +stmt = conn.statement() +``` + +##### 解析 sql + +调用 stmt 的 `prepare` 方法来解析 insert 语句。 + +``` +stmt.prepare("insert into t1 values (?, ?, ?, ?)") +``` + +##### 参数绑定 + +调用 stmt 的 `bind_param` 方法绑定参数。 + +``` +stmt.bind_param([ + taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]), + taosws.ints_to_column([1, 2, 3, 4]), + taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]), + taosws.varchar_to_column(['a', 'b', 'c', 'd']), +]) +``` + +调用 stmt 的 `add_batch` 方法，将参数加入批处理。 + +``` +stmt.add_batch() +``` + +##### 执行 sql + +调用 stmt 的 `execute` 方法执行 sql -```python -{{#include docs/examples/python/tmq_websocket_example.py}} +``` +stmt.execute() +``` + +##### 关闭 stmt + +最后需要关闭 stmt。 + +``` +stmt.close() ``` +##### 示例代码 + +```python +{{#include docs/examples/python/stmt_websocket_example.py}} +``` @@ -491,19 +759,19 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 -简单写入 +##### 简单写入 ```python {{#include docs/examples/python/schemaless_insert.py}} ``` -带有 ttl 参数的写入 +##### 带有 ttl 参数的写入 ```python {{#include docs/examples/python/schemaless_insert_ttl.py}} ``` -带有 req_id 参数的写入 +##### 带有 req_id 参数的写入 ```python {{#include docs/examples/python/schemaless_insert_req_id.py}} @@ -513,19 +781,19 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 -简单写入 +##### 简单写入 ```python {{#include docs/examples/python/schemaless_insert_raw.py}} ``` -带有 ttl 参数的写入 +##### 带有 ttl 参数的写入 ```python {{#include docs/examples/python/schemaless_insert_raw_ttl.py}} ``` -带有 req_id 参数的写入 +##### 带有 req_id 参数的写入 ```python {{#include docs/examples/python/schemaless_insert_raw_req_id.py}} @@ -534,7 +802,215 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 -### 其它示例程序 +### 执行带有 reqId 的无模式写入 + +连接器的 `schemaless_insert` 和 `schemaless_insert_raw` 方法支持 `req_id` 可选参数，此 `req_Id` 可用于请求链路追踪。 + +```python +{{#include docs/examples/python/schemaless_insert_req_id.py}} +``` + +```python +{{#include docs/examples/python/schemaless_insert_raw_req_id.py}} +``` + +### 数据订阅 + +连接器支持数据订阅功能，数据订阅功能请参考 [数据订阅文档](../../develop/tmq/)。 + +#### 创建 Topic + +创建 Topic 相关请参考 [数据订阅文档](../../develop/tmq/#创建-topic)。 + +#### 创建 Consumer + + + + + +`Consumer` 提供了 Python 连接器订阅 TMQ 数据的 API。创建 Consumer 语法为 `consumer = Consumer(configs)`，参数定义请参考 [数据订阅文档](../../develop/tmq/#创建消费者-consumer)。 + +```python +from taos.tmq import Consumer + +consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) +``` + + + + +除了原生的连接方式，Python 连接器还支持通过 websocket 订阅 TMQ 数据，使用 websocket 方式订阅 TMQ 数据需要安装 `taos-ws-py`。 + +taosws `Consumer` API 提供了基于 Websocket 订阅 TMQ 数据的 API。创建 Consumer 语法为 `consumer = Consumer(conf=configs)`，使用时需要指定 `td.connect.websocket.scheme` 参数值为 "ws"，参数定义请参考 [数据订阅文档](../../develop/tmq/#%E5%88%9B%E5%BB%BA%E6%B6%88%E8%B4%B9%E8%80%85-consumer)。 + +```python +import taosws + +consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) +``` + + + + +#### 订阅 topics + + + + + +Consumer API 的 `subscribe` 方法用于订阅 topics，consumer 支持同时订阅多个 topic。 + +```python +consumer.subscribe(['topic1', 'topic2']) +``` + + + + +Consumer API 的 `subscribe` 方法用于订阅 topics，consumer 支持同时订阅多个 topic。 + +```python +consumer.subscribe(['topic1', 'topic2']) +``` + + + + +#### 消费数据 + + + + + +Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。消费者必须通过 Message 的 `error()` 方法校验返回数据的 error 信息。 + +```python +while True: + res = consumer.poll(1) + if not res: + continue + err = res.error() + if err is not None: + raise err + val = res.value() + + for block in val: + print(block.fetchall()) +``` + + + + +Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。消费者必须通过 Message 的 `error()` 方法校验返回数据的 error 信息。 + +```python +while True: + res = consumer.poll(timeout=1.0) + if not res: + continue + err = res.error() + if err is not None: + raise err + for block in message: + for row in block: + print(row) +``` + + + + +#### 获取消费进度 + + + + + +Consumer API 的 `assignment` 方法用于获取 Consumer 订阅的所有 topic 的消费进度，返回结果类型为 TopicPartition 列表。 + +```python +assignments = consumer.assignment() +``` + +Consumer API 的 `seek` 方法用于重置 Consumer 的消费进度到指定位置，方法参数类型为 TopicPartition。 + +```python +tp = TopicPartition(topic='topic1', partition=0, offset=0) +consumer.seek(tp) +``` + + + + +Consumer API 的 `assignment` 方法用于获取 Consumer 订阅的所有 topic 的消费进度，返回结果类型为 TopicPartition 列表。 + +```python +assignments = consumer.assignment() +``` + +Consumer API 的 `seek` 方法用于重置 Consumer 的消费进度到指定位置。 + +```python +consumer.seek(topic='topic1', partition=0, offset=0) +``` + + + + +#### 关闭订阅 + + + + + +消费结束后，应当取消订阅，并关闭 Consumer。 + +```python +consumer.unsubscribe() +consumer.close() +``` + + + + +消费结束后，应当取消订阅，并关闭 Consumer。 + +```python +consumer.unsubscribe() +consumer.close() +``` + + + + +#### 完整示例 + + + + + +```python +{{#include docs/examples/python/tmq_example.py}} +``` + +```python +{{#include docs/examples/python/tmq_assignment_example.py:taos_get_assignment_and_seek_demo}} +``` + + + + +```python +{{#include docs/examples/python/tmq_websocket_example.py}} +``` + +```python +{{#include docs/examples/python/tmq_websocket_assgnment_example.py:taosws_get_assignment_and_seek_demo}} +``` + + + + +### 更多示例程序 | 示例程序链接 | 示例程序内容 | | ------------------------------------------------------------------------------------------------------------- | ----------------------- | @@ -546,14 +1022,6 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 ## 其它说明 -### 异常处理 - -所有数据库操作如果出现异常，都会直接抛出来。由应用程序负责异常处理。比如： - -```python -{{#include docs/examples/python/handle_exception.py}} -``` -`` ### 关于纳秒 (nanosecond) 由于目前 Python 对 nanosecond 支持的不完善(见下面的链接)，目前的实现方式是在 nanosecond 精度时返回整数，而不是 ms 和 us 返回的 datetime 类型，应用开发者需要自行处理，建议使用 pandas 的 to_datetime()。未来如果 Python 正式完整支持了纳秒，Python 连接器可能会修改相关接口。 diff --git a/docs/zh/08-connector/43-r-lang.mdx b/docs/zh/08-connector/43-r-lang.mdx new file mode 100644 index 0000000000000000000000000000000000000000..a181f68abac0e79e3f6db512711e644709f8b2dd --- /dev/null +++ b/docs/zh/08-connector/43-r-lang.mdx @@ -0,0 +1,89 @@ +--- +toc_max_heading_level: 4 +sidebar_label: R +title: R Language Connector +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +import Rdemo from "../07-develop/01-connect/_connect_r.mdx" + +通过 R 语言中的 RJDBC 库可以使 R 语言程序支持访问 TDengine 数据。以下是安装过程、配置过程以及 R 语言示例代码。 + +## 安装过程 + +在开始之前，请确保已经安装了R语言环境。然后按照以下步骤安装和配置RJDBC库： + +1. 安装Java Development Kit (JDK)：RJDBC库需要依赖Java环境。请从Oracle官方网站下载适合您操作系统的JDK，并按照安装指南进行安装。 + +2. 安装RJDBC库：在R控制台中执行以下命令来安装RJDBC库。 + +```r +install.packages("RJDBC", repos='http://cran.us.r-project.org') +``` + +:::note +1. Ubuntu 系统自带的 R 语言软件版本 4.2 在调用 RJDBC 库会产生无响应 bug，请安装 R 语言[官网](https://www.r-project.org/)的安装包。 +2. 在 Linux 上安装 RJDBC 包可能需要安装编译需要的组件，以 Ubuntu 为例执行 `apt install -y libbz2-dev libpcre2-dev libicu-dev` 命令安装。 +3. 在 Windows 系统上需要设置 JAVA_HOME 环境变量。 +::: + +3. 下载 TDengine JDBC 驱动程序：访问 maven.org 网站，下载 TDengine JDBC 驱动程序（taos-jdbcdriver-X.X.X-dist.jar）。 + +4. 将 TDengine JDBC 驱动程序放置在适当的位置：在您的计算机上选择一个合适的位置，将 TDengine JDBC 驱动程序文件（taos-jdbcdriver-X.X.X-dist.jar）保存在此处。 + +## 配置过程 + +完成了安装步骤后，您需要进行一些配置，以便RJDBC库能够正确连接和访问TDengine时序数据库。 + +1. 在 R 脚本中加载 RJDBC 和其他必要的库： + +```r +library(DBI) +library(rJava) +library(RJDBC) +``` + +2. 设置 JDBC 驱动程序和 JDBC URL： + +```r +# 设置JDBC驱动程序路径（根据您实际保存的位置进行修改） +driverPath <- "/path/to/taos-jdbcdriver-X.X.X-dist.jar" + +# 设置JDBC URL（根据您的具体环境进行修改） +url <- "jdbc:TAOS://localhost:6030/?user=root&password=taosdata" +``` + +3. 加载 JDBC 驱动程序： + +```r +# 加载JDBC驱动程序 +drv <- JDBC("com.taosdata.jdbc.TSDBDriver", driverPath) +``` + +4. 创建 TDengine 数据库连接： + +```r +# 创建数据库连接 +conn <- dbConnect(drv, url) +``` + +5. 连接成功后，您可以使用 conn 对象进行各种数据库操作，如查询数据、插入数据等。 + +6. 最后，不要忘记在使用完成后关闭数据库连接： + +```r +# 关闭数据库连接 +dbDisconnect(conn) +``` + +## 使用 RJDBC 的 R 语言示例代码 + +以下是一个使用 RJDBC 库连接 TDengine 时序数据库并执行查询操作的示例代码： + + + +请根据您的实际情况修改JDBC驱动程序、JDBC URL、用户名、密码以及SQL查询语句，以适配您的 TDengine 时序数据库环境和要求。 + +通过以上步骤和示例代码，您可以在 R 语言环境中使用 RJDBC 库访问 TDengine 时序数据库，进行数据查询和分析等操作。 diff --git a/docs/zh/08-connector/_verify_windows.mdx b/docs/zh/08-connector/_verify_windows.mdx index 850fb5735de4ab4a094c4beb05c6bb6eb2f3e9f3..bd9547f93703318cf654a069ab85916ba4b7ff89 100644 --- a/docs/zh/08-connector/_verify_windows.mdx +++ b/docs/zh/08-connector/_verify_windows.mdx @@ -2,10 +2,10 @@ ```text taos> show databases; - name | create_time | vgroups | ntables | replica | strict | duration | keep | buffer | pagesize | pages | minrows | maxrows | comp | precision | status | retention | single_stable | cachemodel | cachesize | wal_level | wal_fsync_period | wal_retention_period | wal_retention_size | wal_roll_period | wal_seg_size | -========================================================================================================================================================================================================================================================================================================================================================================================================================================================================= - information_schema | NULL | NULL | 14 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | - performance_schema | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | + name | create_time | vgroups | ntables | replica | strict | duration | keep | buffer | pagesize | pages | minrows | maxrows | comp | precision | status | retention | single_stable | cachemodel | cachesize | wal_level | wal_fsync_period | wal_retention_period | wal_retention_size | +=============================================================================================================================================================================================================================================================================================================================================================================================================================== + information_schema | NULL | NULL | 14 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | + performance_schema | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | test | 2022-08-04 16:46:40.506 | 2 | 0 | 1 | off | 14400m | 5256000m,5256000m,5256000m | 96 | 4 | 256 | 100 | 4096 | 2 | ms | ready | NULL | false | none | 1 | 1 | 3000 | 0 | 0 | 0 | 0 | Query OK, 3 rows in database (0.123000s) diff --git a/docs/zh/08-connector/index.md b/docs/zh/08-connector/index.md index bb8c95a15aaa9e46b64e19d8d4b647edd83f5eac..6220a46b067b59ea2ce5c93ae03791de1aa0bbf0 100644 --- a/docs/zh/08-connector/index.md +++ b/docs/zh/08-connector/index.md @@ -45,9 +45,8 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 | **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **Schemaless** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 | :::info 由于不同编程语言数据库框架规范不同，并不意味着所有 C/C++ 接口都需要对应封装支持。 @@ -59,11 +58,10 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 | ------------------------------ | -------- | ---------- | -------- | -------- | ----------- | -------- | | **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **参数绑定** | 暂不支持 | 暂不支持 | 支持 | 支持 | 暂不支持 | 支持 | +| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 暂不支持 | 支持 | | **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 暂不支持 | 暂不支持 | 支持 | -| **Schemaless** | 支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | +| **Schemaless** | 支持 | 支持 | 支持 | 暂不支持 | 暂不支持 | 支持 | | **批量拉取（基于 WebSocket）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 | :::warning diff --git a/docs/zh/10-deployment/01-deploy.md b/docs/zh/10-deployment/01-deploy.md index b68bf7b743c8f5ee8b8a0b91919d2399cbd6bc3e..0ffbb8467b56d0cab5d12555e91c4c9212bcb1d6 100644 --- a/docs/zh/10-deployment/01-deploy.md +++ b/docs/zh/10-deployment/01-deploy.md @@ -62,12 +62,13 @@ serverPort 6030 加入到集群中的数据节点 dnode，下表中涉及集群相关的参数必须完全相同，否则不能成功加入到集群中。 -| **#** | **配置参数名称** | **含义** | -| ----- | ------------------ | ------------------------------------------- | -| 1 | statusInterval | dnode 向 mnode 报告状态时长 | -| 2 | timezone | 时区 | -| 3 | locale | 系统区位信息及编码格式 | -| 4 | charset | 字符集编码 | +| **#** | **配置参数名称** | **含义** | +| ----- | ---------------- | ------------------------------------ | +| 1 | statusInterval | dnode 向 mnode 报告状态时长 | +| 2 | timezone | 时区 | +| 3 | locale | 系统区位信息及编码格式 | +| 4 | charset | 字符集编码 | +| 5 | ttlChangeOnWrite | ttl 到期时间是否伴随表的修改操作改变 | ## 启动集群 @@ -196,10 +197,10 @@ dnodeID 是集群自动分配的，不得人工指定。它在生成时是递增 1、建立集群时使用 CREATE DNODE 增加新节点后，新节点始终显示 offline 状态？ ```sql 1）首先要检查增加的新节点上的 taosd 服务是否已经正常启动 - + 2）如果已经启动，再检查到新节点的网络是否通畅，可以使用 ping fqdn 验证下 - + 3）如果前面两步都没有问题，这一步要检查新节点做为独立集群在运行了，可以使用 taos -h fqdn 连接上后，show dnodes; 命令查看. - 如果显示的列表与你主节点上显示的不一致，说明此节点自己单独成立了一个集群，解决的方法是停止新节点上的服务，然后清空新节点上 + 如果显示的列表与你主节点上显示的不一致，说明此节点自己单独成立了一个集群，解决的方法是停止新节点上的服务，然后清空新节点上 taos.cfg 中配置的 dataDir 目录下的所有文件，重新启动新节点服务即可解决。 -``` +``` diff --git a/docs/zh/14-reference/11-docker/index.md b/docs/zh/10-deployment/02-docker.md similarity index 90% rename from docs/zh/14-reference/11-docker/index.md rename to docs/zh/10-deployment/02-docker.md index a6696977f9cb08da2eb8ff68c42c54d388e74980..3dbfabca7df75f9fc4513b62a9f3044f1521a282 100644 --- a/docs/zh/14-reference/11-docker/index.md +++ b/docs/zh/10-deployment/02-docker.md @@ -1,5 +1,6 @@ --- title: 用 Docker 部署 TDengine +sidebar_label: Docker description: '本章主要介绍如何在容器中启动 TDengine 服务并访问它' --- @@ -10,8 +11,17 @@ description: '本章主要介绍如何在容器中启动 TDengine 服务并访 TDengine 镜像启动时默认激活 HTTP 服务，使用下列命令 ```shell -docker run -d --name tdengine -p 6041:6041 tdengine/tdengine +docker run -d --name tdengine \ +-v ~/data/taos/dnode/data:/var/lib/taos \ +-v ~/data/taos/dnode/log:/var/log/taos \ +-p 6041:6041 tdengine/tdengine ``` +:::note + +- /var/lib/taos: TDengine 默认数据文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode/data为你自己的数据目录 +- /var/log/taos: TDengine 默认日志文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode/log为你自己的日志目录 + +::: 以上命令启动了一个名为“tdengine”的容器，并把其中的 HTTP 服务的端 6041 映射到了主机端口 6041。使用如下命令可以验证该容器中提供的 HTTP 服务是否可用： @@ -291,38 +301,37 @@ services: environment: TAOS_FQDN: "td-1" TAOS_FIRST_EP: "td-1" + ports: + - 6041:6041 + - 6030:6030 volumes: - - taosdata-td1:/var/lib/taos/ - - taoslog-td1:/var/log/taos/ + # /var/lib/taos: TDengine 默认数据文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode1/data为你自己的数据目录 + - ~/data/taos/dnode1/data:/var/lib/taos + # /var/log/taos: TDengine 默认日志文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode1/log为你自己的日志目录 + - ~/data/taos/dnode1/log:/var/log/taos td-2: image: tdengine/tdengine:$VERSION environment: TAOS_FQDN: "td-2" TAOS_FIRST_EP: "td-1" volumes: - - taosdata-td2:/var/lib/taos/ - - taoslog-td2:/var/log/taos/ + - ~/data/taos/dnode2/data:/var/lib/taos + - ~/data/taos/dnode2/log:/var/log/taos td-3: image: tdengine/tdengine:$VERSION environment: TAOS_FQDN: "td-3" TAOS_FIRST_EP: "td-1" volumes: - - taosdata-td3:/var/lib/taos/ - - taoslog-td3:/var/log/taos/ -volumes: - taosdata-td1: - taoslog-td1: - taosdata-td2: - taoslog-td2: - taosdata-td3: - taoslog-td3: + - ~/data/taos/dnode3/data:/var/lib/taos + - ~/data/taos/dnode3/log:/var/log/taos ``` :::note * `VERSION` 环境变量被用来设置 tdengine image tag * 在新创建的实例上必须设置 `TAOS_FIRST_EP` 以使其能够加入 TDengine 集群；如果有高可用需求，则需要同时使用 `TAOS_SECOND_EP` + ::: 2. 启动集群 @@ -397,24 +406,22 @@ networks: services: td-1: image: tdengine/tdengine:$VERSION - networks: - - inter environment: TAOS_FQDN: "td-1" TAOS_FIRST_EP: "td-1" volumes: - - taosdata-td1:/var/lib/taos/ - - taoslog-td1:/var/log/taos/ + # /var/lib/taos: TDengine 默认数据文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode1/data为你自己的数据目录 + - ~/data/taos/dnode1/data:/var/lib/taos + # /var/log/taos: TDengine 默认日志文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode1/log为你自己的日志目录 + - ~/data/taos/dnode1/log:/var/log/taos td-2: image: tdengine/tdengine:$VERSION - networks: - - inter environment: TAOS_FQDN: "td-2" TAOS_FIRST_EP: "td-1" volumes: - - taosdata-td2:/var/lib/taos/ - - taoslog-td2:/var/log/taos/ + - ~/data/taos/dnode2/data:/var/lib/taos + - ~/data/taos/dnode2/log:/var/log/taos adapter: image: tdengine/tdengine:$VERSION entrypoint: "taosadapter" @@ -446,11 +453,6 @@ services: >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf; nginx -g 'daemon off;'", ] -volumes: - taosdata-td1: - taoslog-td1: - taosdata-td2: - taoslog-td2: ``` ## 使用 docker swarm 部署 diff --git a/docs/zh/10-deployment/03-k8s.md b/docs/zh/10-deployment/03-k8s.md index 39ca56f3d978062f175ef8196ad1affe4bcd562b..16e2be0dfde51266783087bd8f456cfa36c5ec1f 100644 --- a/docs/zh/10-deployment/03-k8s.md +++ b/docs/zh/10-deployment/03-k8s.md @@ -4,23 +4,31 @@ title: 在 Kubernetes 上部署 TDengine 集群 description: 利用 Kubernetes 部署 TDengine 集群的详细指南 --- -作为面向云原生架构设计的时序数据库，TDengine 支持 Kubernetes 部署。这里介绍如何使用 YAML 文件一步一步从头创建一个 TDengine 集群，并重点介绍 Kubernetes 环境下 TDengine 的常用操作。 +## 概述 + +作为面向云原生架构设计的时序数据库，TDengine 本身就支持 Kubernetes 部署。这里介绍如何使用 YAML 文件从头一步一步创建一个可用于生产使用的高可用 TDengine 集群，并重点介绍 Kubernetes 环境下 TDengine 的常用操作。 + +为了满足[高可用](https://docs.taosdata.com/tdinternal/high-availability/)的需求，集群需要满足如下要求： + +- 3个及以上 dnode ：TDengine 的同一个 vgroup 中的多个 vnode ，不允许同时分布在一个 dnode ，所以如果创建3副本的数据库，则 dnode 数大于等于3 +- 3个 mnode ：mnode 负责整个集群的管理工作，TDengine 默认是一个 mnode。如果这个 mnode 所在的 dnode 掉线，则整个集群不可用。 +- 数据库的3副本：TDengine 的副本配置是数据库级别，所以数据库3副本可满足在3个 dnode 的集群中，任意一个 dnode 下线，都不影响集群的正常使用。**如果下线** **dnode** **个数为2时，此时集群不可用，****因为****RAFT无法完成选举****。**（企业版：在灾难恢复场景，任一节点数据文件损坏，都可以通过重新拉起dnode进行恢复） ## 前置条件 要使用 Kubernetes 部署管理 TDengine 集群，需要做好如下准备工作。 -* 本文适用 Kubernetes v1.5 以上版本 -* 本文和下一章使用 minikube、kubectl 和 helm 等工具进行安装部署，请提前安装好相应软件 -* Kubernetes 已经安装部署并能正常访问使用或更新必要的容器仓库或其他服务 +- 本文适用 Kubernetes v1.19 以上版本 +- 本文使用 kubectl 工具进行安装部署，请提前安装好相应软件 +- Kubernetes 已经安装部署并能正常访问使用或更新必要的容器仓库或其他服务 以下配置文件也可以从 [GitHub 仓库](https://github.com/taosdata/TDengine-Operator/tree/3.0/src/tdengine) 下载。 ## 配置 Service 服务 -创建一个 Service 配置文件：`taosd-service.yaml`，服务名称 `metadata.name` (此处为 "taosd") 将在下一步中使用到。添加 TDengine 所用到的端口： +创建一个 Service 配置文件：`taosd-service.yaml`，服务名称 `metadata.name` (此处为 "taosd") 将在下一步中使用到。首先添加 TDengine 所用到的端口，然后在选择器设置确定的标签 app (此处为 “tdengine”)。 -```yaml +```YAML --- apiVersion: v1 kind: Service @@ -42,10 +50,11 @@ spec: ## 有状态服务 StatefulSet -根据 Kubernetes 对各类部署的说明，我们将使用 StatefulSet 作为 TDengine 的服务类型。 -创建文件 `tdengine.yaml`，其中 replicas 定义集群节点的数量为 3。节点时区为中国（Asia/Shanghai），每个节点分配 10G 标准（standard）存储。你也可以根据实际情况进行相应修改。 +根据 Kubernetes 对各类部署的说明，我们将使用 StatefulSet 作为 TDengine 的部署资源类型。 创建文件 `tdengine.yaml`，其中 replicas 定义集群节点的数量为 3。节点时区为中国（Asia/Shanghai），每个节点分配 5G 标准（standard）存储（参考[Storage Classes](https://kubernetes.io/docs/concepts/storage/storage-classes/) 配置 storage class ）。你也可以根据实际情况进行相应修改。 + +请特别注意startupProbe的配置，在 dnode 的 Pod 掉线一段时间后，再重新启动，这个时候新上线的 dnode 会短暂不可用。如果startupProbe配置过小，Kubernetes 会认为该 Pod 处于不正常的状态，并尝试重启该 Pod，该 dnode 的 Pod 会频繁重启，始终无法恢复到正常状态。参考 [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) -```yaml +```YAML --- apiVersion: apps/v1 kind: StatefulSet @@ -69,7 +78,7 @@ spec: spec: containers: - name: "tdengine" - image: "tdengine/tdengine:3.0.0.0" + image: "tdengine/tdengine:3.0.7.1" imagePullPolicy: "IfNotPresent" ports: - name: tcp6030 @@ -108,6 +117,12 @@ spec: volumeMounts: - name: taosdata mountPath: /var/lib/taos + startupProbe: + exec: + command: + - taos-check + failureThreshold: 360 + periodSeconds: 10 readinessProbe: exec: command: @@ -129,199 +144,373 @@ spec: storageClassName: "standard" resources: requests: - storage: "10Gi" + storage: "5Gi" ``` ## 使用 kubectl 命令部署 TDengine 集群 -顺序执行以下命令。 +首先创建对应的 namespace，然后顺序执行以下命令： -```bash -kubectl apply -f taosd-service.yaml -kubectl apply -f tdengine.yaml +```Bash +kubectl apply -f taosd-service.yaml -n tdengine-test +kubectl apply -f tdengine.yaml -n tdengine-test ``` 上面的配置将生成一个三节点的 TDengine 集群，dnode 为自动配置，可以使用 show dnodes 命令查看当前集群的节点： -```bash -kubectl exec -i -t tdengine-0 -- taos -s "show dnodes" -kubectl exec -i -t tdengine-1 -- taos -s "show dnodes" -kubectl exec -i -t tdengine-2 -- taos -s "show dnodes" +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show dnodes" +kubectl exec -it tdengine-2 -n tdengine-test -- taos -s "show dnodes" ``` 输出如下： -``` +```Bash taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | -Query OK, 3 rows in database (0.003655s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 0 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-19 17:54:18.469 | | | | + 2 | tdengine-1.ta... | 0 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-19 17:54:38.698 | | | | + 3 | tdengine-2.ta... | 0 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-19 17:55:02.039 | | | | +Query OK, 3 row(s) in set (0.001853s) +``` + +查看当前mnode + +```Bash +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G" +taos> show mnodes\G +*************************** 1.row *************************** + id: 1 + endpoint: tdengine-0.taosd.tdengine-test.svc.cluster.local:6030 + role: leader + status: ready +create_time: 2023-07-19 17:54:18.559 +reboot_time: 2023-07-19 17:54:19.520 +Query OK, 1 row(s) in set (0.001282s) +``` + +## 创建mnode + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "create mnode on dnode 2" +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "create mnode on dnode 3" +``` + +查看mnode + +```Bash +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G" + +taos> show mnodes\G +*************************** 1.row *************************** + id: 1 + endpoint: tdengine-0.taosd.tdengine-test.svc.cluster.local:6030 + role: leader + status: ready +create_time: 2023-07-19 17:54:18.559 +reboot_time: 2023-07-20 09:19:36.060 +*************************** 2.row *************************** + id: 2 + endpoint: tdengine-1.taosd.tdengine-test.svc.cluster.local:6030 + role: follower + status: ready +create_time: 2023-07-20 09:22:05.600 +reboot_time: 2023-07-20 09:22:12.838 +*************************** 3.row *************************** + id: 3 + endpoint: tdengine-2.taosd.tdengine-test.svc.cluster.local:6030 + role: follower + status: ready +create_time: 2023-07-20 09:22:20.042 +reboot_time: 2023-07-20 09:22:23.271 +Query OK, 3 row(s) in set (0.003108s) ``` ## 使能端口转发 利用 kubectl 端口转发功能可以使应用可以访问 Kubernetes 环境运行的 TDengine 集群。 -``` -kubectl port-forward tdengine-0 6041:6041 & +```Plain +kubectl port-forward -n tdengine-test tdengine-0 6041:6041 & ``` 使用 curl 命令验证 TDengine REST API 使用的 6041 接口。 -``` -$ curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql -Handling connection for 6041 -{"code":0,"column_meta":[["name","VARCHAR",64],["create_time","TIMESTAMP",8],["vgroups","SMALLINT",2],["ntables","BIGINT",8],["replica","TINYINT",1],["strict","VARCHAR",4],["duration","VARCHAR",10],["keep","VARCHAR",32],["buffer","INT",4],["pagesize","INT",4],["pages","INT",4],["minrows","INT",4],["maxrows","INT",4],["comp","TINYINT",1],["precision","VARCHAR",2],["status","VARCHAR",10],["retention","VARCHAR",60],["single_stable","BOOL",1],["cachemodel","VARCHAR",11],["cachesize","INT",4],["wal_level","TINYINT",1],["wal_fsync_period","INT",4],["wal_retention_period","INT",4],["wal_retention_size","BIGINT",8],["wal_roll_period","INT",4],["wal_segment_size","BIGINT",8]],"data":[["information_schema",null,null,16,null,null,null,null,null,null,null,null,null,null,null,"ready",null,null,null,null,null,null,null,null,null,null],["performance_schema",null,null,10,null,null,null,null,null,null,null,null,null,null,null,"ready",null,null,null,null,null,null,null,null,null,null]],"rows":2} +```Plain +curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql +{"code":0,"column_meta":[["name","VARCHAR",64]],"data":[["information_schema"],["performance_schema"],["test"],["test1"]],"rows":4} ``` -## 使用 dashboard 进行图形化管理 +## 集群测试 - minikube 提供 dashboard 命令支持图形化管理界面。 +### 数据准备 -``` -$ minikube dashboard -* Verifying dashboard health ... -* Launching proxy ... -* Verifying proxy health ... -* Opening http://127.0.0.1:46617/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/ in your default browser... -http://127.0.0.1:46617/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/ -``` +#### taosBenchmark -对于某些公有云环境，minikube 绑定在 127.0.0.1 IP 地址上无法通过远程访问，需要使用 kubectl proxy 命令将端口映射到 0.0.0.0 IP 地址上，再通过浏览器访问虚拟机公网 IP 和端口以及相同的 dashboard URL 路径即可远程访问 dashboard。 +通过taosBenchmark 创建一个3副本的数据库，同时写入1亿条数据，同时查看数据 +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taosBenchmark -I stmt -d test -n 10000 -t 10000 -a 3 + +# query data +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "select count(*) from test.meters;" + +taos> select count(*) from test.meters; + count(*) | +======================== + 100000000 | +Query OK, 1 row(s) in set (0.103537s) ``` -$ kubectl proxy --accept-hosts='^.*$' --address='0.0.0.0' -``` + +查看vnode分布，通过show dnodes + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" + +taos> show dnodes + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 8 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-19 17:54:18.469 | | | | + 2 | tdengine-1.ta... | 8 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-19 17:54:38.698 | | | | + 3 | tdengine-2.ta... | 8 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-19 17:55:02.039 | | | | +Query OK, 3 row(s) in set (0.001357s) +``` + +通过show vgroup 查看 vnode 分布情况 + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show test.vgroups" + +taos> show test.vgroups + vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode | v4_status | cacheload | cacheelements | tsma | +============================================================================================================================================================================================== + 2 | test | 1267 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | + 3 | test | 1215 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 4 | test | 1215 | 1 | leader | 2 | follower | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 5 | test | 1307 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 6 | test | 1245 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | + 7 | test | 1275 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 8 | test | 1231 | 1 | leader | 2 | follower | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 9 | test | 1245 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | +Query OK, 8 row(s) in set (0.001488s) +``` + +#### 手工创建 + +常见一个三副本的test1，并创建一张表，写入2条数据 + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- \ + taos -s \ + "create database if not exists test1 replica 3; + use test1; + create table if not exists t1(ts timestamp, n int); + insert into t1 values(now, 1)(now+1s, 2);" +``` + +通过show test1.vgroup 查看xnode分布情况 + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show test1.vgroups" + +taos> show test1.vgroups + vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode | v4_status | cacheload | cacheelements | tsma | +============================================================================================================================================================================================== + 10 | test1 | 1 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | + 11 | test1 | 0 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | +Query OK, 2 row(s) in set (0.001489s) +``` + +### 容错测试 + +Mnode leader 所在的 dnode 掉线，dnode1 + +```Bash +kubectl get pod -l app=tdengine -n tdengine-test -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 0/1 ErrImagePull 2 (2s ago) 20m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (6m48s ago) 20m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 21m 10.244.1.223 node85 +``` + +此时集群mnode发生重新选举，dnode1上的monde 成为leader + +```Bash +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G" +Welcome to the TDengine Command Line Interface, Client Version:3.0.7.1.202307190706 +Copyright (c) 2022 by TDengine, all rights reserved. + +taos> show mnodes\G +*************************** 1.row *************************** + id: 1 + endpoint: tdengine-0.taosd.tdengine-test.svc.cluster.local:6030 + role: offline + status: offline +create_time: 2023-07-19 17:54:18.559 +reboot_time: 1970-01-01 08:00:00.000 +*************************** 2.row *************************** + id: 2 + endpoint: tdengine-1.taosd.tdengine-test.svc.cluster.local:6030 + role: leader + status: ready +create_time: 2023-07-20 09:22:05.600 +reboot_time: 2023-07-20 09:32:00.227 +*************************** 3.row *************************** + id: 3 + endpoint: tdengine-2.taosd.tdengine-test.svc.cluster.local:6030 + role: follower + status: ready +create_time: 2023-07-20 09:22:20.042 +reboot_time: 2023-07-20 09:32:00.026 +Query OK, 3 row(s) in set (0.001513s) +``` + +集群可以正常读写 + +```Bash +# insert +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "insert into test1.t1 values(now, 1)(now+1s, 2);" + +taos> insert into test1.t1 values(now, 1)(now+1s, 2); +Insert OK, 2 row(s) affected (0.002098s) + +# select +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "select *from test1.t1" + +taos> select *from test1.t1 + ts | n | +======================================== + 2023-07-19 18:04:58.104 | 1 | + 2023-07-19 18:04:59.104 | 2 | + 2023-07-19 18:06:00.303 | 1 | + 2023-07-19 18:06:01.303 | 2 | +Query OK, 4 row(s) in set (0.001994s) +``` + +同理，至于非leader得mnode掉线，读写当然可以正常进行，这里就不做过多的展示。 ## 集群扩容 TDengine 集群支持自动扩容： -```bash +```Bash kubectl scale statefulsets tdengine --replicas=4 ``` 上面命令行中参数 `--replica=4` 表示要将 TDengine 集群扩容到 4 个节点，执行后首先检查 POD 的状态： -```bash -kubectl get pods -l app=tdengine +```Bash +kubectl get pod -l app=tdengine -n tdengine-test -o wide ``` 输出如下： -``` -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 161m -tdengine-1 1/1 Running 0 161m -tdengine-2 1/1 Running 0 32m -tdengine-3 1/1 Running 0 32m +```Plain +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 1/1 Running 4 (6h26m ago) 6h53m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (6h39m ago) 6h53m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 5h16m 10.244.1.224 node85 +tdengine-3 1/1 Running 0 3m24s 10.244.2.76 node86 ``` -此时 POD 的状态仍然是 Running，TDengine 集群中的 dnode 状态要等 POD 状态为 `ready` 之后才能看到： +此时 Pod 的状态仍然是 Running，TDengine 集群中的 dnode 状态要等 Pod 状态为 `ready` 之后才能看到： -```bash -kubectl exec -i -t tdengine-3 -- taos -s "show dnodes" +```Bash +kubectl exec -it tdengine-3 -n tdengine-test -- taos -s "show dnodes" ``` 扩容后的四节点 TDengine 集群的 dnode 列表: -``` +```Plain taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | - 4 | tdengine-3.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:33:16.039 | | -Query OK, 4 rows in database (0.008377s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | + 4 | tdengine-3.ta... | 0 | 16 | ready | 2023-07-20 16:01:44.007 | 2023-07-20 16:01:44.889 | | | | +Query OK, 4 row(s) in set (0.003628s) ``` ## 集群缩容 -由于 TDengine 集群在扩缩容时会对数据进行节点间迁移，使用 kubectl 命令进行缩容需要首先使用 "drop dnodes" 命令，节点删除完成后再进行 Kubernetes 集群缩容。 +由于 TDengine 集群在扩缩容时会对数据进行节点间迁移，使用 kubectl 命令进行缩容需要首先使用 "drop dnodes" 命令（**如果集群中存在3副本的db，那么缩容后的** **dnode** **个数也要必须大于等于3，否则drop dnode操作会被中止**），然后再节点删除完成后再进行 Kubernetes 集群缩容。 注意：由于 Kubernetes Statefulset 中 Pod 的只能按创建顺序逆序移除，所以 TDengine drop dnode 也需要按照创建顺序逆序移除，否则会导致 Pod 处于错误状态。 -``` -$ kubectl exec -i -t tdengine-0 -- taos -s "drop dnode 4" -``` - -```bash -$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "drop dnode 4" +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | -Query OK, 3 rows in database (0.004861s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | +Query OK, 3 row(s) in set (0.003324s) ``` 确认移除成功后（使用 kubectl exec -i -t tdengine-0 -- taos -s "show dnodes" 查看和确认 dnode 列表），使用 kubectl 命令移除 POD： -``` -kubectl scale statefulsets tdengine --replicas=3 +```Plain +kubectl scale statefulsets tdengine --replicas=3 -n tdengine-test ``` 最后一个 POD 将会被删除。使用命令 kubectl get pods -l app=tdengine 查看POD状态： -``` -$ kubectl get pods -l app=tdengine -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 4m7s -tdengine-1 1/1 Running 0 3m55s -tdengine-2 1/1 Running 0 2m28s +```Plain +kubectl get pod -l app=tdengine -n tdengine-test -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 1/1 Running 4 (6h55m ago) 7h22m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (7h9m ago) 7h23m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 5h45m 10.244.1.224 node85 ``` POD删除后，需要手动删除PVC，否则下次扩容时会继续使用以前的数据导致无法正常加入集群。 -```bash -$ kubectl delete pvc taosdata-tdengine-3 +```Bash +kubectl delete pvc aosdata-tdengine-3 -n tdengine-test ``` 此时的集群状态是安全的，需要时还可以再次进行扩容： -```bash -$ kubectl scale statefulsets tdengine --replicas=4 +```Bash +kubectl scale statefulsets tdengine --replicas=4 -n tdengine-test statefulset.apps/tdengine scaled -it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl get pods -l app=tdengine -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 35m -tdengine-1 1/1 Running 0 34m -tdengine-2 1/1 Running 0 12m -tdengine-3 0/1 ContainerCreating 0 4s -it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl get pods -l app=tdengine -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 35m -tdengine-1 1/1 Running 0 34m -tdengine-2 1/1 Running 0 12m -tdengine-3 0/1 Running 0 7s -it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" + +kubectl get pod -l app=tdengine -n tdengine-test -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 1/1 Running 4 (6h59m ago) 7h27m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (7h13m ago) 7h27m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 5h49m 10.244.1.224 node85 +tdengine-3 1/1 Running 0 20s 10.244.2.77 node86 + +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" taos> show dnodes -id | endpoint | vnodes | support_vnodes | status | create_time | offline reason | -====================================================================================================================================== -1 | tdengine-0.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 17:38:49.012 | | -2 | tdengine-1.taosd.default.sv... | 1 | 4 | ready | 2022-07-25 17:39:01.517 | | -5 | tdengine-2.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 18:01:36.479 | | -6 | tdengine-3.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 18:13:54.411 | | -Query OK, 4 row(s) in set (0.001348s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | + 5 | tdengine-3.ta... | 0 | 16 | ready | 2023-07-20 16:31:34.092 | 2023-07-20 16:38:17.419 | | | | +Query OK, 4 row(s) in set (0.003881s) ``` ## 清理 TDengine 集群 -完整移除 TDengine 集群，需要分别清理 statefulset、svc、configmap、pvc。 +> **删除pvc时需要注意下pv persistentVolumeReclaimPolicy策略，建议改为Delete，这样在删除pvc时才会自动清理pv，同时会清理底层的csi存储资源，如果没有配置删除pvc自动清理pv的策略，再删除pvc后，在手动清理pv时，pv对应的csi存储资源可能不会被释放。** -```bash -kubectl delete statefulset -l app=tdengine -kubectl delete svc -l app=tdengine -kubectl delete pvc -l app=tdengine -kubectl delete configmap taoscfg +完整移除 TDengine 集群，需要分别清理 statefulset、svc、configmap、pvc。 +```Bash +kubectl delete statefulset -l app=tdengine -n tdengine-test +kubectl delete svc -l app=tdengine -n tdengine-test +kubectl delete pvc -l app=tdengine -n tdengine-test +kubectl delete configmap taoscfg -n tdengine-test ``` ## 常见错误 @@ -330,65 +519,26 @@ kubectl delete configmap taoscfg 未进行 "drop dnode" 直接进行缩容，由于 TDengine 尚未删除节点，缩容 pod 导致 TDengine 集群中部分节点处于 offline 状态。 -``` -$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" +```Plain +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" taos> show dnodes -id | endpoint | vnodes | support_vnodes | status | create_time | offline reason | -====================================================================================================================================== -1 | tdengine-0.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 17:38:49.012 | | -2 | tdengine-1.taosd.default.sv... | 1 | 4 | ready | 2022-07-25 17:39:01.517 | | -5 | tdengine-2.taosd.default.sv... | 0 | 4 | offline | 2022-07-25 18:01:36.479 | status msg timeout | -6 | tdengine-3.taosd.default.sv... | 0 | 4 | offline | 2022-07-25 18:13:54.411 | status msg timeout | -Query OK, 4 row(s) in set (0.001323s) -``` - -### 错误二 - -TDengine 集群会持有 replica 参数，如果缩容后的节点数小于这个值，集群将无法使用： - -创建一个库使用 replica 参数为 2，插入部分数据： - -```bash -kubectl exec -i -t tdengine-0 -- \ - taos -s \ - "create database if not exists test replica 2; - use test; - create table if not exists t1(ts timestamp, n int); - insert into t1 values(now, 1)(now+1s, 2);" - - -``` - -缩容到单节点： - -```bash -kubectl scale statefulsets tdengine --replicas=1 - -``` - -在 TDengine CLI 中的所有数据库操作将无法成功。 - + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | + 5 | tdengine-3.ta... | 0 | 16 | offline | 2023-07-20 16:31:34.092 | 2023-07-20 16:38:17.419 | status msg timeout | | | +Query OK, 4 row(s) in set (0.003862s) ``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 2 | 40 | ready | any | 2021-06-01 15:55:52.562 | | - 2 | tdengine-1.taosd.default.sv... | 1 | 40 | offline | any | 2021-06-01 15:56:07.212 | status msg timeout | -Query OK, 2 row(s) in set (0.000845s) -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 2 | 40 | ready | any | 2021-06-01 15:55:52.562 | | - 2 | tdengine-1.taosd.default.sv... | 1 | 40 | offline | any | 2021-06-01 15:56:07.212 | status msg timeout | -Query OK, 2 row(s) in set (0.000837s) +## 最后 -taos> use test; -Database changed. +对于在 Kubernetes 环境下 TDengine 的高可用和高可靠来说，对于硬件损坏、灾难恢复，分为两个层面来讲： -taos> insert into t1 values(now, 3); +1. 底层的分布式块存储具备的灾难恢复能力，块存储的多副本，当下流行的分布式块存储如 Ceph，就具备多副本能力，将存储副本扩展到不同的机架、机柜、机房、数据中心（或者直接使用公有云厂商提供的块存储服务） +2. TDengine的灾难恢复，在 TDengine Enterprise 中，本身具备了当一个 dnode 永久下线（物理机磁盘损坏，数据分拣丢失）后，重新拉起一个空白的dnode来恢复原dnode的工作。 -DB error: Unable to resolve FQDN (0.013874s) +最后，欢迎使用[TDengine Cloud](https://cloud.taosdata.com/)，来体验一站式全托管的TDengine云服务。 -``` +> TDengine Cloud 是一个极简的全托管时序数据处理云服务平台，它是基于开源的时序数据库 TDengine 而开发的。除高性能的时序数据库之外，它还具有缓存、订阅和流计算等系统功能，而且提供了便利而又安全的数据分享、以及众多的企业级功能。它可以让物联网、工业互联网、金融、IT 运维监控等领域企业在时序数据的管理上大幅降低人力成本和运营成本。 diff --git a/docs/zh/10-deployment/index.md b/docs/zh/10-deployment/index.md index 4ff1add779c68a7098002dd95dcf28c9dc1acf72..f2ce51983705586fbafec4dd3ae6fcc22bfbe43f 100644 --- a/docs/zh/10-deployment/index.md +++ b/docs/zh/10-deployment/index.md @@ -6,7 +6,7 @@ description: 部署 TDengine 集群的多种方式 TDengine 支持集群，提供水平扩展的能力。如果需要获得更高的处理能力，只需要多增加节点即可。TDengine 采用虚拟节点技术，将一个节点虚拟化为多个虚拟节点，以实现负载均衡。同时，TDengine可以将多个节点上的虚拟节点组成虚拟节点组，通过多副本机制，以保证供系统的高可用。TDengine的集群功能完全开源。 -本章节主要介绍如何在主机上人工部署集群，以及如何使用 Kubernetes 和 Helm部署集群。 +本章节主要介绍如何在主机上人工部署集群，docker部署，以及如何使用 Kubernetes 和 Helm部署集群。 ```mdx-code-block import DocCardList from '@theme/DocCardList'; diff --git a/docs/zh/12-taos-sql/01-data-type.md b/docs/zh/12-taos-sql/01-data-type.md index f014573ca691d3df9e5f27ff5af533035381e55a..1df07e7e7f009d202c6fe71a1940dc6ab388d845 100644 --- a/docs/zh/12-taos-sql/01-data-type.md +++ b/docs/zh/12-taos-sql/01-data-type.md @@ -42,12 +42,21 @@ CREATE DATABASE db_name PRECISION 'ns'; | 14 | NCHAR | 自定义 | 记录包含多字节字符在内的字符串，如中文字符。每个 NCHAR 字符占用 4 字节的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 `\'`。NCHAR 使用时须指定字符串大小，类型为 NCHAR(10) 的列表示此列的字符串最多存储 10 个 NCHAR 字符。如果用户字符串长度超出声明长度，将会报错。 | | 15 | JSON | | JSON 数据类型， 只有 Tag 可以是 JSON 格式 | | 16 | VARCHAR | 自定义 | BINARY 类型的别名 | +| 17 | GEOMETRY | 自定义 | 几何类型 | :::note -- 表的每行长度不能超过 48KB（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置）。 +- 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）（注意：每个 BINARY/NCHAR/GEOMETRY 类型的列还会额外占用 2 个字节的存储位置）。 - 虽然 BINARY 类型在底层存储上支持字节型的二进制字符，但不同编程语言对二进制数据的处理方式并不保证一致，因此建议在 BINARY 类型中只存储 ASCII 可见字符，而避免存储不可见字符。多字节的数据，例如中文字符，则需要使用 NCHAR 类型进行保存。如果强行使用 BINARY 类型保存中文字符，虽然有时也能正常读写，但并不带有字符集信息，很容易出现数据乱码甚至数据损坏等情况。 -- BINARY 类型理论上最长可以有 16,374 字节。BINARY 仅支持字符串输入，字符串两端需使用单引号引用。使用时须指定大小，如 BINARY(20) 定义了最长为 20 个单字节字符的字符串，每个字符占 1 字节的存储空间，总共固定占用 20 字节的空间，此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\'`。 +- BINARY 类型理论上最长可以有 16,374（从 3.0.5.0 版本开始，数据列为 65,517，标签列为 16,382） 字节。BINARY 仅支持字符串输入，字符串两端需使用单引号引用。使用时须指定大小，如 BINARY(20) 定义了最长为 20 个单字节字符的字符串，每个字符占 1 字节的存储空间，总共固定占用 20 字节的空间，此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\'`。 +- GEOMETRY 类型数据列为最大长度为 65,517 字节，标签列最大长度为 16,382 字节。支持 2D 的 POINT、LINESTRING 和 POLYGON 子类型数据。长度计算方式如下表所示： + + | # | **语法** | **最小长度** | **最大长度** | **每组坐标长度增长** | + |---|--------------------------------------|----------|------------|--------------| + | 1 | POINT(1.0 1.0) | 21 | 21 | 无 | + | 2 | LINESTRING(1.0 1.0, 2.0 2.0) | 9+2*16 | 9+4094*16 | +16 | + | 3 | POLYGON((1.0 1.0, 2.0 2.0, 1.0 1.0)) | 13+3*16 | 13+4094*16 | +16 | + - SQL 语句中的数值类型将依据是否存在小数点，或使用科学计数法表示，来判断数值类型是否为整型或者浮点型，因此在使用时要注意相应类型越界的情况。例如，9999999999999999999 会认为超过长整型的上边界而溢出，而 9999999999999999999.0 会被认为是有效的浮点数。 ::: diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index a2a09141206fa3f07a9e8289ca009fde2ebcec98..ac435debea6bc2652f9ba0b30cadb90866ee1d22 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -36,13 +36,12 @@ database_option: { | TSDB_PAGESIZE value | WAL_RETENTION_PERIOD value | WAL_RETENTION_SIZE value - | WAL_SEGMENT_SIZE value } ``` ### 参数说明 -- BUFFER: 一个 VNODE 写入内存池大小，单位为 MB，默认为 96，最小为 3，最大为 16384。 +- BUFFER: 一个 VNODE 写入内存池大小，单位为 MB，默认为 256，最小为 3，最大为 16384。 - CACHEMODEL：表示是否在内存中缓存子表的最近数据。默认为 none。 - none：表示不缓存。 - last_row：表示缓存子表最近一行数据。这将显著改善 LAST_ROW 函数的性能表现。 @@ -74,10 +73,8 @@ database_option: { - TABLE_PREFIX：当其为正值时，在决定把一个表分配到哪个 vgroup 时要忽略表名中指定长度的前缀；当其为负值时，在决定把一个表分配到哪个 vgroup 时只使用表名中指定长度的前缀；例如，假定表名为 "v30001"，当 TSDB_PREFIX = 2 时 使用 "0001" 来决定分配到哪个 vgroup ，当 TSDB_PREFIX = -2 时使用 "v3" 来决定分配到哪个 vgroup - TABLE_SUFFIX：当其为正值时，在决定把一个表分配到哪个 vgroup 时要忽略表名中指定长度的后缀；当其为负值时，在决定把一个表分配到哪个 vgroup 时只使用表名中指定长度的后缀；例如，假定表名为 "v30001"，当 TSDB_SUFFIX = 2 时 使用 "v300" 来决定分配到哪个 vgroup ，当 TSDB_SUFFIX = -2 时使用 "01" 来决定分配到哪个 vgroup。 - TSDB_PAGESIZE：一个 VNODE 中时序数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB到 16 MB。 -- WAL_RETENTION_PERIOD: 为了数据订阅消费，需要WAL日志文件额外保留的最大时长策略。WAL日志清理，不受订阅客户端消费状态影响。单位为 s。默认为 0，表示无需为订阅保留。新建订阅，应先设置恰当的时长策略。 +- WAL_RETENTION_PERIOD: 为了数据订阅消费，需要WAL日志文件额外保留的最大时长策略。WAL日志清理，不受订阅客户端消费状态影响。单位为 s。默认为 3600，表示在 WAL 保留最近 3600 秒的数据，请根据数据订阅的需要修改这个参数为适当值。 - WAL_RETENTION_SIZE：为了数据订阅消费，需要WAL日志文件额外保留的最大累计大小策略。单位为 KB。默认为 0，表示累计大小无上限。 -- WAL_ROLL_PERIOD：wal 文件切换时长，单位为 s。当WAL文件创建并写入后，经过该时间，会自动创建一个新的WAL文件。默认为 0，即仅在TSDB落盘时创建新文件。 -- WAL_SEGMENT_SIZE：wal 单个文件大小，单位为 KB。当前写入文件大小超过上限后会自动创建一个新的WAL文件。默认为 0，即仅在TSDB落盘时创建新文件。 ### 创建数据库示例 ```sql @@ -85,7 +82,7 @@ create database if not exists db vgroups 10 buffer 10 ``` -以上示例创建了一个有 10 个 vgroup 名为 db 的数据库， 其中每个 vnode 分配也 10MB 的写入缓存 +以上示例创建了一个有 10 个 vgroup 名为 db 的数据库， 其中每个 vnode 分配 10MB 的写入缓存 ### 使用数据库 @@ -121,6 +118,8 @@ alter_database_option: { | WAL_LEVEL value | WAL_FSYNC_PERIOD value | KEEP value + | WAL_RETENTION_PERIOD value + | WAL_RETENTION_SIZE value } ``` diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index 5687c7e740766ad7a3fbea03ff42ce3137bb140e..9258258263282c6d3f9df5aa8dc650ec511a9680 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -43,12 +43,11 @@ table_option: { 1. 表的第一个字段必须是 TIMESTAMP，并且系统自动将其设为主键； 2. 表名最大长度为 192； -3. 表的每行长度不能超过 48KB;（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） +3. 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）;（注意：每个 BINARY/NCHAR/GEOMETRY 类型的列还会额外占用 2 个字节的存储位置） 4. 子表名只能由字母、数字和下划线组成，且不能以数字开头，不区分大小写 -5. 使用数据类型 binary 或 nchar，需指定其最长的字节数，如 binary(20)，表示 20 字节； +5. 使用数据类型 BINARY/NCHAR/GEOMETRY，需指定其最长的字节数，如 BINARY(20)，表示 20 字节； 6. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"，可以让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。但是同样具有长度限制要求。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 例如：\`aBc\` 和 \`abc\` 是不同的表名，但是 abc 和 aBc 是相同的表名。 - 需要注意的是转义字符中的内容必须是可打印字符。 **参数说明** diff --git a/docs/zh/12-taos-sql/04-stable.md b/docs/zh/12-taos-sql/04-stable.md index 93decf190dc5bb1f37d4c896d25c86c158c572e1..853d2bf981a665fbfa4a439e57cd27cbbf7ccd84 100644 --- a/docs/zh/12-taos-sql/04-stable.md +++ b/docs/zh/12-taos-sql/04-stable.md @@ -51,6 +51,11 @@ DESCRIBE [db_name.]stb_name; ### 获取超级表中所有子表的标签信息 +``` +SHOW TABLE TAGS FROM table_name [FROM db_name]; +SHOW TABLE TAGS FROM [db_name.]table_name; +``` + ``` taos> SHOW TABLE TAGS FROM st1; tbname | id | loc | diff --git a/docs/zh/12-taos-sql/05-insert.md b/docs/zh/12-taos-sql/05-insert.md index b72754b15437a081b89056fca3a76030cac62363..c03ad9bd8f3ef1485df6ee1e536f15825bcb459c 100644 --- a/docs/zh/12-taos-sql/05-insert.md +++ b/docs/zh/12-taos-sql/05-insert.md @@ -82,7 +82,7 @@ INSERT INTO d1001 (ts, current, phase) VALUES ('2021-07-13 14:06:33.196', 10.27, ```sql INSERT INTO d1001 VALUES ('2021-07-13 14:06:34.630', 10.2, 219, 0.32) ('2021-07-13 14:06:35.779', 10.15, 217, 0.33) - d1002 (ts, current, phase) VALUES ('2021-07-13 14:06:34.255', 10.27, 0.31）; + d1002 (ts, current, phase) VALUES ('2021-07-13 14:06:34.255', 10.27, 0.31); ``` ## 插入记录时自动建表 diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 870df734718fde878cc3f4f234b6e0bcfe8a5303..9560c3c4dfae1597049a8d0c8e9f035b229e88b5 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -55,7 +55,7 @@ window_clause: { | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [WATERMARK(watermark_val)] [FILL(fill_mod_and_val)] interp_clause: - RANGE(ts_val, ts_val) EVERY(every_val) FILL(fill_mod_and_val) + RANGE(ts_val [, ts_val]) EVERY(every_val) FILL(fill_mod_and_val) partition_by_clause: PARTITION BY expr [, expr] ... @@ -315,7 +315,7 @@ WHERE (column|tbname) match/MATCH/nmatch/NMATCH _regex_ ### 使用限制 -只能针对表名（即 tbname 筛选）、binary/nchar 类型标签值进行正则表达式过滤，不支持普通列的过滤。 +只能针对表名（即 tbname 筛选）、binary/nchar 类型值进行正则表达式过滤。 正则匹配字符串长度不能超过 128 字节。可以通过参数 _maxRegexStringLen_ 设置和调整最大允许的正则匹配字符串，该参数是客户端配置参数，需要重启才能生效。 diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 458fc9c7a2bd434822c414d80adab69f17ce58b0..3c0ee06caf98342f39ece6c5ecc0d92db5a89afb 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -700,7 +700,7 @@ ELAPSED(ts_primary_key [, time_unit]) LEASTSQUARES(expr, start_val, step_val) ``` -**功能说明**：统计表中某列的值是主键（时间戳）的拟合直线方程。start_val 是自变量初始值，step_val 是自变量的步长值。 +**功能说明**：统计表中某列的值的拟合直线方程。start_val 是自变量初始值，step_val 是自变量的步长值。 **返回数据类型**：字符串表达式（斜率, 截距）。 @@ -869,10 +869,15 @@ FIRST(expr) ### INTERP ```sql -INTERP(expr) +INTERP(expr [, ignore_null_values]) + +ignore_null_values: { + 0 + | 1 +} ``` -**功能说明**：返回指定时间截面指定列的记录值或插值。 +**功能说明**：返回指定时间截面指定列的记录值或插值。ignore_null_values 参数的值可以是 0 或 1，为 1 时表示忽略 NULL 值, 缺省值为0。 **返回数据类型**：同字段类型。 @@ -885,9 +890,10 @@ INTERP(expr) - INTERP 用于在指定时间断面获取指定列的记录值，如果该时间断面不存在符合条件的行数据，那么会根据 FILL 参数的设定进行插值。 - INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。 - INTERP 需要同时与 RANGE，EVERY 和 FILL 关键字一起使用。 -- INTERP 的输出时间范围根据 RANGE(timestamp1,timestamp2)字段来指定，需满足 timestamp1 <= timestamp2。其中 timestamp1（必选值）为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2（必选值）为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。 +- INTERP 的输出时间范围根据 RANGE(timestamp1, timestamp2)字段来指定，需满足 timestamp1 <= timestamp2。其中 timestamp1 为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2 为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。 - INTERP 根据 EVERY(time_unit) 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（time_unit 值）进行插值，time_unit 可取值时间单位：1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天)，1w(周)。例如 EVERY(500a) 将对于指定数据每500毫秒间隔进行一次插值. - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。关于 FILL 子句如何使用请参考 [FILL 子句](../distinguished/#fill-子句) +- INTERP 可以在 RANGE 字段中只指定唯一的时间戳对单个时间点进行插值，在这种情况下，EVERY 字段可以省略。例如：SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00') FILL(linear). - INTERP 作用于超级表时, 会将该超级表下的所有子表数据按照主键列排序后进行插值计算，也可以搭配 PARTITION BY tbname 使用，将结果强制规约到单个时间线。 - INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(3.0.2.0版本以后支持)。 - INTERP 可以与伪列 _isfilled 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(3.0.3.0版本以后支持)。 @@ -985,19 +991,14 @@ SAMPLE(expr, k) **功能说明**： 获取数据的 k 个采样值。参数 k 的合法输入范围是 1≤ k ≤ 1000。 -**返回结果类型**： 同原始数据类型， 返回结果中带有该行记录的时间戳。 +**返回结果类型**： 同原始数据类型。 -**适用数据类型**： 在超级表查询中使用时，不能应用在标签之上。 +**适用数据类型**： 全部类型字段。 **嵌套子查询支持**： 适用于内层查询和外层查询。 **适用于**：表和超级表。 -**使用说明**： - -- 不能参与表达式计算；该函数可以应用在普通表和超级表上； -- 使用在超级表上的时候，需要搭配 PARTITION by tbname 使用，将结果强制规约到单个时间线。 - ### TAIL @@ -1042,11 +1043,11 @@ TOP(expr, k) UNIQUE(expr) ``` -**功能说明**：返回该列的数值首次出现的值。该函数功能与 distinct 相似，但是可以匹配标签和时间戳信息。可以针对除时间列以外的字段进行查询，可以匹配标签和时间戳，其中的标签和时间戳是第一次出现时刻的标签和时间戳。 +**功能说明**：返回该列数据首次出现的值。该函数功能与 distinct 相似。 **返回数据类型**：同应用的字段。 -**适用数据类型**：适合于除时间类型以外的字段。 +**适用数据类型**：全部类型字段。 **适用于**: 表和超级表。 @@ -1075,7 +1076,6 @@ CSUM(expr) - 不支持 +、-、*、/ 运算，如 csum(col1) + csum(col2)。 - 只能与聚合（Aggregation）函数一起使用。 该函数可以应用在普通表和超级表上。 -- 使用在超级表上的时候，需要搭配 PARTITION BY tbname使用，将结果强制规约到单个时间线。 ### DERIVATIVE @@ -1099,7 +1099,6 @@ ignore_negative: { **使用说明**: -- DERIVATIVE 函数可以在由 PARTITION BY 划分出单独时间线的情况下用于超级表（也即 PARTITION BY tbname）。 - 可以与选择相关联的列一起使用。 例如: select \_rowts, DERIVATIVE() from。 ### DIFF @@ -1162,7 +1161,6 @@ MAVG(expr, k) - 不支持 +、-、*、/ 运算，如 mavg(col1, k1) + mavg(col2, k1); - 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用； -- 使用在超级表上的时候，需要搭配 PARTITION BY tbname使用，将结果强制规约到单个时间线。 ### STATECOUNT @@ -1188,7 +1186,6 @@ STATECOUNT(expr, oper, val) **使用说明**： -- 该函数可以应用在普通表上，在由 PARTITION BY 划分出单独时间线的情况下用于超级表（也即 PARTITION BY tbname） - 不能和窗口操作一起使用，例如 interval/state_window/session_window。 @@ -1216,7 +1213,6 @@ STATEDURATION(expr, oper, val, unit) **使用说明**： -- 该函数可以应用在普通表上，在由 PARTITION BY 划分出单独时间线的情况下用于超级表（也即 PARTITION BY tbname） - 不能和窗口操作一起使用，例如 interval/state_window/session_window。 @@ -1234,8 +1230,6 @@ TWA(expr) **适用于**：表和超级表。 -**使用说明**： TWA 函数可以在由 PARTITION BY 划分出单独时间线的情况下用于超级表（也即 PARTITION BY tbname）。 - ## 系统信息函数 @@ -1271,3 +1265,140 @@ SELECT SERVER_STATUS(); ``` **说明**：检测服务端是否所有 dnode 都在线，如果是则返回成功，否则返回无法建立连接的错误。 + + +## Geometry 函数 + +### Geometry 输入函数： + +#### ST_GeomFromText + +```sql +ST_GeomFromText(VARCHAR WKT expr) +``` + +**功能说明**：根据 Well-Known Text (WKT) 表示从指定的几何值创建几何数据。 + +**返回值类型**：GEOMETRY + +**适用数据类型**：VARCHAR + +**适用表类型**：标准表和超表 + +**使用说明**：输入可以是 WKT 字符串之一，例如点（POINT）、线串（LINESTRING）、多边形（POLYGON）、多点集（MULTIPOINT）、多线串（MULTILINESTRING）、多多边形（MULTIPOLYGON）、几何集合（GEOMETRYCOLLECTION）。输出是以二进制字符串形式定义的 GEOMETRY 数据类型。 + +### Geometry 输出函数： + +#### ST_AsText + +```sql +ST_AsText(GEOMETRY geom) +``` + +**功能说明**：从几何数据中返回指定的 Well-Known Text (WKT) 表示。 + +**返回值类型**：VARCHAR + +**适用数据类型**：GEOMETRY + +**适用表类型**：标准表和超表 + +**使用说明**：输出可以是 WKT 字符串之一，例如点（POINT）、线串（LINESTRING）、多边形（POLYGON）、多点集（MULTIPOINT）、多线串（MULTILINESTRING）、多多边形（MULTIPOLYGON）、几何集合（GEOMETRYCOLLECTION）。 + +### Geometry 关系函数： + +#### ST_Intersects + +```sql +ST_Intersects(GEOMETRY geomA, GEOMETRY geomB) +``` + +##功能说明**：比较两个几何对象，并在它们相交时返回 true。 + +**返回值类型**：BOOL + +**适用数据类型**：GEOMETRY，GEOMETRY + +**适用表类型**：标准表和超表 + +**使用说明**：如果两个几何对象有任何一个共享点，则它们相交。 + +#### ST_Equals + +```sql +ST_Equals(GEOMETRY geomA, GEOMETRY geomB) +``` + +**功能说明**：如果给定的几何对象是"空间相等"的，则返回 TRUE。 + +**返回值类型**：BOOL + +**适用数据类型**：GEOMETRY，GEOMETRY + +**适用表类型**：标准表和超表 + +**使用说明**："空间相等"意味着 ST_Contains(A,B) = true 和 ST_Contains(B,A) = true，并且点的顺序可能不同，但表示相同的几何结构。 + +#### ST_Touches + +```sql +ST_Touches(GEOMETRY geomA, GEOMETRY geomB) +``` + +**功能说明**：如果 A 和 B 相交，但它们的内部不相交，则返回 TRUE。 + +**返回值类型**：BOOL + +**适用数据类型**：GEOMETRY，GEOMETRY + +**适用表类型**：标准表和超表 + +**使用说明**：A 和 B 至少有一个公共点，并且这些公共点位于至少一个边界中。对于点/点输入，关系始终为 FALSE，因为点没有边界。 + +#### ST_Covers + +```sql +ST_Covers(GEOMETRY geomA, GEOMETRY geomB) +``` + +**功能说明**：如果 B 中的每个点都位于几何形状 A 内部（与内部或边界相交），则返回 TRUE。 + +**返回值类型**：BOOL + +**适用数据类型**：GEOMETRY，GEOMETRY + +**适用表类型**：标准表和超表 + +**使用说明**：A 包含 B 意味着 B 中的没有点位于 A 的外部（在外部）。 + +#### ST_Contains + +```sql +ST_Contains(GEOMETRY geomA, GEOMETRY geomB) +``` + +**功能说明**：如果 A 包含 B，描述：如果几何形状 A 包含几何形状 B，则返回 TRUE。 + +**返回值类型**：BOOL + +**适用数据类型**：GEOMETRY，GEOMETRY + +**适用表类型**：标准表和超表 + +**使用说明**：A 包含 B 当且仅当 B 的所有点位于 A 的内部（即位于内部或边界上）（或等效地，B 的没有点位于 A 的外部），并且 A 和 B 的内部至少有一个公共点。 + +#### ST_ContainsProperly + +```sql +ST_ContainsProperly(GEOMETRY geomA, GEOMETRY geomB) +``` + +**功能说明**：如果 B 的每个点都位于 A 内部，则返回 TRUE。 + +**返回值类型**：BOOL + +**适用数据类型**：GEOMETRY，GEOMETRY + +**适用表类型**：标准表和超表 + +**使用说明**：B 的没有点位于 A 的边界或外部。 diff --git a/docs/zh/12-taos-sql/16-operators.md b/docs/zh/12-taos-sql/16-operators.md index 48e9991799abf99ca868fc30e34f0435054afa0b..0636121edda2d2cb25e54861db4f18b45c2491ee 100644 --- a/docs/zh/12-taos-sql/16-operators.md +++ b/docs/zh/12-taos-sql/16-operators.md @@ -39,7 +39,7 @@ TDengine 支持 `UNION ALL` 和 `UNION` 操作符。UNION ALL 将查询返回的 | 3 | \>, < | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型 | 大于，小于 | | 4 | \>=, <= | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型 | 大于等于，小于等于 | | 5 | IS [NOT] NULL | 所有类型 | 是否为空值 | -| 6 | [NOT] BETWEEN AND | 除 BOOL、BLOB、MEDIUMBLOB 和 JSON 外的所有类型 | 闭区间比较 | +| 6 | [NOT] BETWEEN AND | 除 BOOL、BLOB、MEDIUMBLOB、JSON 和 GEOMETRY 外的所有类型 | 闭区间比较 | | 7 | IN | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型，且不可以为表的时间戳主键列 | 与列表内的任意值相等 | | 8 | LIKE | BINARY、NCHAR 和 VARCHAR | 通配符匹配 | | 9 | MATCH, NMATCH | BINARY、NCHAR 和 VARCHAR | 正则表达式匹配 | diff --git a/docs/zh/12-taos-sql/19-limit.md b/docs/zh/12-taos-sql/19-limit.md index 7b6692f1b733042fde3103de1e9fa06bf4dda542..73107bf3baecfe38d3c3a4cda8ffdc5d3df89a08 100644 --- a/docs/zh/12-taos-sql/19-limit.md +++ b/docs/zh/12-taos-sql/19-limit.md @@ -10,11 +10,9 @@ description: 合法字符集和命名中的限制规则 2. 允许英文字符或下划线开头，不允许以数字开头 3. 不区分大小写 4. 转义后表（列）名规则： - 为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"。可用让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查 - 转义后的表（列）名同样受到长度限制要求，且长度计算的时候不计算转义符。使用转义字符以后，不再对转义字符中的内容进行大小写统一 + 为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"。使用转义字符以后，不再对转义字符中的内容进行大小写统一，即可以保留用户指定表名中的大小写属性。 例如：\`aBc\` 和 \`abc\` 是不同的表（列）名，但是 abc 和 aBc 是相同的表（列）名。 - 需要注意的是转义字符中的内容必须是可打印字符。 ## 密码合法字符集 @@ -26,7 +24,7 @@ description: 合法字符集和命名中的限制规则 - 数据库名最大长度为 64 字节 - 表名最大长度为 192 字节，不包括数据库名前缀和分隔符 -- 每行数据最大长度 48KB （注意：数据行内每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） +- 每行数据最大长度 48KB（从 3.0.5.0 版本开始为 64KB） （注意：数据行内每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） - 列名最大长度为 64 字节 - 最多允许 4096 列，最少需要 2 列，第一列必须是时间戳。 - 标签名最大长度为 64 字节 @@ -36,7 +34,7 @@ description: 合法字符集和命名中的限制规则 - 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制 - 数据库的副本数只能设置为 1 或 3 - 用户名的最大长度是 23 字节 -- 用户密码的最大长度是 128 字节 +- 用户密码的最大长度是 31 字节 - 总数据行数取决于可用资源 - 单个数据库的虚拟结点数上限为 1024 @@ -48,13 +46,13 @@ description: 合法字符集和命名中的限制规则 ### 转义后表（列）名规则： -为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"，可以避免表名与关键词的冲突，同时不受限于上述表名合法性约束检查，转义符不计入表名的长度。 +为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"，可以避免表名与关键词的冲突，转义符不计入表名的长度。 转义后的表（列）名同样受到长度限制要求，且长度计算的时候不计算转义符。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 例如： \`aBc\` 和 \`abc\` 是不同的表（列）名，但是 abc 和 aBc 是相同的表（列）名。 :::note -转义字符中的内容必须是可打印字符。 +转义字符中的内容必须符合命名规则中的字符约束。 ::: diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md index d416febd55521853812fbe6952d403f4d265ac4e..f52af2f282c7ecdab89dcd5b1236746a2d111348 100644 --- a/docs/zh/12-taos-sql/20-keywords.md +++ b/docs/zh/12-taos-sql/20-keywords.md @@ -178,6 +178,7 @@ description: TDengine 保留关键字的详细列表 - MATCH - MAX_DELAY +- MAX_SPEED - MAXROWS - MERGE - META @@ -334,8 +335,6 @@ description: TDengine 保留关键字的详细列表 - WAL_LEVEL - WAL_RETENTION_PERIOD - WAL_RETENTION_SIZE -- WAL_ROLL_PERIOD -- WAL_SEGMENT_SIZE - WATERMARK - WHERE - WINDOW_CLOSE diff --git a/docs/zh/12-taos-sql/22-meta.md b/docs/zh/12-taos-sql/22-meta.md index 7fb60b85a7db75e2df2e7612862963f3e847e4e5..35794ec2699eba8111096022e04632853cfc3056 100644 --- a/docs/zh/12-taos-sql/22-meta.md +++ b/docs/zh/12-taos-sql/22-meta.md @@ -28,15 +28,15 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 提供 dnode 的相关信息。也可以使用 SHOW DNODES 来查询这些信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :------------: | ------------ | ------------------------- | +| # | **列名** | **数据类型** | **说明** | +| --- | :------------: | ------------ | ----------------------------------------------------------------------------------------------------- | | 1 | vnodes | SMALLINT | dnode 中的实际 vnode 个数。需要注意，`vnodes` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 2 | support_vnodes | SMALLINT | 最多支持的 vnode 个数 | -| 3 | status | BINARY(10) | 当前状态 | -| 4 | note | BINARY(256) | 离线原因等信息 | -| 5 | id | SMALLINT | dnode id | -| 6 | endpoint | BINARY(134) | dnode 的地址 | -| 7 | create | TIMESTAMP | 创建时间 | +| 2 | support_vnodes | SMALLINT | 最多支持的 vnode 个数 | +| 3 | status | BINARY(10) | 当前状态 | +| 4 | note | BINARY(256) | 离线原因等信息 | +| 5 | id | SMALLINT | dnode id | +| 6 | endpoint | BINARY(134) | dnode 的地址 | +| 7 | create | TIMESTAMP | 创建时间 | ## INS_MNODES @@ -81,7 +81,7 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 | 3 | ntables | INT | 数据库中表的数量，包含子表和普通表但不包含超级表 | | 4 | vgroups | INT | 数据库中有多少个 vgroup。需要注意，`vgroups` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 6 | replica | INT | 副本数。需要注意，`replica` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 7 | strict | BINARY(3) | 强一致性。需要注意，`strict` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 7 | strict | BINARY(4) | 废弃参数 | | 8 | duration | INT | 单文件存储数据的时间跨度。需要注意，`duration` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 9 | keep | INT | 数据保留时长。需要注意，`keep` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 10 | buffer | INT | 每个 vnode 写缓存的内存块大小，单位 MB。需要注意，`buffer` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | @@ -98,79 +98,77 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 | 21 | cachesize | INT | 表示每个 vnode 中用于缓存子表最近数据的内存大小。需要注意，`cachesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 22 | wal_level | INT | WAL 级别。需要注意，`wal_level` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 23 | wal_fsync_period | INT | 数据落盘周期。需要注意，`wal_fsync_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 24 | wal_retention_period | INT | WAL 的保存时长。需要注意，`wal_retention_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 24 | wal_retention_period | INT | WAL 的保存时长，单位为秒。需要注意，`wal_retention_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 25 | wal_retention_size | INT | WAL 的保存上限。需要注意，`wal_retention_size` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 26 | wal_roll_period | INT | wal 文件切换时长。需要注意，`wal_roll_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 27 | wal_segment_size | BIGINT | wal 单个文件大小。需要注意，`wal_segment_size` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 28 | stt_trigger | SMALLINT | 触发文件合并的落盘文件的个数。需要注意，`stt_trigger` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 29 | table_prefix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度。需要注意，`table_prefix` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 30 | table_suffix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度。需要注意，`table_suffix` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 31 | tsdb_pagesize | INT | 时序数据存储引擎中的页大小。需要注意，`tsdb_pagesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 26 | stt_trigger | SMALLINT | 触发文件合并的落盘文件的个数。需要注意，`stt_trigger` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 27 | table_prefix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度。需要注意，`table_prefix` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 28 | table_suffix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度。需要注意，`table_suffix` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 29 | tsdb_pagesize | INT | 时序数据存储引擎中的页大小。需要注意，`tsdb_pagesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | ## INS_FUNCTIONS 用户创建的自定义函数的信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :---------: | ------------ | -------------- | -| 1 | name | BINARY(64) | 函数名 | -| 2 | comment | BINARY(255) | 补充说明。需要注意，`comment` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 3 | aggregate | INT | 是否为聚合函数。需要注意，`aggregate` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 4 | output_type | BINARY(31) | 输出类型 | -| 5 | create_time | TIMESTAMP | 创建时间 | -| 6 | code_len | INT | 代码长度 | -| 7 | bufsize | INT | buffer 大小 | -| 8 | func_language | BINARY(31) | 自定义函数编程语言 | -| 9 | func_body | BINARY(16384) | 函数体定义 | -| 10 | func_version | INT | 函数版本号。初始版本为0，每次替换更新，版本号加1。| +| # | **列名** | **数据类型** | **说明** | +| --- | :-----------: | ------------- | --------------------------------------------------------------------------------------------- | +| 1 | name | BINARY(64) | 函数名 | +| 2 | comment | BINARY(255) | 补充说明。需要注意，`comment` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 3 | aggregate | INT | 是否为聚合函数。需要注意，`aggregate` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 4 | output_type | BINARY(31) | 输出类型 | +| 5 | create_time | TIMESTAMP | 创建时间 | +| 6 | code_len | INT | 代码长度 | +| 7 | bufsize | INT | buffer 大小 | +| 8 | func_language | BINARY(31) | 自定义函数编程语言 | +| 9 | func_body | BINARY(16384) | 函数体定义 | +| 10 | func_version | INT | 函数版本号。初始版本为0，每次替换更新，版本号加1。 | ## INS_INDEXES 提供用户创建的索引的相关信息。也可以使用 SHOW INDEX 来查询这些信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :--------------: | ------------ | ---------------------------------------------------------------------------------- | -| 1 | db_name | BINARY(32) | 包含此索引的表所在的数据库名 | -| 2 | table_name | BINARY(192) | 包含此索引的表的名称 | -| 3 | index_name | BINARY(192) | 索引名 | -| 4 | column_name | BINARY(64) | 建索引的列的列名 | -| 5 | index_type | BINARY(10) | 目前有 SMA 和 FULLTEXT | -| 6 | index_extensions | BINARY(256) | 索引的额外信息。对 SMA 类型的索引，是函数名的列表。对 FULLTEXT 类型的索引为 NULL。 | +| # | **列名** | **数据类型** | **说明** | +| --- | :--------------: | ------------ | ------------------------------------------------------- | +| 1 | db_name | BINARY(32) | 包含此索引的表所在的数据库名 | +| 2 | table_name | BINARY(192) | 包含此索引的表的名称 | +| 3 | index_name | BINARY(192) | 索引名 | +| 4 | column_name | BINARY(64) | 建索引的列的列名 | +| 5 | index_type | BINARY(10) | 目前有 SMA 和 tag | +| 6 | index_extensions | BINARY(256) | 索引的额外信息。对 SMA/tag 类型的索引，是函数名的列表。 | ## INS_STABLES 提供用户创建的超级表的相关信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :-----------: | ------------ | ------------------------ | -| 1 | stable_name | BINARY(192) | 超级表表名 | -| 2 | db_name | BINARY(64) | 超级表所在的数据库的名称 | -| 3 | create_time | TIMESTAMP | 创建时间 | -| 4 | columns | INT | 列数目 | -| 5 | tags | INT | 标签数目。需要注意，`tags` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 6 | last_update | TIMESTAMP | 最后更新时间 | -| 7 | table_comment | BINARY(1024) | 表注释 | -| 8 | watermark | BINARY(64) | 窗口的关闭时间。需要注意，`watermark` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 9 | max_delay | BINARY(64) | 推送计算结果的最大延迟。需要注意，`max_delay` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 10 | rollup | BINARY(128) | rollup 聚合函数。需要注意，`rollup` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| # | **列名** | **数据类型** | **说明** | +| --- | :-----------: | ------------ | ----------------------------------------------------------------------------------------------------- | +| 1 | stable_name | BINARY(192) | 超级表表名 | +| 2 | db_name | BINARY(64) | 超级表所在的数据库的名称 | +| 3 | create_time | TIMESTAMP | 创建时间 | +| 4 | columns | INT | 列数目 | +| 5 | tags | INT | 标签数目。需要注意，`tags` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 6 | last_update | TIMESTAMP | 最后更新时间 | +| 7 | table_comment | BINARY(1024) | 表注释 | +| 8 | watermark | BINARY(64) | 窗口的关闭时间。需要注意，`watermark` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 9 | max_delay | BINARY(64) | 推送计算结果的最大延迟。需要注意，`max_delay` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 10 | rollup | BINARY(128) | rollup 聚合函数。需要注意，`rollup` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | ## INS_TABLES 提供用户创建的普通表和子表的相关信息 -| # | **列名** | **数据类型** | **说明** | -| --- | :-----------: | ------------ | ---------------- | -| 1 | table_name | BINARY(192) | 表名 | -| 2 | db_name | BINARY(64) | 数据库名 | -| 3 | create_time | TIMESTAMP | 创建时间 | -| 4 | columns | INT | 列数目 | -| 5 | stable_name | BINARY(192) | 所属的超级表表名 | -| 6 | uid | BIGINT | 表 id | -| 7 | vgroup_id | INT | vgroup id | -| 8 | ttl | INT | 表的生命周期。需要注意，`ttl` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 9 | table_comment | BINARY(1024) | 表注释 | -| 10 | type | BINARY(21) | 表类型 | +| # | **列名** | **数据类型** | **说明** | +| --- | :-----------: | ------------ | ------------------------------------------------------------------------------------- | +| 1 | table_name | BINARY(192) | 表名 | +| 2 | db_name | BINARY(64) | 数据库名 | +| 3 | create_time | TIMESTAMP | 创建时间 | +| 4 | columns | INT | 列数目 | +| 5 | stable_name | BINARY(192) | 所属的超级表表名 | +| 6 | uid | BIGINT | 表 id | +| 7 | vgroup_id | INT | vgroup id | +| 8 | ttl | INT | 表的生命周期。需要注意，`ttl` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 9 | table_comment | BINARY(1024) | 表注释 | +| 10 | type | BINARY(21) | 表类型 | ## INS_TAGS @@ -185,17 +183,17 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 ## INS_COLUMNS -| # | **列名** | **数据类型** | **说明** | -| --- | :---------: | ------------- | ---------------------- | -| 1 | table_name | BINARY(192) | 表名 | -| 2 | db_name | BINARY(64) | 该表所在的数据库的名称 | -| 3 | table_type | BINARY(21) | 表类型 | -| 4 | col_name | BINARY(64) | 列 的名称 | -| 5 | col_type | BINARY(32) | 列 的类型 | -| 6 | col_length | INT | 列 的长度 | -| 7 | col_precision | INT | 列 的精度 | -| 8 | col_scale | INT | 列 的比例 | -| 9 | col_nullable | INT | 列 是否可以为空 | +| # | **列名** | **数据类型** | **说明** | +| --- | :-----------: | ------------ | ---------------------- | +| 1 | table_name | BINARY(192) | 表名 | +| 2 | db_name | BINARY(64) | 该表所在的数据库的名称 | +| 3 | table_type | BINARY(21) | 表类型 | +| 4 | col_name | BINARY(64) | 列 的名称 | +| 5 | col_type | BINARY(32) | 列 的类型 | +| 6 | col_length | INT | 列 的长度 | +| 7 | col_precision | INT | 列 的精度 | +| 8 | col_scale | INT | 列 的比例 | +| 9 | col_nullable | INT | 列 是否可以为空 | ## INS_USERS @@ -211,60 +209,60 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 提供企业版授权的相关信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :---------: | ------------ | -------------------------------------------------- | -| 1 | version | BINARY(9) | 企业版授权说明：official(官方授权的)/trial(试用的) | -| 2 | cpu_cores | BINARY(9) | 授权使用的 CPU 核心数量 | -| 3 | dnodes | BINARY(10) | 授权使用的 dnode 节点数量。需要注意，`dnodes` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 4 | streams | BINARY(10) | 授权创建的流数量。需要注意，`streams` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 5 | users | BINARY(10) | 授权创建的用户数量。需要注意，`users` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 6 | accounts | BINARY(10) | 授权创建的帐户数量。需要注意，`accounts` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 7 | storage | BINARY(21) | 授权使用的存储空间大小。需要注意，`storage` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 8 | connections | BINARY(21) | 授权使用的客户端连接数量。需要注意，`connections` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 9 | databases | BINARY(11) | 授权使用的数据库数量。需要注意，`databases` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 10 | speed | BINARY(9) | 授权使用的数据点每秒写入数量 | -| 11 | querytime | BINARY(9) | 授权使用的查询总时长 | -| 12 | timeseries | BINARY(21) | 授权使用的测点数量 | -| 13 | expired | BINARY(5) | 是否到期，true：到期，false：未到期 | -| 14 | expire_time | BINARY(19) | 试用期到期时间 | +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | --------------------------------------------------------------------------------------------------------- | +| 1 | version | BINARY(9) | 企业版授权说明：official(官方授权的)/trial(试用的) | +| 2 | cpu_cores | BINARY(9) | 授权使用的 CPU 核心数量 | +| 3 | dnodes | BINARY(10) | 授权使用的 dnode 节点数量。需要注意，`dnodes` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 4 | streams | BINARY(10) | 授权创建的流数量。需要注意，`streams` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 5 | users | BINARY(10) | 授权创建的用户数量。需要注意，`users` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 6 | accounts | BINARY(10) | 授权创建的帐户数量。需要注意，`accounts` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 7 | storage | BINARY(21) | 授权使用的存储空间大小。需要注意，`storage` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 8 | connections | BINARY(21) | 授权使用的客户端连接数量。需要注意，`connections` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 9 | databases | BINARY(11) | 授权使用的数据库数量。需要注意，`databases` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 10 | speed | BINARY(9) | 授权使用的数据点每秒写入数量 | +| 11 | querytime | BINARY(9) | 授权使用的查询总时长 | +| 12 | timeseries | BINARY(21) | 授权使用的测点数量 | +| 13 | expired | BINARY(5) | 是否到期，true：到期，false：未到期 | +| 14 | expire_time | BINARY(19) | 试用期到期时间 | ## INS_VGROUPS 系统中所有 vgroups 的信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :-------: | ------------ | ------------------------------------------------------ | -| 1 | vgroup_id | INT | vgroup id | -| 2 | db_name | BINARY(32) | 数据库名 | -| 3 | tables | INT | 此 vgroup 内有多少表。需要注意，`tables` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 4 | status | BINARY(10) | 此 vgroup 的状态 | -| 5 | v1_dnode | INT | 第一个成员所在的 dnode 的 id | -| 6 | v1_status | BINARY(10) | 第一个成员的状态 | -| 7 | v2_dnode | INT | 第二个成员所在的 dnode 的 id | -| 8 | v2_status | BINARY(10) | 第二个成员的状态 | -| 9 | v3_dnode | INT | 第三个成员所在的 dnode 的 id | -| 10 | v3_status | BINARY(10) | 第三个成员的状态 | -| 11 | nfiles | INT | 此 vgroup 中数据/元数据文件的数量 | -| 12 | file_size | INT | 此 vgroup 中数据/元数据文件的大小 | -| 13 | tsma | TINYINT | 此 vgroup 是否专用于 Time-range-wise SMA，1: 是, 0: 否 | +| # | **列名** | **数据类型** | **说明** | +| --- | :-------: | ------------ | ------------------------------------------------------------------------------------------------ | +| 1 | vgroup_id | INT | vgroup id | +| 2 | db_name | BINARY(32) | 数据库名 | +| 3 | tables | INT | 此 vgroup 内有多少表。需要注意，`tables` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 4 | status | BINARY(10) | 此 vgroup 的状态 | +| 5 | v1_dnode | INT | 第一个成员所在的 dnode 的 id | +| 6 | v1_status | BINARY(10) | 第一个成员的状态 | +| 7 | v2_dnode | INT | 第二个成员所在的 dnode 的 id | +| 8 | v2_status | BINARY(10) | 第二个成员的状态 | +| 9 | v3_dnode | INT | 第三个成员所在的 dnode 的 id | +| 10 | v3_status | BINARY(10) | 第三个成员的状态 | +| 11 | nfiles | INT | 此 vgroup 中数据/元数据文件的数量 | +| 12 | file_size | INT | 此 vgroup 中数据/元数据文件的大小 | +| 13 | tsma | TINYINT | 此 vgroup 是否专用于 Time-range-wise SMA，1: 是, 0: 否 | ## INS_CONFIGS 系统配置参数。 -| # | **列名** | **数据类型** | **说明** | -| --- | :------: | ------------ | ------------ | -| 1 | name | BINARY(32) | 配置项名称 | +| # | **列名** | **数据类型** | **说明** | +| --- | :------: | ------------ | --------------------------------------------------------------------------------------- | +| 1 | name | BINARY(32) | 配置项名称 | | 2 | value | BINARY(64) | 该配置项的值。需要注意，`value` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | ## INS_DNODE_VARIABLES 系统中每个 dnode 的配置参数。 -| # | **列名** | **数据类型** | **说明** | -| --- | :------: | ------------ | ------------ | -| 1 | dnode_id | INT | dnode 的 ID | -| 2 | name | BINARY(32) | 配置项名称 | +| # | **列名** | **数据类型** | **说明** | +| --- | :------: | ------------ | --------------------------------------------------------------------------------------- | +| 1 | dnode_id | INT | dnode 的 ID | +| 2 | name | BINARY(32) | 配置项名称 | | 3 | value | BINARY(64) | 该配置项的值。需要注意，`value` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | ## INS_TOPICS @@ -284,17 +282,29 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 | 2 | consumer_group | BINARY(193) | 订阅者的消费者组 | | 3 | vgroup_id | INT | 消费者被分配的 vgroup id | | 4 | consumer_id | BIGINT | 消费者的唯一 id | +| 5 | offset | BINARY(64) | 消费者的消费进度 | +| 6 | rows | BIGINT | 消费者的消费的数据条数 | ## INS_STREAMS -| # | **列名** | **数据类型** | **说明** | -| --- | :----------: | ------------ | --------------------------------------- | -| 1 | stream_name | BINARY(64) | 流计算名称 | -| 2 | create_time | TIMESTAMP | 创建时间 | -| 3 | sql | BINARY(1024) | 创建流计算时提供的 SQL 语句 | -| 4 | status | BINARY(20) | 流当前状态 | -| 5 | source_db | BINARY(64) | 源数据库 | -| 6 | target_db | BINARY(64) | 目的数据库 | -| 7 | target_table | BINARY(192) | 流计算写入的目标表 | -| 8 | watermark | BIGINT | watermark，详见 SQL 手册流式计算。需要注意，`watermark` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| # | **列名** | **数据类型** | **说明** | +| --- | :----------: | ------------ | -------------------------------------------------------------------------------------------------------------------- | +| 1 | stream_name | BINARY(64) | 流计算名称 | +| 2 | create_time | TIMESTAMP | 创建时间 | +| 3 | sql | BINARY(1024) | 创建流计算时提供的 SQL 语句 | +| 4 | status | BINARY(20) | 流当前状态 | +| 5 | source_db | BINARY(64) | 源数据库 | +| 6 | target_db | BINARY(64) | 目的数据库 | +| 7 | target_table | BINARY(192) | 流计算写入的目标表 | +| 8 | watermark | BIGINT | watermark，详见 SQL 手册流式计算。需要注意，`watermark` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 9 | trigger | INT | 计算结果推送模式，详见 SQL 手册流式计算。需要注意，`trigger` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | + +## INS_USER_PRIVILEGES + +| # | **列名** | **数据类型** | **说明** | +| --- | :----------: | ------------ | -------------------------------------------------------------------------------------------------------------------- | +| 1 | user_name | VARCHAR(24) | 用户名 +| 2 | privilege | VARCHAR(10) | 权限描述 +| 3 | db_name | VARCHAR(65) | 数据库名称 +| 4 | table_name | VARCHAR(193) | 表名称 +| 5 | condition | VARCHAR(49152) | 子表权限过滤条件 diff --git a/docs/zh/12-taos-sql/24-show.md b/docs/zh/12-taos-sql/24-show.md index 12ad665e42fc8bf8b177efdbe43b5356d7a04d3b..6e102e2356daa15bd766639028fbf8ebcedd96b2 100644 --- a/docs/zh/12-taos-sql/24-show.md +++ b/docs/zh/12-taos-sql/24-show.md @@ -36,7 +36,7 @@ SHOW CONNECTIONS; SHOW CONSUMERS; ``` -显示当前数据库下所有活跃的消费者的信息。 +显示当前数据库下所有消费者的信息。 ## SHOW CREATE DATABASE @@ -101,6 +101,7 @@ SHOW GRANTS; ```sql SHOW INDEXES FROM tbl_name [FROM db_name]; +SHOW INDEXES FROM [db_name.]tbl_name; ``` 显示已创建的索引。 @@ -269,6 +270,7 @@ Query OK, 24 row(s) in set (0.002444s) ```sql SHOW TAGS FROM child_table_name [FROM db_name]; +SHOW TAGS FROM [db_name.]child_table_name; ``` 显示子表的标签信息。 diff --git a/docs/zh/12-taos-sql/25-grant.md b/docs/zh/12-taos-sql/25-grant.md index 7fb944710125de6fe4d6efcedbb0677b33e1fd0f..a9c3910500d7ad3ba2435eb4bebb74085a10da78 100644 --- a/docs/zh/12-taos-sql/25-grant.md +++ b/docs/zh/12-taos-sql/25-grant.md @@ -16,7 +16,7 @@ CREATE USER use_name PASS 'password' [SYSINFO {1|0}]; use_name 最长为 23 字节。 -password 最长为 128 字节，合法字符包括"a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/"，不可以出现单双引号、撇号、反斜杠和空格，且不可以为空。 +password 最长为 31 字节，合法字符包括"a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/"，不可以出现单双引号、撇号、反斜杠和空格，且不可以为空。 SYSINFO 表示用户是否可以查看系统信息。1 表示可以查看，0 表示不可以查看。系统信息包括服务端配置信息、服务端各种节点信息（如 DNODE、QNODE等）、存储相关的信息等。默认为可以查看系统信息。 diff --git a/docs/zh/12-taos-sql/26-udf.md b/docs/zh/12-taos-sql/26-udf.md index c1d2761d7d7eb30fdb8da5c1854d1dcb45d5d492..307831ce805b27411b69d69540e29e41f1eee953 100644 --- a/docs/zh/12-taos-sql/26-udf.md +++ b/docs/zh/12-taos-sql/26-udf.md @@ -38,7 +38,7 @@ CREATE [OR REPLACE] AGGREGATE FUNCTION function_name AS library_path OUTPUTTYPE ``` - OR REPLACE: 如果函数已经存在，会修改已有的函数属性。 - function_name：聚合函数未来在 SQL 中被调用时的函数名，必须与函数实现中 udfNormalFunc 的实际名称一致； - - LANGUAGE 'C|Python'：函数编程语言，目前支持C语言和Python语言。 + - LANGUAGE 'C|Python'：函数编程语言，目前支持C语言和Python语言（v3.7+）。 - library_path：如果编程语言是C，路径是包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件）。如果编程语言是Python，路径是包含 UDF 函数实现的Python文件路径。这个路径需要用英文单引号或英文双引号括起来；； - output_type：此函数计算结果的数据类型名称； - buffer_size：中间计算结果的缓冲区大小，单位是字节。如果不使用可以不设置。 diff --git a/docs/zh/12-taos-sql/27-index.md b/docs/zh/12-taos-sql/27-index.md index aa84140296832f79a6498d0da2b5a8f500cd1e90..da8f38eb225933b89f5b403e374dea3a1604c364 100644 --- a/docs/zh/12-taos-sql/27-index.md +++ b/docs/zh/12-taos-sql/27-index.md @@ -4,12 +4,13 @@ title: 索引 description: 索引功能的使用细节 --- -TDengine 从 3.0.0.0 版本开始引入了索引功能，支持 SMA 索引和 FULLTEXT 索引。 +TDengine 从 3.0.0.0 版本开始引入了索引功能，支持 SMA 索引和 tag 索引。 ## 创建索引 ```sql -CREATE FULLTEXT INDEX index_name ON tb_name (col_name [, col_name] ...) + +CREATE INDEX index_name ON tb_name index_option CREATE SMA INDEX index_name ON tb_name index_option @@ -28,9 +29,23 @@ functions: - WATERMARK: 最小单位毫秒，取值范围 [0ms, 900000ms]，默认值为 5 秒，只可用于超级表。 - MAX_DELAY: 最小单位毫秒，取值范围 [1ms, 900000ms]，默认值为 interval 的值(但不能超过最大值)，只可用于超级表。注：不建议 MAX_DELAY 设置太小，否则会过于频繁的推送结果，影响存储和查询性能，如无特殊需求，取默认值即可。 -### FULLTEXT 索引 - -对指定列建立文本索引，可以提升含有文本过滤的查询的性能。FULLTEXT 索引不支持 index_option 语法。现阶段只支持对 JSON 类型的标签列创建 FULLTEXT 索引。不支持多列联合索引，但可以为每个列分布创建 FULLTEXT 索引。 +```sql +DROP DATABASE IF EXISTS d0; +CREATE DATABASE d0; +USE d0; +CREATE TABLE IF NOT EXISTS st1 (ts timestamp, c1 int, c2 float, c3 double) TAGS (t1 int unsigned); +CREATE TABLE ct1 USING st1 TAGS(1000); +CREATE TABLE ct2 USING st1 TAGS(2000); +INSERT INTO ct1 VALUES(now+0s, 10, 2.0, 3.0); +INSERT INTO ct1 VALUES(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3); +CREATE SMA INDEX sma_index_name1 ON st1 FUNCTION(max(c1),max(c2),min(c1)) INTERVAL(5m,10s) SLIDING(5m) WATERMARK 5s MAX_DELAY 1m; +-- 从 SMA 索引查询 +ALTER LOCAL 'querySmaOptimize' '1'; +SELECT max(c2),min(c1) FROM st1 INTERVAL(5m,10s) SLIDING(5m); +SELECT _wstart,_wend,_wduration,max(c2),min(c1) FROM st1 INTERVAL(5m,10s) SLIDING(5m); +-- 从原始数据查询 +ALTER LOCAL 'querySmaOptimize' '0'; +``` ## 删除索引 @@ -41,8 +56,8 @@ DROP INDEX index_name; ## 查看索引 ````sql -```sql SHOW INDEXES FROM tbl_name [FROM db_name]; +SHOW INDEXES FROM [db_name.]tbl_name; ```` 显示在所指定的数据库或表上已创建的索引。 diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index 27dd3294b74943a289f25246e8a08a6ef416cd48..2a1e5f092cec8ea4fd0f8c77b44843d04071e259 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -18,6 +18,7 @@ description: "TDengine 3.0 版本的语法变更说明" | 8 | 混合运算 | 增强 | 查询中的混合运算（标量运算和矢量运算混合）全面增强，SELECT的各个子句均全面支持符合语法语义的混合运算。 | 9 | 标签运算 | 新增 |在查询中，标签列可以像普通列一样参与各种运算，用于各种子句。 | 10 | 时间线子句和时间函数用于超级表查询 | 增强 |没有PARTITION BY时，超级表的数据会被合并成一条时间线。 +| 11 | GEOMETRY | 新增 | 几何类型。 ## SQL 语句变更 @@ -33,7 +34,7 @@ description: "TDengine 3.0 版本的语法变更说明" | 6 | ALTER USER | 调整 | 废除
  • PRIVILEGE：修改用户权限。3.0版本使用GRANT和REVOKE来授予和回收权限。
    新增
  • ENABLE：启用或停用此用户。
  • SYSINFO：修改用户是否可查看系统信息。
| 7 | COMPACT VNODES | 暂不支持 | 整理指定VNODE的数据。3.0.0版本暂不支持。 | 8 | CREATE ACCOUNT | 废除 | 2.x中为企业版功能，3.0不再支持。语法暂时保留了，执行报“This statement is no longer supported”错误。 -| 9 | CREATE DATABASE | 调整 |

废除

  • BLOCKS：VNODE使用的内存块数。3.0版本使用BUFFER来表示VNODE写入内存池的大小。
  • CACHE：VNODE使用的内存块的大小。3.0版本使用BUFFER来表示VNODE写入内存池的大小。
  • CACHELAST：缓存最新一行数据的模式。3.0版本用CACHEMODEL代替。
  • DAYS：数据文件存储数据的时间跨度。3.0版本使用DURATION代替。
  • FSYNC：当 WAL 设置为 2 时，执行 fsync 的周期。3.0版本使用WAL_FSYNC_PERIOD代替。
  • QUORUM：写入需要的副本确认数。3.0版本使用STRICT来指定强一致还是弱一致。
  • UPDATE：更新操作的支持模式。3.0版本所有数据库都支持部分列更新。
  • WAL：WAL 级别。3.0版本使用WAL_LEVEL代替。

新增

  • BUFFER：一个 VNODE 写入内存池大小。
  • CACHEMODEL：表示是否在内存中缓存子表的最近数据。
  • CACHESIZE：表示缓存子表最近数据的内存大小。
  • DURATION：代替原DAYS参数。新增支持带单位的设置方式。
  • PAGES：一个 VNODE 中元数据存储引擎的缓存页个数。
  • PAGESIZE：一个 VNODE 中元数据存储引擎的页大小。
  • RETENTIONS：表示数据的聚合周期和保存时长。
  • STRICT：表示数据同步的一致性要求。
  • SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表。
  • VGROUPS：数据库中初始VGROUP的数目。
  • WAL_FSYNC_PERIOD：代替原FSYNC参数。
  • WAL_LEVEL：代替原WAL参数。
  • WAL_RETENTION_PERIOD：wal文件的额外保留策略，用于数据订阅。
  • WAL_RETENTION_SIZE：wal文件的额外保留策略，用于数据订阅。
  • WAL_ROLL_PERIOD：wal文件切换时长。
  • WAL_SEGMENT_SIZE：wal单个文件大小。

调整

  • KEEP：3.0版本新增支持带单位的设置方式。
+| 9 | CREATE DATABASE | 调整 |

废除

  • BLOCKS：VNODE使用的内存块数。3.0版本使用BUFFER来表示VNODE写入内存池的大小。
  • CACHE：VNODE使用的内存块的大小。3.0版本使用BUFFER来表示VNODE写入内存池的大小。
  • CACHELAST：缓存最新一行数据的模式。3.0版本用CACHEMODEL代替。
  • DAYS：数据文件存储数据的时间跨度。3.0版本使用DURATION代替。
  • FSYNC：当 WAL 设置为 2 时，执行 fsync 的周期。3.0版本使用WAL_FSYNC_PERIOD代替。
  • QUORUM：写入需要的副本确认数。3.0版本使用STRICT来指定强一致还是弱一致。
  • UPDATE：更新操作的支持模式。3.0版本所有数据库都支持部分列更新。
  • WAL：WAL 级别。3.0版本使用WAL_LEVEL代替。

新增

  • BUFFER：一个 VNODE 写入内存池大小。
  • CACHEMODEL：表示是否在内存中缓存子表的最近数据。
  • CACHESIZE：表示缓存子表最近数据的内存大小。
  • DURATION：代替原DAYS参数。新增支持带单位的设置方式。
  • PAGES：一个 VNODE 中元数据存储引擎的缓存页个数。
  • PAGESIZE：一个 VNODE 中元数据存储引擎的页大小。
  • RETENTIONS：表示数据的聚合周期和保存时长。
  • STRICT：表示数据同步的一致性要求。
  • SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表。
  • VGROUPS：数据库中初始VGROUP的数目。
  • WAL_FSYNC_PERIOD：代替原FSYNC参数。
  • WAL_LEVEL：代替原WAL参数。
  • WAL_RETENTION_PERIOD：wal文件的额外保留策略，用于数据订阅。
  • WAL_RETENTION_SIZE：wal文件的额外保留策略，用于数据订阅。

调整

  • KEEP：3.0版本新增支持带单位的设置方式。
| 10 | CREATE DNODE | 调整 | 新增主机名和端口号分开指定语法
  • CREATE DNODE dnode_host_name PORT port_val
| 11 | CREATE INDEX | 新增 | 创建SMA索引。 | 12 | CREATE MNODE | 新增 | 创建管理节点。 diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index c5d98767f9134c8eeb9bbc1ee2fb887370d9da54..e4c3efba17788171c9454173a6866c91c1f71d2d 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -362,6 +362,8 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **max** : 数据类型的 列/标签 的最大值。生成的值将小于最小值。 +- **fun** : 此列数据以函数填充，目前只支持 sin 和 cos 两函数，输入参数为时间戳换算成角度值，换算公式： 角度 x = 输入的时间列ts值 % 360。同时支持系数调节，随机波动因子调节，以固定格式的表达式展现，如 fun=“10\*sin(x)+100\*random(5)” , x 表示角度，取值 0 ~ 360度，增长步长与时间列步长一致。10 表示乘的系数，100 表示加或减的系数，5 表示波动幅度在 5% 的随机范围内。目前支持的数据类型为 int, bigint, float, double 四种数据类型。注意：表达式为固定模式，不可前后颠倒。 + - **values** : nchar/binary 列/标签的值域，将从值中随机选择。 - **sma**: 将该列加入 SMA 中，值为 "yes" 或者 "no"，默认为 "no"。 @@ -437,3 +439,29 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **sqls** ： - **sql** : 执行的 SQL 命令，必填。 + +#### 配置文件中数据类型书写对照表 + +| # | **引擎** | **taosBenchmark** +| --- | :----------------: | :---------------: +| 1 | TIMESTAMP | timestamp +| 2 | INT | int +| 3 | INT UNSIGNED | uint +| 4 | BIGINT | bigint +| 5 | BIGINT UNSIGNED | ubigint +| 6 | FLOAT | float +| 7 | DOUBLE | double +| 8 | BINARY | binary +| 9 | SMALLINT | smallint +| 10 | SMALLINT UNSIGNED | usmallint +| 11 | TINYINT | tinyint +| 12 | TINYINT UNSIGNED | utinyint +| 13 | BOOL | bool +| 14 | NCHAR | nchar +| 15 | VARCHAR | varchar +| 15 | JSON | json + +注意：taosBenchmark 配置文件中数据类型必须小写方可识别 + + + diff --git a/docs/zh/14-reference/06-taosdump.md b/docs/zh/14-reference/06-taosdump.md index 8ff1287c3ec65bbb9975dd8530b60c7acd108b2b..12122edd32dd3aa0f7b335ac7335c82d1c0e2379 100644 --- a/docs/zh/14-reference/06-taosdump.md +++ b/docs/zh/14-reference/06-taosdump.md @@ -82,8 +82,6 @@ Usage: taosdump [OPTION...] dbname [tbname ...] -e, --escape-character Use escaped character for database name -N, --without-property Dump database without its properties. -s, --schemaonly Only dump tables' schema. - -y, --answer-yes Input yes for prompt. It will skip data file - checking! -d, --avro-codec=snappy Choose an avro codec among null, deflate, snappy, and lzma. -S, --start-time=START_TIME Start time to dump. Either epoch or diff --git a/docs/zh/14-reference/07-tdinsight/index.mdx b/docs/zh/14-reference/07-tdinsight/index.mdx index b4eda61ebdb4f6ab9b8b650571a35196c76eda56..83e2a72b60d7fc89cd2c7f49aaa2c7684e439cfb 100644 --- a/docs/zh/14-reference/07-tdinsight/index.mdx +++ b/docs/zh/14-reference/07-tdinsight/index.mdx @@ -177,7 +177,7 @@ Install and configure TDinsight dashboard in Grafana on Ubuntu 18.04/20.04 syste 假设您在主机 `tdengine` 上启动 TDengine 数据库，HTTP API 端口为 `6041`，用户为 `root1`，密码为 `pass5ord`。执行脚本： ```bash -sudo ./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord +./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord ``` 我们提供了一个“-E”选项，用于从命令行配置 TDinsight 使用现有的通知通道(Notification Channel)。假设你的 Grafana 用户和密码是 `admin:admin`，使用以下命令获取已有的通知通道的`uid`： @@ -189,7 +189,7 @@ curl --no-progress-meter -u admin:admin http://localhost:3000/api/alert-notifica 使用上面获取的 `uid` 值作为 `-E` 输入。 ```bash -sudo ./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord -E existing-notifier +./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord -E existing-notifier ``` 如果要监控多个 TDengine 集群，则需要设置多个 TDinsight 仪表盘。设置非默认 TDinsight 需要进行一些更改： `-n` `-i` `-t` 选项需要更改为非默认名称，如果使用 内置短信告警功能，`-N` 和 `-L` 也应该改变。 diff --git a/docs/zh/14-reference/09-support-platform/index.md b/docs/zh/14-reference/09-support-platform/index.md index 500eeeb14c9c1f587435a0223b15ffc6ca840550..ba3b3deee14e6a7f41965e4d29c65edf04324d7d 100644 --- a/docs/zh/14-reference/09-support-platform/index.md +++ b/docs/zh/14-reference/09-support-platform/index.md @@ -5,14 +5,15 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" ## TDengine 服务端支持的平台列表 -| | **Windows server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18/20** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **macOS** | +| | **Windows server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18 以上** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **macOS** | | ------------ | ---------------------------- | ----------------- | ---------------- | ---------------- | ------------ | ----------------- | ---------------- | --------- | -| X64 | ● | ● | ● | ● | ● | ● | ● | ● | -| 树莓派 ARM64 | | | ● | | | | | | -| 华为云 ARM64 | | | | ● | | | | | -| M1 | | | | | | | | ● | +| X64 | ●/E | ●/E | ● | ● | ●/E | ●/E | ●/E | ● | +| 树莓派 ARM64 | | | ● | | | | | | +| 华为云 ARM64 | | | | ● | | | | | +| M1 | | | | | | | | ● | -注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。 +注：1) ● 表示经过官方测试验证， ○ 表示非官方测试验证，E 表示仅企业版支持。 + 2) 社区版仅支持主流操作系统的较新版本，包括 Ubuntu 18+/CentOS 7+/RetHat/Debian/CoreOS/FreeBSD/OpenSUSE/SUSE Linux/Fedora/macOS 等。如果有其他操作系统及版本的需求，请联系企业版支持。 ## TDengine 客户端和连接器支持的平台列表 diff --git a/docs/zh/14-reference/11-docker/_category_.yml b/docs/zh/14-reference/11-docker/_category_.yml deleted file mode 100644 index 68c16927f4a9098311dec116aab41e8faa07fe81..0000000000000000000000000000000000000000 --- a/docs/zh/14-reference/11-docker/_category_.yml +++ /dev/null @@ -1 +0,0 @@ -label: TDengine Docker 镜像 \ No newline at end of file diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md old mode 100644 new mode 100755 index 2694086f595e09c09114e628cb993b30e3641dd6..519b84ba71be996ff3f5811fad7314a261927b8b --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -5,7 +5,7 @@ description: "TDengine 客户端和服务配置列表" ## 为服务端指定配置文件 -TDengine 系统后台服务由 taosd 提供，可以在配置文件 taos.cfg 里修改配置参数，以满足不同场景的需求。配置文件的缺省位置在/etc/taos 目录，可以通过 taosd 命令行执行参数 -c 指定配置文件目录。比如，指定配置文件位于`/home/user` 这个目录： +TDengine 系统后台服务由 taosd 提供，可以在配置文件 taos.cfg 里修改配置参数，以满足不同场景的需求。在 Linux 系统上，配置文件的缺省位置在 `/etc/taos` 目录，在 Windows 系统上缺省位置在 `C:\TDengine` 。可以通过 taosd 命令行执行参数 -c 指定配置文件所在目录。比如，在 Linux 系统上可以指定配置文件位于 `/home/user` 这个目录： ``` taosd -c /home/user @@ -19,16 +19,20 @@ taosd -C ## 为客户端指定配置文件 -TDengine 系统的前台交互客户端应用程序为 taos，以及应用驱动，它可以与 taosd 共享同一个配置文件 taos.cfg，也可以使用单独指定配置文件。运行 taos 时，使用参数-c 指定配置文件目录，如 taos -c /home/cfg，表示使用/home/cfg/目录下的 taos.cfg 配置文件中的参数，缺省目录是/etc/taos。更多 taos 的使用方法请见帮助信息 `taos --help`。 +TDengine 系统的前台交互客户端应用程序为 taos，以及应用驱动，它可以与 taosd 共享同一个配置文件 taos.cfg，也可以使用单独指定配置文件。运行 taos 时，使用参数-c 指定配置文件目录，如： -```bash -taos -C +``` +taos -c /home/cfg ``` +表示使用/home/cfg/目录下的 taos.cfg 配置文件中的参数，缺省目录是/etc/taos。 另外可以使用 `-C` 显示当前服务器配置参数： + ```bash -taos --dump-config +taos -C ``` +更多 taos 的使用方法请见帮助信息 `taos --help`。 + ## 配置参数详细列表 :::note @@ -79,14 +83,14 @@ taos --dump-config :::note 确保集群中所有主机在端口 6030 上的 TCP 协议能够互通。（详细的端口情况请参见下表） ::: -| 协议 | 默认端口 | 用途说明 | 修改方法 | -| :--- | :-------- | :---------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------- | -| TCP | 6030 | 客户端与服务端之间通讯，多节点集群的节点间通讯。 | 由配置文件设置 serverPort 决定。 | -| TCP | 6041 | 客户端与服务端之间的 RESTful 通讯。 | 随 serverPort 端口变化。注意 taosAdapter 配置或有不同，请参考相应[文档](/reference/taosadapter/)。 | -| TCP | 6043 | taosKeeper 监控服务端口。 | 随 taosKeeper 启动参数设置变化。 | -| TCP | 6044 | 支持 StatsD 的数据接入端口。 | 随 taosAdapter 启动参数设置变化| -| UDP | 6045 | 支持 collectd 数据接入端口。 | 随 taosAdapter 启动参数设置变化 | -| TCP | 6060 | 企业版内 Monitor 服务的网络端口。 | | +| 协议 | 默认端口 | 用途说明 | 修改方法 | +| :--- | :------- | :----------------------------------------------- | :------------------------------------------------------------------------------------------------- | +| TCP | 6030 | 客户端与服务端之间通讯，多节点集群的节点间通讯。 | 由配置文件设置 serverPort 决定。 | +| TCP | 6041 | 客户端与服务端之间的 RESTful 通讯。 | 随 serverPort 端口变化。注意 taosAdapter 配置或有不同，请参考相应[文档](/reference/taosadapter/)。 | +| TCP | 6043 | taosKeeper 监控服务端口。 | 随 taosKeeper 启动参数设置变化。 | +| TCP | 6044 | 支持 StatsD 的数据接入端口。 | 随 taosAdapter 启动参数设置变化 | +| UDP | 6045 | 支持 collectd 数据接入端口。 | 随 taosAdapter 启动参数设置变化 | +| TCP | 6060 | 企业版内 Monitor 服务的网络端口。 | | ### maxShellConns @@ -97,11 +101,31 @@ taos --dump-config | 取值范围 | 10-50000000 | | 缺省值 | 5000 | +### numOfRpcSessions + +| 属性 | 说明 | +| -------- | ---------------------------- | +| 适用范围 | 客户端和服务端都适用 | +| 含义 | 一个客户端能创建的最大连接数 | +| 取值范围 | 100-100000 | +| 缺省值 | 30000 | + +### timeToGetAvailableConn + +| 属性 | 说明 | +| -------- | -------------------------- | +| 适用范围 | 客户端和服务端都适用 | +| 含义 | 获得可用连接的最长等待时间 | +| 取值范围 | 10-50000000(单位为毫秒) | +| 缺省值 | 500000 | + ## 监控相关 :::note 请注意，完整的监控功能需要安装并运行 `taoskeeper` 服务。taoskeeper 负责接收监控指标数据并创建 `log` 库。 +::: + ### monitor | 属性 | 说明 | @@ -139,43 +163,43 @@ taos --dump-config ### telemetryReporting -| 属性 | 说明 | -| -------- | -------------------------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 |是否上传 telemetry | -| 取值范围 | 0,1 0: 不上传；1：上传 | -| 缺省值 | 1 | +| 属性 | 说明 | +| -------- | ------------------------ | +| 适用范围 | 客户端和服务端都适用 | +| 含义 | 是否上传 telemetry | +| 取值范围 | 0,1 0: 不上传；1：上传 | +| 缺省值 | 1 | ### crashReporting -| 属性 | 说明 | -| -------- | -------------------------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 |是否上传 crash 信息 | -| 取值范围 | 0,1 0: 不上传；1：上传 | -| 缺省值 | 1 | +| 属性 | 说明 | +| -------- | ------------------------ | +| 适用范围 | 客户端和服务端都适用 | +| 含义 | 是否上传 crash 信息 | +| 取值范围 | 0,1 0: 不上传；1：上传 | +| 缺省值 | 1 | ## 查询相关 ### queryPolicy -| 属性 | 说明 | -| -------- | ----------------------------- | -| 适用范围 | 仅客户端适用 | -| 含义 | 查询语句的执行策略 | -| 单位 | 无 | -| 缺省值 | 1 | +| 属性 | 说明 | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| 适用范围 | 仅客户端适用 | +| 含义 | 查询语句的执行策略 | +| 单位 | 无 | +| 缺省值 | 1 | | 补充说明 | 1: 只使用 vnode，不使用 qnode; 2: 没有扫描算子的子任务在 qnode 执行，带扫描算子的子任务在 vnode 执行; 3: vnode 只运行扫描算子，其余算子均在 qnode 执行 | ### querySmaOptimize -| 属性 | 说明 | -| -------- | -------------------- | -| 适用范围 | 仅客户端适用 | -| 含义 | sma index 的优化策略 | -| 单位 | 无 | -| 缺省值 | 0 | -| 补充说明 |0: 表示不使用 sma index，永远从原始数据进行查询; 1: 表示使用 sma index，对符合的语句，直接从预计算的结果进行查询 | +| 属性 | 说明 | +| -------- | ---------------------------------------------------------------------------------------------------------------- | +| 适用范围 | 仅客户端适用 | +| 含义 | sma index 的优化策略 | +| 单位 | 无 | +| 缺省值 | 0 | +| 补充说明 | 0: 表示不使用 sma index，永远从原始数据进行查询; 1: 表示使用 sma index，对符合的语句，直接从预计算的结果进行查询 | ### maxNumOfDistinctRes @@ -188,21 +212,21 @@ taos --dump-config ### keepColumnName -| 属性 | 说明 | -| -------- | -------------------------------- | -| 适用范围 | 仅客户端适用 | +| 属性 | 说明 | +| -------- | ----------------------------------------------------------- | +| 适用范围 | 仅客户端适用 | | 含义 | Last、First、LastRow 函数查询时，返回的列名是否包含函数名。 | -| 取值范围 | 0 表示包含函数名，1 表示不包含函数名。 | -| 缺省值 | 0 | +| 取值范围 | 0 表示包含函数名，1 表示不包含函数名。 | +| 缺省值 | 0 | ### countAlwaysReturnValue -| 属性 | 说明 | -| -------- | -------------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 | count/hyperloglog函数在输入数据为空或者NULL的情况下是否返回值 | -| 取值范围 | 0：返回空行，1：返回 0 | -| 缺省值 | 1 | +| 属性 | 说明 | +| -------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| 适用范围 | 仅服务端适用 | +| 含义 | count/hyperloglog函数在输入数据为空或者NULL的情况下是否返回值 | +| 取值范围 | 0：返回空行，1：返回 0 | +| 缺省值 | 1 | | 补充说明 | 该参数设置为 1 时，如果查询中含有 GROUP BY，PARTITION BY 以及 INTERVAL 子句且相应的组或窗口内数据为空或者NULL， 对应的组或窗口将不返回查询结果 | ## 区域相关 @@ -314,20 +338,20 @@ charset 的有效值是 UTF-8。 ### dataDir -| 属性 | 说明 | -| -------- | ------------------------------------------ | -| 适用范围 | 仅服务端适用 | -| 含义 | 数据文件目录，所有的数据文件都将写入该目录 | -| 缺省值 | /var/lib/taos | +| 属性 | 说明 | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 适用范围 | 仅服务端适用 | +| 含义 | 数据文件目录，所有的数据文件都将写入该目录 | +| 缺省值 | /var/lib/taos | | 补充说明 | [多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8) 功能需要与 [KEEP](https://docs.taosdata.com/taos-sql/database/#%E5%8F%82%E6%95%B0%E8%AF%B4%E6%98%8E) 参数配合使用 | ### tempDir -| 属性 | 说明 | -| -------- | ------------------------------------------ | -| 适用范围 | 仅服务端适用 | +| 属性 | 说明 | +| -------- | ------------------------------------------------ | +| 适用范围 | 仅服务端适用 | | 含义 | 该参数指定所有系统运行过程中的临时文件生成的目录 | -| 缺省值 | /tmp | +| 缺省值 | /tmp | ### minimalTmpDirGB @@ -340,12 +364,21 @@ charset 的有效值是 UTF-8。 ### minimalDataDirGB -| 属性 | 说明 | -| -------- | ------------------------------------------------ | -| 适用范围 | 仅服务端适用 | +| 属性 | 说明 | +| -------- | ---------------------------------------------- | +| 适用范围 | 仅服务端适用 | | 含义 | dataDir 指定的时序数据存储目录所需要保留的最小 | -| 单位 | GB | -| 缺省值 | 2.0 | +| 单位 | GB | +| 缺省值 | 2.0 | + +### metaCacheMaxSize + +| 属性 | 说明 | +| -------- | ------------------------------------ | +| 适用范围 | 仅客户端适用 | +| 含义 | 指定单个客户端元数据缓存大小的最大值 | +| 单位 | MB | +| 缺省值 | -1 (无限制) | ## 集群相关 @@ -381,12 +414,12 @@ charset 的有效值是 UTF-8。 ### minimalLogDirGB -| 属性 | 说明 | -| -------- | -------------------------------------------- | -| 适用范围 | 服务端和客户端均适用 | +| 属性 | 说明 | +| -------- | ------------------------------------------------------ | +| 适用范围 | 服务端和客户端均适用 | | 含义 | 当日志文件夹所在磁盘可用空间大小小于该值时，停止写日志 | -| 单位 | GB | -| 缺省值 | 1.0 | +| 单位 | GB | +| 缺省值 | 1.0 | ### numOfLogLines @@ -415,6 +448,26 @@ charset 的有效值是 UTF-8。 | 缺省值 | 0 | | 补充说明 | 大于 0 时，日志文件会被重命名为 taosdlog.xxx，其中 xxx 为日志文件最后修改的时间戳。 | +### slowLogThreshold + +| 属性 | 说明 | +| -------- | ------------------------------------------------------------- | +| 适用范围 | 仅客户端适用 | +| 含义 | 指定慢查询门限值，大于等于门限值认为是慢查询 | +| 单位 | 秒 | +| 缺省值 | 3 | +| 补充说明 | 每个客户端中所有慢查询会被记录在日志目录下的taosSlowLog文件中 | + +### slowLogScope + +| 属性 | 说明 | +| -------- | ---------------------------------------------------------- | +| 适用范围 | 仅客户端适用 | +| 含义 | 指定启动记录哪些类型的慢查询 | +| 可选值 | ALL, QUERY, INSERT, OTHERS, NONE | +| 缺省值 | ALL | +| 补充说明 | 默认记录所有类型的慢查询，可通过配置只记录某一类型的慢查询 | + ### debugFlag | 属性 | 说明 | @@ -590,12 +643,12 @@ charset 的有效值是 UTF-8。 ### smlChildTableName -| 属性 | 说明 | -| -------- | ------------------------- | -| 适用范围 | 仅客户端适用 | -| 含义 | schemaless 自定义的子表名的 key | -| 类型 | 字符串 | -| 缺省值 | 无 | +| 属性 | 说明 | +| -------- | ------------------------------- | +| 适用范围 | 仅客户端适用 | +| 含义 | schemaless 自定义的子表名的 key | +| 类型 | 字符串 | +| 缺省值 | 无 | ### smlTagName @@ -608,12 +661,21 @@ charset 的有效值是 UTF-8。 ### smlDataFormat -| 属性 | 说明 | -| -------- | ----------------------------- | -| 适用范围 | 仅客户端适用 | +| 属性 | 说明 | +| -------- | -------------------------------------------------------- | +| 适用范围 | 仅客户端适用 | | 含义 | schemaless 列数据是否顺序一致，从3.0.3.0开始，该配置废弃 | -| 值域 | 0：不一致；1: 一致 | -| 缺省值 | 0 | +| 值域 | 0：不一致；1: 一致 | +| 缺省值 | 0 | + +### smlTsDefaultName + +| 属性 | 说明 | +| -------- | -------------------------------------------- | +| 适用范围 | 仅客户端适用 | +| 含义 | schemaless自动建表的时间列名字通过该配置设置 | +| 类型 | 字符串 | +| 缺省值 | _ts | ## 其他 @@ -629,12 +691,12 @@ charset 的有效值是 UTF-8。 ### enableScience -| 属性 | 说明 | -| -------- | ------------------------------------------------------------------------------------------------------------------------------------------ | -| 适用范围 | 仅客户端 TAOS-CLI 适用 | -| 含义 | 是否开启科学计数法显示浮点数 | -| 取值范围 | 0：否，1：是 | -| 缺省值 | 0 | +| 属性 | 说明 | +| -------- | ---------------------------- | +| 适用范围 | 仅客户端 TAOS-CLI 适用 | +| 含义 | 是否开启科学计数法显示浮点数 | +| 取值范围 | 0：否，1：是 | +| 缺省值 | 0 | ### udf @@ -645,157 +707,185 @@ charset 的有效值是 UTF-8。 | 取值范围 | 0: 不启动；1：启动 | | 缺省值 | 1 | +### ttlChangeOnWrite + +| 属性 | 说明 | +| -------- | ------------------------------------ | +| 适用范围 | 仅服务端适用 | +| 含义 | ttl 到期时间是否伴随表的修改操作改变 | +| 取值范围 | 0: 不改变；1：改变 | +| 缺省值 | 0 | + +### keepTimeOffset + +| 属性 | 说明 | +| -------- | -------------- | +| 适用范围 | 仅服务端适用 | +| 含义 | 迁移操作的延时 | +| 单位 | 小时 | +| 取值范围 | 0-23 | +| 缺省值 | 0 | + +### tmqMaxTopicNum + +| 属性 | 说明 | +| -------- | --------------------------- | +| 适用范围 | 仅服务端适用 | +| 含义 | 订阅最多可建立的 topic 数量 | +| 取值范围 | 1-10000 | +| 缺省值 | 20 | + ## 压缩参数 ### compressMsgSize -| 属性 | 说明 | -| -------- | ----------------------------- | -| 适用于 | 服务端和客户端均适用 | -| 含义 | 是否对 RPC 消息进行压缩 | -| 取值范围 | -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩 | -| 缺省值 | -1 | +| 属性 | 说明 | +| -------- | ------------------------------------------------------------------------------- | +| 适用于 | 服务端和客户端均适用 | +| 含义 | 是否对 RPC 消息进行压缩 | +| 取值范围 | -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩 | +| 缺省值 | -1 | ## 3.0 中有效的配置参数列表 -| # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 | -| --- | :---------------------: | --------------- | --------------- | ------------------------------------------------- | -| 1 | firstEp | 是 | 是 | | -| 2 | secondEp | 是 | 是 | | -| 3 | fqdn | 是 | 是 | | -| 4 | serverPort | 是 | 是 | | -| 5 | maxShellConns | 是 | 是 | | -| 6 | monitor | 是 | 是 | | -| 7 | monitorFqdn | 否 | 是 | | -| 8 | monitorPort | 否 | 是 | | -| 9 | monitorInterval | 是 | 是 | | -| 10 | queryPolicy | 否 | 是 | | -| 11 | querySmaOptimize | 否 | 是 | | -| 12 | maxNumOfDistinctRes | 是 | 是 | | -| 15 | countAlwaysReturnValue | 是 | 是 | | -| 16 | dataDir | 是 | 是 | | -| 17 | minimalDataDirGB | 是 | 是 | | -| 18 | supportVnodes | 否 | 是 | | -| 19 | tempDir | 是 | 是 | | -| 20 | minimalTmpDirGB | 是 | 是 | | -| 21 | smlChildTableName | 是 | 是 | | -| 22 | smlTagName | 是 | 是 | | -| 23 | smlDataFormat | 否 | 是（从3.0.3.0开始，该配置废弃） | | -| 24 | statusInterval | 是 | 是 | | -| 25 | logDir | 是 | 是 | | -| 26 | minimalLogDirGB | 是 | 是 | | -| 27 | numOfLogLines | 是 | 是 | | -| 28 | asyncLog | 是 | 是 | | -| 29 | logKeepDays | 是 | 是 | | -| 30 | debugFlag | 是 | 是 | | -| 31 | tmrDebugFlag | 是 | 是 | | -| 32 | uDebugFlag | 是 | 是 | | -| 33 | rpcDebugFlag | 是 | 是 | | -| 34 | jniDebugFlag | 是 | 是 | | -| 35 | qDebugFlag | 是 | 是 | | -| 36 | cDebugFlag | 是 | 是 | | -| 37 | dDebugFlag | 是 | 是 | | -| 38 | vDebugFlag | 是 | 是 | | -| 39 | mDebugFlag | 是 | 是 | | -| 40 | wDebugFlag | 是 | 是 | | -| 41 | sDebugFlag | 是 | 是 | | -| 42 | tsdbDebugFlag | 是 | 是 | | -| 43 | tqDebugFlag | 否 | 是 | | -| 44 | fsDebugFlag | 是 | 是 | | -| 45 | udfDebugFlag | 否 | 是 | | -| 46 | smaDebugFlag | 否 | 是 | | -| 47 | idxDebugFlag | 否 | 是 | | -| 48 | tdbDebugFlag | 否 | 是 | | -| 49 | metaDebugFlag | 否 | 是 | | -| 50 | timezone | 是 | 是 | | -| 51 | locale | 是 | 是 | | -| 52 | charset | 是 | 是 | | -| 53 | udf | 是 | 是 | | -| 54 | enableCoreFile | 是 | 是 | | +| # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 | +| --- | :--------------------: | --------------- | ------------------------------- | ------------------ | +| 1 | firstEp | 是 | 是 | | +| 2 | secondEp | 是 | 是 | | +| 3 | fqdn | 是 | 是 | | +| 4 | serverPort | 是 | 是 | | +| 5 | maxShellConns | 是 | 是 | | +| 6 | monitor | 是 | 是 | | +| 7 | monitorFqdn | 否 | 是 | | +| 8 | monitorPort | 否 | 是 | | +| 9 | monitorInterval | 是 | 是 | | +| 10 | queryPolicy | 否 | 是 | | +| 11 | querySmaOptimize | 否 | 是 | | +| 12 | maxNumOfDistinctRes | 是 | 是 | | +| 15 | countAlwaysReturnValue | 是 | 是 | | +| 16 | dataDir | 是 | 是 | | +| 17 | minimalDataDirGB | 是 | 是 | | +| 18 | supportVnodes | 否 | 是 | | +| 19 | tempDir | 是 | 是 | | +| 20 | minimalTmpDirGB | 是 | 是 | | +| 21 | smlChildTableName | 是 | 是 | | +| 22 | smlTagName | 是 | 是 | | +| 23 | smlDataFormat | 否 | 是（从3.0.3.0开始，该配置废弃） | | +| 24 | statusInterval | 是 | 是 | | +| 25 | logDir | 是 | 是 | | +| 26 | minimalLogDirGB | 是 | 是 | | +| 27 | numOfLogLines | 是 | 是 | | +| 28 | asyncLog | 是 | 是 | | +| 29 | logKeepDays | 是 | 是 | | +| 30 | debugFlag | 是 | 是 | | +| 31 | tmrDebugFlag | 是 | 是 | | +| 32 | uDebugFlag | 是 | 是 | | +| 33 | rpcDebugFlag | 是 | 是 | | +| 34 | jniDebugFlag | 是 | 是 | | +| 35 | qDebugFlag | 是 | 是 | | +| 36 | cDebugFlag | 是 | 是 | | +| 37 | dDebugFlag | 是 | 是 | | +| 38 | vDebugFlag | 是 | 是 | | +| 39 | mDebugFlag | 是 | 是 | | +| 40 | wDebugFlag | 是 | 是 | | +| 41 | sDebugFlag | 是 | 是 | | +| 42 | tsdbDebugFlag | 是 | 是 | | +| 43 | tqDebugFlag | 否 | 是 | | +| 44 | fsDebugFlag | 是 | 是 | | +| 45 | udfDebugFlag | 否 | 是 | | +| 46 | smaDebugFlag | 否 | 是 | | +| 47 | idxDebugFlag | 否 | 是 | | +| 48 | tdbDebugFlag | 否 | 是 | | +| 49 | metaDebugFlag | 否 | 是 | | +| 50 | timezone | 是 | 是 | | +| 51 | locale | 是 | 是 | | +| 52 | charset | 是 | 是 | | +| 53 | udf | 是 | 是 | | +| 54 | enableCoreFile | 是 | 是 | | +| 55 | ttlChangeOnWrite | 否 | 是 | | +| 56 | keepTimeOffset | 是 | 是 | | ## 2.x->3.0 的废弃参数 -| # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 | -| --- | :---------------------: | --------------- | --------------- | ------------------------------------------------- | -| 1 | arbitrator | 是 | 否 | 通过 RAFT 协议选主 | -| 2 | numOfThreadsPerCore | 是 | 否 | 有其它参数设置多种线程池的大小 | -| 3 | numOfMnodes | 是 | 否 | 通过 create mnode 命令动态创建 mnode | -| 4 | vnodeBak | 是 | 否 | 3.0 行为未知 | -| 5 | balance | 是 | 否 | 负载均衡功能由 split/merge vgroups 实现 （暂不支持） | -| 6 | balanceInterval | 是 | 否 | 随着 balance 参数失效 | -| 7 | offlineThreshold | 是 | 否 | 3.0 行为未知 | -| 8 | role | 是 | 否 | 由 supportVnode 决定是否能够创建 | -| 9 | dnodeNopLoop | 是 | 否 | 2.6 文档中未找到此参数 | -| 10 | keepTimeOffset | 是 | 否 | 2.6 文档中未找到此参数 | -| 11 | rpcTimer | 是 | 否 | 3.0 行为未知 | -| 12 | rpcMaxTime | 是 | 否 | 3.0 行为未知 | -| 13 | rpcForceTcp | 是 | 否 | 默认为 TCP | -| 14 | tcpConnTimeout | 是 | 否 | 3.0 行为未知 | -| 15 | syncCheckInterval | 是 | 否 | 3.0 行为未知 | -| 16 | maxTmrCtrl | 是 | 否 | 3.0 行为未知 | -| 17 | monitorReplica | 是 | 否 | 由 RAFT 协议管理多副本 | -| 18 | smlTagNullName | 是 | 否 | 3.0 行为未知 | -| 20 | ratioOfQueryCores | 是 | 否 | 由 线程池 相关配置参数决定 | -| 21 | maxStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 22 | maxFirstStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 23 | retryStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 24 | streamCompDelayRatio | 是 | 否 | 3.0 行为未知 | -| 25 | maxVgroupsPerDb | 是 | 否 | 由 create db 的参数 vgroups 指定实际 vgroups 数量 | -| 26 | maxTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 27 | minTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 28 | tableIncStepPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 29 | cache | 是 | 否 | 由 buffer 代替 cache\*blocks | -| 30 | blocks | 是 | 否 | 由 buffer 代替 cache\*blocks | -| 31 | days | 是 | 否 | 由 create db 的参数 duration 取代 | -| 32 | keep | 是 | 否 | 由 create db 的参数 keep 取代 | -| 33 | minRows | 是 | 否 | 由 create db 的参数 minRows 取代 | -| 34 | maxRows | 是 | 否 | 由 create db 的参数 maxRows 取代 | -| 35 | quorum | 是 | 否 | 由 RAFT 协议决定 | -| 36 | comp | 是 | 否 | 由 create db 的参数 comp 取代 | -| 37 | walLevel | 是 | 否 | 由 create db 的参数 wal_level 取代 | -| 38 | fsync | 是 | 否 | 由 create db 的参数 wal_fsync_period 取代 | -| 39 | replica | 是 | 否 | 由 create db 的参数 replica 取代 | -| 40 | partitions | 是 | 否 | 3.0 行为未知 | -| 41 | update | 是 | 否 | 允许更新部分列 | -| 42 | cachelast | 是 | 否 | 由 create db 的参数 cacheModel 取代 | -| 43 | maxSQLLength | 是 | 否 | SQL 上限为 1MB，无需参数控制 | -| 44 | maxWildCardsLength | 是 | 否 | 3.0 行为未知 | -| 45 | maxRegexStringLen | 是 | 否 | 3.0 行为未知 | -| 46 | maxNumOfOrderedRes | 是 | 否 | 3.0 行为未知 | -| 47 | maxConnections | 是 | 否 | 取决于系统配置和系统处理能力，详见后面的 Note | -| 48 | mnodeEqualVnodeNum | 是 | 否 | 3.0 行为未知 | -| 49 | http | 是 | 否 | http 服务由 taosAdapter 提供 | -| 50 | httpEnableRecordSql | 是 | 否 | taosd 不提供 http 服务 | -| 51 | httpMaxThreads | 是 | 否 | taosd 不提供 http 服务 | -| 52 | restfulRowLimit | 是 | 否 | taosd 不提供 http 服务 | -| 53 | httpDbNameMandatory | 是 | 否 | taosd 不提供 http 服务 | -| 54 | httpKeepAlive | 是 | 否 | taosd 不提供 http 服务 | -| 55 | enableRecordSql | 是 | 否 | 3.0 行为未知 | -| 56 | maxBinaryDisplayWidth | 是 | 否 | 3.0 行为未知 | -| 57 | stream | 是 | 否 | 默认启用连续查询 | -| 58 | retrieveBlockingModel | 是 | 否 | 3.0 行为未知 | -| 59 | tsdbMetaCompactRatio | 是 | 否 | 3.0 行为未知 | -| 60 | defaultJSONStrType | 是 | 否 | 3.0 行为未知 | -| 61 | walFlushSize | 是 | 否 | 3.0 行为未知 | -| 62 | keepTimeOffset | 是 | 否 | 3.0 行为未知 | -| 63 | flowctrl | 是 | 否 | 3.0 行为未知 | -| 64 | slaveQuery | 是 | 否 | 3.0 行为未知： slave vnode 是否能够处理查询？ | -| 65 | adjustMaster | 是 | 否 | 3.0 行为未知 | -| 66 | topicBinaryLen | 是 | 否 | 3.0 行为未知 | -| 67 | telegrafUseFieldNum | 是 | 否 | 3.0 行为未知 | -| 68 | deadLockKillQuery | 是 | 否 | 3.0 行为未知 | -| 69 | clientMerge | 是 | 否 | 3.0 行为未知 | -| 70 | sdbDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 71 | odbcDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 72 | httpDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 73 | monDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 74 | cqDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 75 | shortcutFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 76 | probeSeconds | 是 | 否 | 3.0 行为未知 | -| 77 | probeKillSeconds | 是 | 否 | 3.0 行为未知 | -| 78 | probeInterval | 是 | 否 | 3.0 行为未知 | -| 79 | lossyColumns | 是 | 否 | 3.0 行为未知 | -| 80 | fPrecision | 是 | 否 | 3.0 行为未知 | -| 81 | dPrecision | 是 | 否 | 3.0 行为未知 | -| 82 | maxRange | 是 | 否 | 3.0 行为未知 | -| 83 | range | 是 | 否 | 3.0 行为未知 | +| # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 | +| --- | :---------------------: | --------------- | --------------- | ---------------------------------------------------- | +| 1 | arbitrator | 是 | 否 | 通过 RAFT 协议选主 | +| 2 | numOfThreadsPerCore | 是 | 否 | 有其它参数设置多种线程池的大小 | +| 3 | numOfMnodes | 是 | 否 | 通过 create mnode 命令动态创建 mnode | +| 4 | vnodeBak | 是 | 否 | 3.0 行为未知 | +| 5 | balance | 是 | 否 | 负载均衡功能由 split/merge vgroups 实现 （暂不支持） | +| 6 | balanceInterval | 是 | 否 | 随着 balance 参数失效 | +| 7 | offlineThreshold | 是 | 否 | 3.0 行为未知 | +| 8 | role | 是 | 否 | 由 supportVnode 决定是否能够创建 | +| 9 | dnodeNopLoop | 是 | 否 | 2.6 文档中未找到此参数 | +| 10 | rpcTimer | 是 | 否 | 3.0 行为未知 | +| 11 | rpcMaxTime | 是 | 否 | 3.0 行为未知 | +| 12 | rpcForceTcp | 是 | 否 | 默认为 TCP | +| 13 | tcpConnTimeout | 是 | 否 | 3.0 行为未知 | +| 14 | syncCheckInterval | 是 | 否 | 3.0 行为未知 | +| 15 | maxTmrCtrl | 是 | 否 | 3.0 行为未知 | +| 16 | monitorReplica | 是 | 否 | 由 RAFT 协议管理多副本 | +| 17 | smlTagNullName | 是 | 否 | 3.0 行为未知 | +| 18 | ratioOfQueryCores | 是 | 否 | 由 线程池 相关配置参数决定 | +| 19 | maxStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 20 | maxFirstStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 21 | retryStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 22 | streamCompDelayRatio | 是 | 否 | 3.0 行为未知 | +| 23 | maxVgroupsPerDb | 是 | 否 | 由 create db 的参数 vgroups 指定实际 vgroups 数量 | +| 24 | maxTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 25 | minTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 26 | tableIncStepPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 27 | cache | 是 | 否 | 由 buffer 代替 cache\*blocks | +| 28 | blocks | 是 | 否 | 由 buffer 代替 cache\*blocks | +| 29 | days | 是 | 否 | 由 create db 的参数 duration 取代 | +| 30 | keep | 是 | 否 | 由 create db 的参数 keep 取代 | +| 31 | minRows | 是 | 否 | 由 create db 的参数 minRows 取代 | +| 32 | maxRows | 是 | 否 | 由 create db 的参数 maxRows 取代 | +| 33 | quorum | 是 | 否 | 由 RAFT 协议决定 | +| 34 | comp | 是 | 否 | 由 create db 的参数 comp 取代 | +| 35 | walLevel | 是 | 否 | 由 create db 的参数 wal_level 取代 | +| 36 | fsync | 是 | 否 | 由 create db 的参数 wal_fsync_period 取代 | +| 37 | replica | 是 | 否 | 由 create db 的参数 replica 取代 | +| 38 | partitions | 是 | 否 | 3.0 行为未知 | +| 39 | update | 是 | 否 | 允许更新部分列 | +| 40 | cachelast | 是 | 否 | 由 create db 的参数 cacheModel 取代 | +| 41 | maxSQLLength | 是 | 否 | SQL 上限为 1MB，无需参数控制 | +| 42 | maxWildCardsLength | 是 | 否 | 3.0 行为未知 | +| 43 | maxRegexStringLen | 是 | 否 | 3.0 行为未知 | +| 44 | maxNumOfOrderedRes | 是 | 否 | 3.0 行为未知 | +| 45 | maxConnections | 是 | 否 | 取决于系统配置和系统处理能力，详见后面的 Note | +| 46 | mnodeEqualVnodeNum | 是 | 否 | 3.0 行为未知 | +| 47 | http | 是 | 否 | http 服务由 taosAdapter 提供 | +| 48 | httpEnableRecordSql | 是 | 否 | taosd 不提供 http 服务 | +| 49 | httpMaxThreads | 是 | 否 | taosd 不提供 http 服务 | +| 50 | restfulRowLimit | 是 | 否 | taosd 不提供 http 服务 | +| 51 | httpDbNameMandatory | 是 | 否 | taosd 不提供 http 服务 | +| 52 | httpKeepAlive | 是 | 否 | taosd 不提供 http 服务 | +| 53 | enableRecordSql | 是 | 否 | 3.0 行为未知 | +| 54 | maxBinaryDisplayWidth | 是 | 否 | 3.0 行为未知 | +| 55 | stream | 是 | 否 | 默认启用连续查询 | +| 56 | retrieveBlockingModel | 是 | 否 | 3.0 行为未知 | +| 57 | tsdbMetaCompactRatio | 是 | 否 | 3.0 行为未知 | +| 58 | defaultJSONStrType | 是 | 否 | 3.0 行为未知 | +| 59 | walFlushSize | 是 | 否 | 3.0 行为未知 | +| 60 | flowctrl | 是 | 否 | 3.0 行为未知 | +| 61 | slaveQuery | 是 | 否 | 3.0 行为未知： slave vnode 是否能够处理查询？ | +| 62 | adjustMaster | 是 | 否 | 3.0 行为未知 | +| 63 | topicBinaryLen | 是 | 否 | 3.0 行为未知 | +| 64 | telegrafUseFieldNum | 是 | 否 | 3.0 行为未知 | +| 65 | deadLockKillQuery | 是 | 否 | 3.0 行为未知 | +| 66 | clientMerge | 是 | 否 | 3.0 行为未知 | +| 67 | sdbDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 68 | odbcDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 69 | httpDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 70 | monDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 71 | cqDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 72 | shortcutFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 73 | probeSeconds | 是 | 否 | 3.0 行为未知 | +| 74 | probeKillSeconds | 是 | 否 | 3.0 行为未知 | +| 75 | probeInterval | 是 | 否 | 3.0 行为未知 | +| 76 | lossyColumns | 是 | 否 | 3.0 行为未知 | +| 77 | fPrecision | 是 | 否 | 3.0 行为未知 | +| 78 | dPrecision | 是 | 否 | 3.0 行为未知 | +| 79 | maxRange | 是 | 否 | 3.0 行为未知 | +| 80 | range | 是 | 否 | 3.0 行为未知 | diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index e5f232c1fc506a6e37cb128cab129a7fe539d60a..9f5bae081cd274fbf8e91759539227b455a75deb 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -35,12 +35,32 @@ tag_set 中的所有的数据自动转化为 nchar 数据类型，并不需要 - 如果两边有英文双引号，表示 BINARY(32) 类型。例如 `"abc"`。 - 如果两边有英文双引号而且带有 L 前缀，表示 NCHAR(32) 类型。例如 `L"报错信息"`。 -- 对空格、等号（=）、逗号（,）、双引号（"），前面需要使用反斜杠（\）进行转义。（都指的是英文半角符号） +- 对空格、等号（=）、逗号（,）、双引号（"）、反斜杠（\），前面需要使用反斜杠（\）进行转义。（都指的是英文半角符号）。具体转义规则如下： + +| **序号** | **域** | **需转义字符** | +| -------- | ----------- | ----------------------------- | +| 1 | 超级表名 | 逗号，空格 | +| 2 | 标签名 | 逗号，等号，空格 | +| 3 | 标签值 | 逗号，等号，空格 | +| 4 | 列名 | 逗号，等号，空格 | +| 5 | 列值 | 双引号，反斜杠 | + +两个连续的反斜杠，第一个作为转义符，只有一个反斜杠则无需转义. 反斜杠转义规则举例如下: + +| **序号** | **反斜杠** | **转义为** | +| -------- | ----------- | ----------------------------- | +| 1 | \ | \ | +| 2 | \\\\ | \ | +| 3 | \\\\\\ | \\\\ | +| 4 | \\\\\\\\ | \\\\ | +| 5 | \\\\\\\\\\ | \\\\\\ | +| 6 | \\\\\\\\\\\\ | \\\\\\ | + - 数值类型将通过后缀来区分数据类型： -| **序号** | **后缀** | **映射类型** | **大小(字节)** | +| **序号** | **后缀** | **映射类型** | **大小(字节)** | | -------- | ----------- | ----------------------------- | -------------- | -| 1 | 无或 f64 | double | 8 | +| 1 | 无或 f64 | double | 8 | | 2 | f32 | float | 4 | | 3 | i8/u8 | TinyInt/UTinyInt | 1 | | 4 | i16/u16 | SmallInt/USmallInt | 2 | @@ -84,10 +104,12 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 6. 对 BINARY 或 NCHAR 列，如果数据行中所提供值的长度超出了列类型的限制，自动增加该列允许存储的字符长度上限（只增不减），以保证数据的完整保存。 7. 整个处理过程中遇到的错误会中断写入过程，并返回错误代码。 8. 为了提高写入的效率，默认假设同一个超级表中 field_set 的顺序是一样的（第一条数据包含所有的 field，后面的数据按照这个顺序），如果顺序不一样，需要配置参数 smlDataFormat 为 false，否则，数据写入按照相同顺序写入，库中数据会异常，从3.0.3.0开始，自动检测顺序是否一致，该配置废弃。 - +9. 由于sql建表表名不支持点号（.），所以schemaless也对点号（.）做了处理，如果schemaless自动建表的表名如果有点号（.），会自动替换为下划线（\_）。如果手动指定子表名的话，子表名里有点号（.），同样转化为下划线（\_）。 +10. taos.cfg 增加 smlTsDefaultName 配置（值为字符串），只在client端起作用，配置后，schemaless自动建表的时间列名字可以通过该配置设置。不配置的话，默认为 _ts + :::tip 无模式所有的处理逻辑，仍会遵循 TDengine 对数据结构的底层限制，例如每行数据的总长度不能超过 -48KB，标签值的总长度不超过16KB。这方面的具体限制约束请参见 [TDengine SQL 边界限制](/taos-sql/limit) +48KB（从 3.0.5.0 版本开始为 64KB），标签值的总长度不超过16KB。这方面的具体限制约束请参见 [TDengine SQL 边界限制](/taos-sql/limit) ::: diff --git a/docs/zh/17-operation/10-monitor.md b/docs/zh/17-operation/10-monitor.md index 7def90916c9ba5b75c4d1e1be8a2836a5251eec6..4f8dccc78d80b8f4dd107f481317d8f82c1be207 100644 --- a/docs/zh/17-operation/10-monitor.md +++ b/docs/zh/17-operation/10-monitor.md @@ -32,7 +32,7 @@ chmod +x TDinsight.sh - 使用已经存在的 Grafana Notification Channel `uid`，参数 `-E`。该参数可以使用 `curl -u admin:admin localhost:3000/api/alert-notifications |jq` 来获取。 ```bash - sudo ./TDinsight.sh -a http://localhost:6041 -u root -p taosdata -E + ./TDinsight.sh -a http://localhost:6041 -u root -p taosdata -E ``` 运行程序并重启 Grafana 服务，打开面板：`http://localhost:3000/d/tdinsight`。 @@ -210,19 +210,6 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### logs 表 - -`logs` 表记录登录信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|level|VARCHAR||log level| -|content|NCHAR||log content，长度不超过1024字节| -|dnode\_id|INT|TAG|dnode id| -|dnode\_ep|NCHAR|TAG|dnode endpoint| -|cluster\_id|NCHAR|TAG|cluster id| - ### log\_summary 表 `log_summary` 记录日志统计信息。 @@ -270,7 +257,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |field|type|is\_tag|comment| |:----|:---|:-----|:------| |\_ts|TIMESTAMP||timestamp| -|guage|DOUBLE||监控指标值| +|gauge|DOUBLE||监控指标值| |client\_ip|NCHAR|TAG|client ip| |endpoint|NCHAR|TAG|taosadpater endpoint| |request\_method|NCHAR|TAG|request method| @@ -284,7 +271,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |field|type|is\_tag|comment| |:----|:---|:-----|:------| |\_ts|TIMESTAMP||timestamp| -|guage|DOUBLE||监控指标值| +|gauge|DOUBLE||监控指标值| |client\_ip|NCHAR|TAG|client ip| |endpoint|NCHAR|TAG|taosadpater endpoint| |request\_method|NCHAR|TAG|request method| @@ -298,7 +285,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |field|type|is\_tag|comment| |:----|:---|:-----|:------| |\_ts|TIMESTAMP||timestamp| -|guage|DOUBLE||监控指标值| +|gauge|DOUBLE||监控指标值| |endpoint|NCHAR|TAG|taosadpater endpoint| ### taosadapter\_restful\_http\_request\_summary\_milliseconds 表 @@ -326,7 +313,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |field|type|is\_tag|comment| |:----|:---|:-----|:------| |\_ts|TIMESTAMP||timestamp| -|guage|DOUBLE||监控指标值| +|gauge|DOUBLE||监控指标值| |endpoint|NCHAR|TAG|taosadpater endpoint| ### taosadapter\_system\_cpu\_percent 表 @@ -336,5 +323,5 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |field|type|is\_tag|comment| |:----|:---|:-----|:------| |\_ts|TIMESTAMP||timestamp| -|guage|DOUBLE||监控指标值| +|gauge|DOUBLE||监控指标值| |endpoint|NCHAR|TAG|taosadpater endpoint| diff --git a/docs/zh/20-third-party/11-kafka.md b/docs/zh/20-third-party/11-kafka.md index 75d8deebb190c8cc14211d34950614cfefd90a1a..dc4f25cbe8422cc73db129b25bacac539cc20130 100644 --- a/docs/zh/20-third-party/11-kafka.md +++ b/docs/zh/20-third-party/11-kafka.md @@ -16,169 +16,78 @@ TDengine Source Connector 用于把数据实时地从 TDengine 读出来发送 ![TDengine Database Kafka Connector -- streaming integration with kafka connect](kafka/streaming-integration-with-kafka-connect.webp) -## 什么是 Confluent？ - -[Confluent](https://www.confluent.io/) 在 Kafka 的基础上增加很多扩展功能。包括： - -1. Schema Registry -2. REST 代理 -3. 非 Java 客户端 -4. 很多打包好的 Kafka Connect 插件 -5. 管理和监控 Kafka 的 GUI —— Confluent 控制中心 - -这些扩展功能有的包含在社区版本的 Confluent 中，有的只有企业版能用。 -![TDengine Database Kafka Connector -- Confluent introduction](kafka/confluentPlatform.webp) - -Confluent 企业版提供了 `confluent` 命令行工具管理各个组件。 - ## 前置条件 运行本教程中示例的前提条件。 1. Linux 操作系统 2. 已安装 Java 8 和 Maven -3. 已安装 Git +3. 已安装 Git、curl、vi 4. 已安装并启动 TDengine。如果还没有可参考[安装和卸载](/operation/pkg-install) -## 安装 Confluent - -Confluent 提供了 Docker 和二进制包两种安装方式。本文仅介绍二进制包方式安装。 +## 安装 Kafka 在任意目录下执行： -``` -curl -O http://packages.confluent.io/archive/7.1/confluent-7.1.1.tar.gz -tar xzf confluent-7.1.1.tar.gz -C /opt/ +```shell +curl -O https://downloads.apache.org/kafka/3.4.0/kafka_2.13-3.4.0.tgz +tar xzf kafka_2.13-3.4.0.tgz -C /opt/ +ln -s /opt/kafka_2.13-3.4.0 /opt/kafka ``` -然后需要把 `$CONFLUENT_HOME/bin` 目录加入 PATH。 +然后需要把 `$KAFKA_HOME/bin` 目录加入 PATH。 ```title=".profile" -export CONFLUENT_HOME=/opt/confluent-7.1.1 -export PATH=$CONFLUENT_HOME/bin:$PATH +export KAFKA_HOME=/opt/kafka +export PATH=$PATH:$KAFKA_HOME/bin ``` 以上脚本可以追加到当前用户的 profile 文件（~/.profile 或 ~/.bash_profile） -安装完成之后，可以输入`confluent version`做简单验证： - -``` -# confluent version -confluent - Confluent CLI - -Version: v2.6.1 -Git Ref: 6d920590 -Build Date: 2022-02-18T06:14:21Z -Go Version: go1.17.6 (linux/amd64) -Development: false -``` - ## 安装 TDengine Connector 插件 -### 从源码安装 +### 编译插件 -``` +```shell git clone --branch 3.0 https://github.com/taosdata/kafka-connect-tdengine.git cd kafka-connect-tdengine -mvn clean package -unzip -d $CONFLUENT_HOME/share/java/ target/components/packages/taosdata-kafka-connect-tdengine-*.zip +mvn clean package -Dmaven.test.skip=true +unzip -d $KAFKA_HOME/components/ target/components/packages/taosdata-kafka-connect-tdengine-*.zip ``` -以上脚本先 clone 项目源码，然后用 Maven 编译打包。打包完成后在 `target/components/packages/` 目录生成了插件的 zip 包。把这个 zip 包解压到安装插件的路径即可。上面的示例中使用了内置的插件安装路径： `$CONFLUENT_HOME/share/java/`。 +以上脚本先 clone 项目源码，然后用 Maven 编译打包。打包完成后在 `target/components/packages/` 目录生成了插件的 zip 包。把这个 zip 包解压到安装插件的路径即可。上面的示例中使用了内置的插件安装路径： `$KAFKA_HOME/components/`。 -### 用 confluent-hub 安装 +### 配置插件 -[Confluent Hub](https://www.confluent.io/hub) 提供下载 Kafka Connect 插件的服务。在 TDengine Kafka Connector 发布到 Confluent Hub 后可以使用命令工具 `confluent-hub` 安装。 -**TDengine Kafka Connector 目前没有正式发布，不能用这种方式安装**。 +将 kafka-connect-tdengine 插件加入 `$KAFKA_HOME/config/connect-distributed.properties` 配置文件 plugin.path 中 -## 启动 Confluent - -``` -confluent local services start +```properties +plugin.path=/usr/share/java,/opt/kafka/components ``` -:::note -一定要先安装插件再启动 Confluent, 否则加载插件会失败。 -::: +## 启动 Kafka -:::tip -若某组件启动失败，可尝试清空数据，重新启动。数据目录在启动时将被打印到控制台，比如 ： - -```title="控制台输出日志" {1} -Using CONFLUENT_CURRENT: /tmp/confluent.106668 -Starting ZooKeeper -ZooKeeper is [UP] -Starting Kafka -Kafka is [UP] -Starting Schema Registry -Schema Registry is [UP] -Starting Kafka REST -Kafka REST is [UP] -Starting Connect -Connect is [UP] -Starting ksqlDB Server -ksqlDB Server is [UP] -Starting Control Center -Control Center is [UP] -``` +```shell +zookeeper-server-start.sh -daemon $KAFKA_HOME/config/zookeeper.properties -清空数据可执行 `rm -rf /tmp/confluent.106668`。 -::: +kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties -### 验证各个组件是否启动成功 - -输入命令： - -``` -confluent local services status -``` - -如果各组件都启动成功，会得到如下输出： - -``` -Connect is [UP] -Control Center is [UP] -Kafka is [UP] -Kafka REST is [UP] -ksqlDB Server is [UP] -Schema Registry is [UP] -ZooKeeper is [UP] +connect-distributed.sh -daemon $KAFKA_HOME/config/connect-distributed.properties ``` -### 验证插件是否安装成功 +### 验证 kafka Connect 是否启动成功 -在 Kafka Connect 组件完全启动后，可用以下命令列出成功加载的插件： +输入命令： -``` -confluent local services connect plugin list +```shell +curl http://localhost:8083/connectors ``` -如果成功安装，会输出如下： - -```txt {4,9} -Available Connect Plugins: -[ - { - "class": "com.taosdata.kafka.connect.sink.TDengineSinkConnector", - "type": "sink", - "version": "1.0.0" - }, - { - "class": "com.taosdata.kafka.connect.source.TDengineSourceConnector", - "type": "source", - "version": "1.0.0" - }, -...... -``` +如果各组件都启动成功，会得到如下输出： -如果插件安装失败，请检查 Kafka Connect 的启动日志是否有异常信息，用以下命令输出日志路径： +```txt +[] ``` -echo `cat /tmp/confluent.current`/connect/connect.stdout -``` -该命令的输出类似： `/tmp/confluent.104086/connect/connect.stdout`。 - -与日志文件 `connect.stdout` 同一目录，还有一个文件名为： `connect.properties`。在这个文件的末尾，可以看到最终生效的 `plugin.path`， 它是一系列用逗号分割的路径。如果插件安装失败，很可能是因为实际的安装路径不包含在 `plugin.path` 中。 - ## TDengine Sink Connector 的使用 @@ -188,40 +97,47 @@ TDengine Sink Connector 内部使用 TDengine [无模式写入接口](../../conn 下面的示例将主题 meters 的数据，同步到目标数据库 power。数据格式为 InfluxDB Line 协议格式。 -### 添加配置文件 +### 添加 Sink Connector 配置文件 -``` +```shell mkdir ~/test cd ~/test -vi sink-demo.properties +vi sink-demo.json ``` -sink-demo.properties 内容如下： - -```ini title="sink-demo.properties" -name=TDengineSinkConnector -connector.class=com.taosdata.kafka.connect.sink.TDengineSinkConnector -tasks.max=1 -topics=meters -connection.url=jdbc:TAOS://127.0.0.1:6030 -connection.user=root -connection.password=taosdata -connection.database=power -db.schemaless=line -data.precision=ns -key.converter=org.apache.kafka.connect.storage.StringConverter -value.converter=org.apache.kafka.connect.storage.StringConverter +sink-demo.json 内容如下： + +```json title="sink-demo.json" +{ + "name": "TDengineSinkConnector", + "config": { + "connector.class":"com.taosdata.kafka.connect.sink.TDengineSinkConnector", + "tasks.max": "1", + "topics": "meters", + "connection.url": "jdbc:TAOS://127.0.0.1:6030", + "connection.user": "root", + "connection.password": "taosdata", + "connection.database": "power", + "db.schemaless": "line", + "data.precision": "ns", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.storage.StringConverter", + "errors.tolerance": "all", + "errors.deadletterqueue.topic.name": "dead_letter_topic", + "errors.deadletterqueue.topic.replication.factor": 1 + } +} ``` 关键配置说明： -1. `topics=meters` 和 `connection.database=power`, 表示订阅主题 meters 的数据，并写入数据库 power。 -2. `db.schemaless=line`, 表示使用 InfluxDB Line 协议格式的数据。 +1. `"topics": "meters"` 和 `"connection.database": "power"`, 表示订阅主题 meters 的数据，并写入数据库 power。 +2. `"db.schemaless": "line"`, 表示使用 InfluxDB Line 协议格式的数据。 -### 创建 Connector 实例 +### 创建 Sink Connector 实例 -``` -confluent local services connect connector load TDengineSinkConnector --config ./sink-demo.properties +```shell +curl -X POST -d @sink-demo.json http://localhost:8083/connectors -H "Content-Type: application/json" ``` 若以上命令执行成功，则有如下输出： @@ -241,7 +157,10 @@ confluent local services connect connector load TDengineSinkConnector --config . "tasks.max": "1", "topics": "meters", "value.converter": "org.apache.kafka.connect.storage.StringConverter", - "name": "TDengineSinkConnector" + "name": "TDengineSinkConnector", + "errors.tolerance": "all", + "errors.deadletterqueue.topic.name": "dead_letter_topic", + "errors.deadletterqueue.topic.replication.factor": "1", }, "tasks": [], "type": "sink" @@ -261,8 +180,8 @@ meters,location=California.LosAngeles,groupid=3 current=11.3,voltage=221,phase=0 使用 kafka-console-producer 向主题 meters 添加测试数据。 -``` -cat test-data.txt | kafka-console-producer --broker-list localhost:9092 --topic meters +```shell +cat test-data.txt | kafka-console-producer.sh --broker-list localhost:9092 --topic meters ``` :::note @@ -273,12 +192,12 @@ cat test-data.txt | kafka-console-producer --broker-list localhost:9092 --topic 使用 TDengine CLI 验证同步是否成功。 -``` +```sql taos> use power; Database changed. taos> select * from meters; - ts | current | voltage | phase | groupid | location | + _ts | current | voltage | phase | groupid | location | =============================================================================================================================================================== 2022-03-28 09:56:51.249000000 | 11.800000000 | 221.000000000 | 0.280000000 | 2 | California.LosAngeles | 2022-03-28 09:56:51.250000000 | 13.400000000 | 223.000000000 | 0.290000000 | 2 | California.LosAngeles | @@ -295,32 +214,39 @@ TDengine Source Connector 的作用是将 TDengine 某个数据库某一时刻 TDengine Source Connector 会将 TDengine 数据表中的数据转换成 [InfluxDB Line 协议格式](/develop/insert-data/influxdb-line/) 或 [OpenTSDB JSON 协议格式](/develop/insert-data/opentsdb-json)， 然后写入 Kafka。 -下面的示例程序同步数据库 test 中的数据到主题 tdengine-source-test。 +下面的示例程序同步数据库 test 中的数据到主题 tdengine-test-meters。 -### 添加配置文件 +### 添加 Source Connector 配置文件 -``` -vi source-demo.properties +```shell +vi source-demo.json ``` 输入以下内容： -```ini title="source-demo.properties" -name=TDengineSourceConnector -connector.class=com.taosdata.kafka.connect.source.TDengineSourceConnector -tasks.max=1 -connection.url=jdbc:TAOS://127.0.0.1:6030 -connection.username=root -connection.password=taosdata -connection.database=test -connection.attempts=3 -connection.backoff.ms=5000 -topic.prefix=tdengine-source- -poll.interval.ms=1000 -fetch.max.rows=100 -out.format=line -key.converter=org.apache.kafka.connect.storage.StringConverter -value.converter=org.apache.kafka.connect.storage.StringConverter +```json title="source-demo.json" +{ + "name":"TDengineSourceConnector", + "config":{ + "connector.class": "com.taosdata.kafka.connect.source.TDengineSourceConnector", + "tasks.max": 1, + "connection.url": "jdbc:TAOS://127.0.0.1:6030", + "connection.username": "root", + "connection.password": "taosdata", + "connection.database": "test", + "connection.attempts": 3, + "connection.backoff.ms": 5000, + "topic.prefix": "tdengine", + "topic.delimiter": "-", + "poll.interval.ms": 1000, + "fetch.max.rows": 100, + "topic.per.stable": true, + "topic.ignore.db": false, + "out.format": "line", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.storage.StringConverter" + } +} ``` ### 准备测试数据 @@ -345,27 +271,27 @@ INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES('2018-1 使用 TDengine CLI, 执行 SQL 文件。 -``` +```shell taos -f prepare-source-data.sql ``` -### 创建 Connector 实例 +### 创建 Source Connector 实例 -``` -confluent local services connect connector load TDengineSourceConnector --config source-demo.properties +```shell +curl -X POST -d @source-demo.json http://localhost:8083/connectors -H "Content-Type: application/json" ``` ### 查看 topic 数据 -使用 kafka-console-consumer 命令行工具监控主题 tdengine-source-test 中的数据。一开始会输出所有历史数据， 往 TDengine 插入两条新的数据之后，kafka-console-consumer 也立即输出了新增的两条数据。 +使用 kafka-console-consumer 命令行工具监控主题 tdengine-test-meters 中的数据。一开始会输出所有历史数据， 往 TDengine 插入两条新的数据之后，kafka-console-consumer 也立即输出了新增的两条数据。 输出数据 InfluxDB line protocol 的格式。 -``` -kafka-console-consumer --bootstrap-server localhost:9092 --from-beginning --topic tdengine-source-test +```shell +kafka-console-consumer.sh --bootstrap-server localhost:9092 --from-beginning --topic tdengine-test-meters ``` 输出： -``` +```txt ...... meters,location="California.SanFrancisco",groupid=2i32 current=10.3f32,voltage=219i32,phase=0.31f32 1538548685000000000 meters,location="California.SanFrancisco",groupid=2i32 current=12.6f32,voltage=218i32,phase=0.33f32 1538548695000000000 @@ -374,7 +300,7 @@ meters,location="California.SanFrancisco",groupid=2i32 current=12.6f32,voltage=2 此时会显示所有历史数据。切换到 TDengine CLI， 插入两条新的数据： -``` +```sql USE test; INSERT INTO d1001 VALUES (now, 13.3, 229, 0.38); INSERT INTO d1002 VALUES (now, 16.3, 233, 0.22); @@ -388,15 +314,15 @@ INSERT INTO d1002 VALUES (now, 16.3, 233, 0.22); 查看当前活跃的 connector： -``` -confluent local services connect connector status +```shell +curl http://localhost:8083/connectors ``` 如果按照前述操作，此时应有两个活跃的 connector。使用下面的命令 unload： -``` -confluent local services connect connector unload TDengineSinkConnector -confluent local services connect connector unload TDengineSourceConnector +```shell +curl -X DELETE http://localhost:8083/connectors/TDengineSinkConnector +curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector ``` ## 配置参考 @@ -434,23 +360,27 @@ confluent local services connect connector unload TDengineSourceConnector ### TDengine Source Connector 特有的配置 1. `connection.database`: 源数据库名称，无缺省值。 -2. `topic.prefix`： 数据导入 kafka 后 topic 名称前缀。 使用 `topic.prefix` + `connection.database` 名称作为完整 topic 名。默认为空字符串 ""。 -3. `timestamp.initial`: 数据同步起始时间。格式为'yyyy-MM-dd HH:mm:ss'。默认为 "1970-01-01 00:00:00"。 -4. `poll.interval.ms`: 拉取数据间隔，单位为 ms。默认为 1000。 +2. `topic.prefix`： 数据导入 kafka 时使用的 topic 名称的前缀。默认为空字符串 ""。 +3. `timestamp.initial`: 数据同步起始时间。格式为'yyyy-MM-dd HH:mm:ss'，若未指定则从指定 DB 中最早的一条记录开始。 +4. `poll.interval.ms`: 检查是否有新建或删除的表的时间间隔，单位为 ms。默认为 1000。 5. `fetch.max.rows` : 检索数据库时最大检索条数。 默认为 100。 -6. `out.format`: 数据格式。取值 line 或 json。line 表示 InfluxDB Line 协议格式， json 表示 OpenTSDB JSON 格式。默认为 line。 +6. `query.interval.ms`: 从 TDengine 一次读取数据的时间跨度，需要根据表中的数据特征合理配置，避免一次查询的数据量过大或过小；在具体的环境中建议通过测试设置一个较优值，默认值为 0，即获取到当前最新时间的所有数据。 +7. `out.format` : 结果集输出格式。`line` 表示输出格式为 InfluxDB Line 协议格式，`json` 表示输出格式是 json。默认为 line。 +8. `topic.per.stable`: 如果设置为 true，表示一个超级表对应一个 Kafka topic，topic的命名规则 ``；如果设置为 false，则指定的 DB 中的所有数据进入一个 Kafka topic，topic 的命名规则为 `` +9. `topic.ignore.db`: topic 命名规则是否包含 database 名称，true 表示规则为 ``，false 表示规则为 ``，默认 false。此配置项在 `topic.per.stable` 设置为 false 时不生效。 +10. `topic.delimiter`: topic 名称分割符，默认为 `-`。 +11. `read.method`: 从 TDengine 读取数据方式，query 或是 subscription。默认为 subscription。 +12. `subscription.group.id`: 指定 TDengine 数据订阅的组 id，当 `read.method` 为 subscription 时，此项为必填项。 +13. `subscription.from`: 指定 TDengine 数据订阅起始位置，latest 或是 earliest。默认为 latest。 ## 其他说明 -1. 插件的安装位置可以自定义，请参考官方文档：https://docs.confluent.io/home/connect/self-managed/install.html#install-connector-manually。 -2. 本教程的示例程序使用了 Confluent 平台，但是 TDengine Kafka Connector 本身同样适用于独立安装的 Kafka, 且配置方法相同。关于如何在独立安装的 Kafka 环境使用 Kafka Connect 插件， 请参考官方文档： https://kafka.apache.org/documentation/#connect。 +1. 关于如何在独立安装的 Kafka 环境使用 Kafka Connect 插件， 请参考官方文档：。 ## 问题反馈 -无论遇到任何问题，都欢迎在本项目的 Github 仓库反馈： https://github.com/taosdata/kafka-connect-tdengine/issues。 +无论遇到任何问题，都欢迎在本项目的 Github 仓库反馈：。 ## 参考 -1. https://www.confluent.io/what-is-apache-kafka -2. https://developer.confluent.io/learn-kafka/kafka-connect/intro -3. https://docs.confluent.io/platform/current/platform.html +1. diff --git a/docs/zh/20-third-party/13-dbeaver.md b/docs/zh/20-third-party/13-dbeaver.md new file mode 100644 index 0000000000000000000000000000000000000000..c096fd41a506111dd39ce65de0516aa12e9e5789 --- /dev/null +++ b/docs/zh/20-third-party/13-dbeaver.md @@ -0,0 +1,37 @@ +--- +sidebar_label: DBeaver +title: DBeaver +description: 使用 DBeaver 存取 TDengine 数据的详细指南 +--- + +DBeaver 是一款流行的跨平台数据库管理工具，方便开发者、数据库管理员、数据分析师等用户管理数据。DBeaver 从 23.1.1 版本开始内嵌支持 TDengine。既支持独立部署的 TDengine 集群也支持 TDengine Cloud。 + +## 前置条件 + +使用 DBeaver 管理 TDengine 需要以下几方面的准备工作。 + +- 安装 DBeaver。DBeaver 支持主流操作系统包括 Windows、macOS 和 Linux。请注意[下载](https://dbeaver.io/download/)正确平台和版本（23.1.1+）的安装包。详细安装步骤请参考 [DBeaver 官方文档](https://github.com/dbeaver/dbeaver/wiki/Installation)。 +- 如果使用独立部署的 TDengine 集群，请确认 TDengine 正常运行，并且 taosAdapter 已经安装并正常运行，具体细节请参考 [taosAdapter 的使用手册](/reference/taosadapter)。 + +## 使用 DBeaver 访问内部部署的 TDengine + +1. 启动 DBeaver 应用，点击按钮或菜单项选择“连接到数据库”，然后在时间序列分类栏中选择 TDengine。 + + ![DBeaver 连接 TDengine](./dbeaver/dbeaver-connect-tdengine-zh.webp) + +2. 配置 TDengine 连接，填入主机地址、端口号、用户名和密码。如果 TDengine 部署在本机，可以只填用户名和密码，默认用户名为 root，默认密码为 taosdata。点击“测试连接”可以对连接是否可用进行测试。如果本机没有安装 TDengine Java + 连接器，DBeaver 会提示下载安装。 + + ![配置 TDengine 连接](./dbeaver/dbeaver-config-tdengine-zh.webp) + +3. 连接成功将显示如下图所示。如果显示连接失败，请检查 TDengine 服务和 taosAdapter 是否正确运行，主机地址、端口号、用户名和密码是否正确。 + + ![连接成功](./dbeaver/dbeaver-connect-tdengine-test-zh.webp) + +4. 使用 DBeaver 选择数据库和表可以浏览 TDengine 服务的数据。 + + ![DBeaver 浏览 TDengine 数据](./dbeaver/dbeaver-browse-data-zh.webp) + +5. 也可以通过执行 SQL 命令的方式对 TDengine 数据进行操作。 + + ![DBeaver SQL 命令](./dbeaver/dbeaver-sql-execution-zh.webp) diff --git a/docs/zh/20-third-party/50-qstudio.md b/docs/zh/20-third-party/50-qstudio.md new file mode 100644 index 0000000000000000000000000000000000000000..0fc1ba6970230d066ff2d6559aaa9f9bac850868 --- /dev/null +++ b/docs/zh/20-third-party/50-qstudio.md @@ -0,0 +1,41 @@ +--- +sidebar_label: qStudio +title: qStudio +description: 使用 qStudio 存取 TDengine 数据的详细指南 +--- + +qStudio 是一款免费的多平台 SQL 数据分析工具，可以轻松浏览数据库中的表、变量、函数和配置设置。最新版本 qStudio 内嵌支持 TDengine。 + +## 前置条件 + +使用 qStudio 连接 TDengine 需要以下几方面的准备工作。 + +- 安装 qStudio。qStudio 支持主流操作系统包括 Windows、macOS 和 Linux。请注意[下载](https://www.timestored.com/qstudio/download/)正确平台的安装包。 +- 安装 TDengine 实例，请确认 TDengine 正常运行，并且 taosAdapter 已经安装并正常运行，具体细节请参考 [taosAdapter 的使用手册](/reference/taosadapter)。 + +## 使用 qStudio 连接 TDengine + +1. 启动 qStudio 应用，从菜单项选择“Server” 和 “Add Server...”，然后在 Server Type 下拉框中选择 TDengine。 + + ![qStudio 连接 TDengine](./qstudio/qstudio-connect-tdengine.webp) + +2. 配置 TDengine 连接，填入主机地址、端口号、用户名和密码。如果 TDengine 部署在本机，可以只填用户名和密码，默认用户名为 root，默认密码为 taosdata。点击“Test”可以对连接是否可用进行测试。如果本机没有安装 TDengine Java + 连接器，qStudio 会提示下载安装。 + + ![下载 Java 连接器](./qstudio/qstudio-jdbc-connector-download.webp) + +3. 连接成功将显示如下图所示。如果显示连接失败，请检查 TDengine 服务和 taosAdapter 是否正确运行，主机地址、端口号、用户名和密码是否正确。 + + ![连接成功](./qstudio/qstudio-connect-tdengine-test.webp) + +4. 使用 qStudio 选择数据库和表可以浏览 TDengine 服务的数据。 + + ![qStudio 浏览 TDengine 数据](./qstudio/qstudio-browse-data.webp) + +5. 也可以通过执行 SQL 命令的方式对 TDengine 数据进行操作。 + + ![qStudio SQL 命令](./qstudio/qstudio-sql-execution.webp) + +6. qStudio 支持根据数据绘制图表等功能，请参考 [qStudio 的帮助文档](https://www.timestored.com/qstudio/help) + + ![qStudio Chart 功能](./qstudio/qstudio-chart.webp) diff --git a/docs/zh/20-third-party/dbeaver/dbeaver-browse-cloud-data-zh.webp b/docs/zh/20-third-party/dbeaver/dbeaver-browse-cloud-data-zh.webp new file mode 100644 index 0000000000000000000000000000000000000000..28580c9dfbbea61c1d7225ee4e632ee4258207b8 Binary files /dev/null and b/docs/zh/20-third-party/dbeaver/dbeaver-browse-cloud-data-zh.webp differ diff --git a/docs/zh/20-third-party/dbeaver/dbeaver-browse-data-zh.webp b/docs/zh/20-third-party/dbeaver/dbeaver-browse-data-zh.webp new file mode 100644 index 0000000000000000000000000000000000000000..b2faca92b2b6d3fa707c8863de4fb77fdeb49c8f Binary files /dev/null and b/docs/zh/20-third-party/dbeaver/dbeaver-browse-data-zh.webp differ diff --git a/docs/zh/20-third-party/dbeaver/dbeaver-config-tdengine-zh.webp b/docs/zh/20-third-party/dbeaver/dbeaver-config-tdengine-zh.webp new file mode 100644 index 0000000000000000000000000000000000000000..c721cf029b3408928bd5daca4ca0a3fe7ed865f5 Binary files /dev/null and b/docs/zh/20-third-party/dbeaver/dbeaver-config-tdengine-zh.webp differ diff --git a/docs/zh/20-third-party/dbeaver/dbeaver-connect-tdengine-cloud-test-zh.webp b/docs/zh/20-third-party/dbeaver/dbeaver-connect-tdengine-cloud-test-zh.webp new file mode 100644 index 0000000000000000000000000000000000000000..721fe50cfb8aace700e6528e0c5e5e6525429a86 Binary files /dev/null and b/docs/zh/20-third-party/dbeaver/dbeaver-connect-tdengine-cloud-test-zh.webp differ diff --git a/docs/zh/20-third-party/dbeaver/dbeaver-connect-tdengine-cloud-zh.webp b/docs/zh/20-third-party/dbeaver/dbeaver-connect-tdengine-cloud-zh.webp new file mode 100644 index 0000000000000000000000000000000000000000..b6f12a47082664b8cb29fa841add40443595d8d4 Binary files /dev/null and b/docs/zh/20-third-party/dbeaver/dbeaver-connect-tdengine-cloud-zh.webp differ diff --git a/docs/zh/20-third-party/dbeaver/dbeaver-connect-tdengine-test-zh.webp b/docs/zh/20-third-party/dbeaver/dbeaver-connect-tdengine-test-zh.webp new file mode 100644 index 0000000000000000000000000000000000000000..2553ea166ae5d1b08f823e0cf71881cd7da31862 Binary files /dev/null and b/docs/zh/20-third-party/dbeaver/dbeaver-connect-tdengine-test-zh.webp differ diff --git a/docs/zh/20-third-party/dbeaver/dbeaver-connect-tdengine-zh.webp b/docs/zh/20-third-party/dbeaver/dbeaver-connect-tdengine-zh.webp new file mode 100644 index 0000000000000000000000000000000000000000..eff8bd9d21163429005961f2b6ec9594271243d8 Binary files /dev/null and b/docs/zh/20-third-party/dbeaver/dbeaver-connect-tdengine-zh.webp differ diff --git a/docs/zh/20-third-party/dbeaver/dbeaver-sql-execution-cloud-zh.webp b/docs/zh/20-third-party/dbeaver/dbeaver-sql-execution-cloud-zh.webp new file mode 100644 index 0000000000000000000000000000000000000000..6efdf48e3e38faea7ed66546fba10c4f1a42460c Binary files /dev/null and b/docs/zh/20-third-party/dbeaver/dbeaver-sql-execution-cloud-zh.webp differ diff --git a/docs/zh/20-third-party/dbeaver/dbeaver-sql-execution-zh.webp b/docs/zh/20-third-party/dbeaver/dbeaver-sql-execution-zh.webp new file mode 100644 index 0000000000000000000000000000000000000000..fa95889d7235ce5101dd37f567681d89aa697041 Binary files /dev/null and b/docs/zh/20-third-party/dbeaver/dbeaver-sql-execution-zh.webp differ diff --git a/docs/zh/20-third-party/dbeaver/tdengine-cloud-jdbc-dsn-zh.webp b/docs/zh/20-third-party/dbeaver/tdengine-cloud-jdbc-dsn-zh.webp new file mode 100644 index 0000000000000000000000000000000000000000..bfa474f3ac5f323e59e35b27616951ce3165f942 Binary files /dev/null and b/docs/zh/20-third-party/dbeaver/tdengine-cloud-jdbc-dsn-zh.webp differ diff --git a/docs/zh/20-third-party/qstudio/qstudio-browse-data.webp b/docs/zh/20-third-party/qstudio/qstudio-browse-data.webp new file mode 100644 index 0000000000000000000000000000000000000000..0de3a569f74615db5a9220d810244382d280fb61 Binary files /dev/null and b/docs/zh/20-third-party/qstudio/qstudio-browse-data.webp differ diff --git a/docs/zh/20-third-party/qstudio/qstudio-chart.webp b/docs/zh/20-third-party/qstudio/qstudio-chart.webp new file mode 100644 index 0000000000000000000000000000000000000000..397d525cd292c5fecf7f90ea1de6c01d82a788e9 Binary files /dev/null and b/docs/zh/20-third-party/qstudio/qstudio-chart.webp differ diff --git a/docs/zh/20-third-party/qstudio/qstudio-connect-tdengine-test.webp b/docs/zh/20-third-party/qstudio/qstudio-connect-tdengine-test.webp new file mode 100644 index 0000000000000000000000000000000000000000..c488dfba5083c9be363387ad6a83c0bcdf590eb7 Binary files /dev/null and b/docs/zh/20-third-party/qstudio/qstudio-connect-tdengine-test.webp differ diff --git a/docs/zh/20-third-party/qstudio/qstudio-connect-tdengine.webp b/docs/zh/20-third-party/qstudio/qstudio-connect-tdengine.webp new file mode 100644 index 0000000000000000000000000000000000000000..c9805d3699c716b2e0a6efa4a17aeee124e63a01 Binary files /dev/null and b/docs/zh/20-third-party/qstudio/qstudio-connect-tdengine.webp differ diff --git a/docs/zh/20-third-party/qstudio/qstudio-jdbc-connector-download.webp b/docs/zh/20-third-party/qstudio/qstudio-jdbc-connector-download.webp new file mode 100644 index 0000000000000000000000000000000000000000..1ae7c73963aad68d2d0ebc86072d50e013100a50 Binary files /dev/null and b/docs/zh/20-third-party/qstudio/qstudio-jdbc-connector-download.webp differ diff --git a/docs/zh/20-third-party/qstudio/qstudio-sql-execution.webp b/docs/zh/20-third-party/qstudio/qstudio-sql-execution.webp new file mode 100644 index 0000000000000000000000000000000000000000..2fb0c0f135af962e637aef45611c469ea720c94d Binary files /dev/null and b/docs/zh/20-third-party/qstudio/qstudio-sql-execution.webp differ diff --git a/docs/zh/21-tdinternal/01-arch.md b/docs/zh/21-tdinternal/01-arch.md index 6150f2e7579fdd64c562b0c18f9963a6eed20d9d..e2480b66825e77b4c5be289f63010f8674cdc390 100644 --- a/docs/zh/21-tdinternal/01-arch.md +++ b/docs/zh/21-tdinternal/01-arch.md @@ -112,7 +112,7 @@ TDengine 3.0 采用 hash 一致性算法，确定每张数据表所在的 vnode ### 数据分区 -TDengine 除 vnode 分片之外，还对时序数据按照时间段进行分区。每个数据文件只包含一个时间段的时序数据，时间段的长度由 DB 的配置参数 days 决定。这种按时间段分区的方法还便于高效实现数据的保留策略，只要数据文件超过规定的天数（系统配置参数 keep），将被自动删除。而且不同的时间段可以存放于不同的路径和存储介质，以便于大数据的冷热管理，实现多级存储。 +TDengine 除 vnode 分片之外，还对时序数据按照时间段进行分区。每个数据文件只包含一个时间段的时序数据，时间段的长度由 DB 的配置参数 duration 决定。这种按时间段分区的方法还便于高效实现数据的保留策略，只要数据文件超过规定的天数（系统配置参数 keep），将被自动删除。而且不同的时间段可以存放于不同的路径和存储介质，以便于大数据的冷热管理，实现多级存储。 总的来说，**TDengine 是通过 vnode 以及时间两个维度，对大数据进行切分**，便于并行高效的管理，实现水平扩展。 @@ -165,9 +165,7 @@ Vnode 会保持一个数据版本号（version），对内存数据进行持久 ### 同步复制 -对于数据一致性要求更高的场景，异步数据复制提供的最终一致性无法满足要求。因此 TDengine 提供同步复制的机制供用户选择。在创建数据库时，除指定副本数 `replica` 之外，用户还需要指定新的参数 `strict`。如果 `strict` 等于 1，它表示每次 leader 转发给副本时，需要等待半数以上副本达成一致后，才能通知应用，数据在 follower 已经写入成功。如果在一定的时间内，得不到半数以上副本的确认，leader vnode 将返回错误给应用。 - -采用同步复制，系统的性能会有所下降，而且 latency 会增加。因为元数据要强一致，mnode 之间的数据同步缺省就是采用的同步复制。 +对于数据一致性要求更高的场景，异步数据复制提供的最终一致性无法满足要求。因此 TDengine 3.0 使用了同步复制的机制（参照 RAFT 协议的标准实现）。每次 leader vnode 转发给其他副本时，需要等待半数以上（包含自己）副本达成一致后，才能通知应用写入成功。如果在一定的时间内，得不到半数以上副本的确认，leader vnode 将返回错误给应用。 ## 缓存与持久化 diff --git a/docs/zh/25-application/03-immigrate.md b/docs/zh/25-application/03-immigrate.md index 75788c0cc7d6e0e84402ba77c4a1aa875d772d8b..389a2b2c5a31f7357fafe9bf4fd178d811acf464 100644 --- a/docs/zh/25-application/03-immigrate.md +++ b/docs/zh/25-application/03-immigrate.md @@ -371,7 +371,7 @@ Select min(val) from table_name 等效函数：sum ```sql -Select max(value) from (select first(val) value from table_name interval(10s) fill(linear)) interval(10s) +Select sum(value) from (select first(val) value from table_name interval(10s) fill(linear)) interval(10s) ``` 备注：该函数无插值需求，因此可用直接计算。 diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index 9e82ea0af00b66a4e1a1b54594a05d8ee3f4d626..15397049dd79d5888242dd4ed17d8395f1d1096e 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -247,4 +247,17 @@ launchctl limit maxfiles 该提示是创建 db 的 vnode 数量不够了，需要的 vnode 不能超过了 dnode 中 vnode 的上限。因为系统默认是一个 dnode 中有 CPU 核数两倍的 vnode，也可以通过配置文件中的参数 supportVnodes 控制。 正常调大 taos.cfg 中 supportVnodes 参数即可。 +### 21 在服务器上的使用 taos-CLI 能查到指定时间段的数据，但在客户端机器上查不到？ +这种情况是因为客户端与服务器上设置的时区不一致导致的，调整客户端与服务器的时区一致即可解决。 +### 22 表名确认是存在的，但在写入或查询时返回表名不存在，什么原因？ +TDengine 中的所有名称，包括数据库名、表名等都是区分大小写的，如果这些名称在程序或 taos-CLI 中没有使用反引号（`）括起来使用，即使你输入的是大写的，引擎也会转化成小写来使用，如果名称前后加上了反引号，引擎就不会再转化成小写，会保持原样来使用。 + +### 23 在 taos-CLI 中查询，字段内容不能完全显示出来怎么办？ +可以使用 \G 参数来竖式显示，如 show databases\G; （为了输入方便，在"\"后加 TAB 键，会自动补全后面的内容） + +### 24 使用 taosBenchmark 测试工具写入数据查询很快，为什么我写入的数据查询非常慢？ +TDengine 在写入数据时如果有很严重的乱序写入问题，会严重影响查询性能，所以需要在写入前解决乱序的问题。如果业务是从 kafka 消费写入，请合理设计消费者，尽可能的一个子表数据由一个消费者去消费并写入，避免由设计产生的乱序。 + +### 25 我想统计下前后两条写入记录之间的时间差值是多少？ +使用 DIFF 函数，可以查看时间列或数值列前后两条记录的差值，非常方便，详细说明见 SQL手册->函数->DIFF diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index bea0adfa824b566b648cc19d5779eb68a1f6e37c..afdf2a76d3106b0ba0dd4aaf77c96dcf387ea67e 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,34 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.1.0.0 + + + +## 3.0.7.1 + + + +## 3.0.7.0 + + + +## 3.0.6.0 + + + +## 3.0.5.1 + + + +## 3.0.5.0 + + + +## 3.0.4.2 + + + ## 3.0.4.1 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index e13ec68c2e6da959b558432c1399e8d1466c7343..ba58ed96005cba56cf4b20dc729b10c2c1844769 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -10,9 +10,13 @@ taosTools 各版本安装包下载链接如下： import Release from "/components/ReleaseV3"; -## 2.5.0 +## 2.5.2 - + + +## 2.5.1 + + ## 2.5.0 diff --git a/examples/C#/taosdemo/README.md b/examples/C#/taosdemo/README.md index 3cba3529bf513e2bf3d4ab0c169e7f3d03b2e6a8..970d5332acabc5e19525fcb1a941dc8145dc8591 100644 --- a/examples/C#/taosdemo/README.md +++ b/examples/C#/taosdemo/README.md @@ -36,7 +36,11 @@ dotnet build -c Release ## Usage ``` -Usage: mono taosdemo.exe [OPTION...] +Usage with mono: +$ mono taosdemo.exe [OPTION...] + +Usage with dotnet: +Usage: .\bin\Release\net5.0\taosdemo.exe [OPTION...] --help Show usage. diff --git a/examples/C#/taosdemo/taosdemo.cs b/examples/C#/taosdemo/taosdemo.cs index e092c48f15314f5cad0a9509190d7b9970a7073a..a48439d192bb5dfcd0083f27abb258ece0f4f28d 100644 --- a/examples/C#/taosdemo/taosdemo.cs +++ b/examples/C#/taosdemo/taosdemo.cs @@ -72,7 +72,7 @@ namespace TDengineDriver { if ("--help" == argv[i]) { - Console.WriteLine("Usage: mono taosdemo.exe [OPTION...]"); + Console.WriteLine("Usage: taosdemo.exe [OPTION...]"); Console.WriteLine(""); HelpPrint("--help", "Show usage."); Console.WriteLine(""); @@ -305,7 +305,7 @@ namespace TDengineDriver this.conn = TDengine.Connect(this.host, this.user, this.password, db, this.port); if (this.conn == IntPtr.Zero) { - Console.WriteLine("Connect to TDengine failed"); + Console.WriteLine("Connect to TDengine failed. Reason: {0}\n", TDengine.Error(0)); CleanAndExitProgram(1); } else diff --git a/examples/JDBC/JDBCDemo/readme.md b/examples/JDBC/JDBCDemo/readme.md index da638a0bcc485cb3d73f75b59348ec260cc871d2..21f9153935f256d2887005ac23c9a3a36f6cd263 100644 --- a/examples/JDBC/JDBCDemo/readme.md +++ b/examples/JDBC/JDBCDemo/readme.md @@ -20,18 +20,12 @@ mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JdbcDemo" -De ``` ## Compile the Demo Code and Run It -To compile taos-jdbcdriver, go to the source directory ``TDengine/src/connector/jdbc`` and execute ``` mvn clean package -Dmaven.test.skip=true ``` -To compile the demo project, go to the source directory ``TDengine/tests/examples/JDBC/JDBCDemo`` and execute +To run JDBCDemo.jar, execute ``` -mvn clean package assembly:single -``` - -To run JDBCDemo.jar, go to ``TDengine/tests/examples/JDBC/JDBCDemo`` and execute -``` -java -Djava.ext.dirs=../../../../src/connector/jdbc/target:$JAVA_HOME/jre/lib/ext -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host [HOSTNAME] +java -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host [HOSTNAME] ``` diff --git a/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java b/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java index 5bc23403087578c0791b0a5e6fca74a47aad8184..aeb75cc3a2f4fc97d7ff7af6e451803e7fcacb8f 100644 --- a/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java +++ b/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java @@ -51,27 +51,27 @@ public class JdbcDemo { private void createDatabase() { String sql = "create database if not exists " + dbName; - exuete(sql); + execute(sql); } private void useDatabase() { String sql = "use " + dbName; - exuete(sql); + execute(sql); } private void dropTable() { final String sql = "drop table if exists " + dbName + "." + tbName + ""; - exuete(sql); + execute(sql); } private void createTable() { final String sql = "create table if not exists " + dbName + "." + tbName + " (ts timestamp, temperature float, humidity int)"; - exuete(sql); + execute(sql); } private void insert() { final String sql = "insert into " + dbName + "." + tbName + " (ts, temperature, humidity) values(now, 20.5, 34)"; - exuete(sql); + execute(sql); } private void select() { @@ -120,7 +120,7 @@ public class JdbcDemo { System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); } - private void exuete(String sql) { + private void execute(String sql) { long start = System.currentTimeMillis(); try (Statement statement = connection.createStatement()) { boolean execute = statement.execute(sql); diff --git a/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java b/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java index d89476b8ca718dab24202e2320e842366533a763..69ef91d380a6f4f35c8fd00e9d6641769a1cb5b5 100644 --- a/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java +++ b/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java @@ -16,8 +16,6 @@ public class JdbcRestfulDemo { Properties properties = new Properties(); properties.setProperty("charset", "UTF-8"); - properties.setProperty("locale", "en_US.UTF-8"); - properties.setProperty("timezone", "UTC-8"); Connection conn = DriverManager.getConnection(url, properties); Statement stmt = conn.createStatement(); diff --git a/examples/JDBC/consumer-demo/pom.xml b/examples/JDBC/consumer-demo/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..6199efb76ea461909e64aa221ed05899001f2312 --- /dev/null +++ b/examples/JDBC/consumer-demo/pom.xml @@ -0,0 +1,70 @@ + + + 4.0.0 + + com.taosdata + consumer + 1.0-SNAPSHOT + + + 8 + 8 + + + + + com.taosdata.jdbc + taos-jdbcdriver + 3.2.1 + + + com.google.guava + guava + 32.0.0-jre + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.3.0 + + + ConsumerDemo + + ConsumerDemo + + + com.taosdata.ConsumerDemo + + + + jar-with-dependencies + + + package + + single + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + UTF-8 + + + + + + + \ No newline at end of file diff --git a/examples/JDBC/consumer-demo/readme.md b/examples/JDBC/consumer-demo/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..c211b017a74cd2a84bbab6953bb31500d73b3ba2 --- /dev/null +++ b/examples/JDBC/consumer-demo/readme.md @@ -0,0 +1,52 @@ +# How to Run the Consumer Demo Code On Linux OS +TDengine's Consumer demo project is organized in a Maven way so that users can easily compile, package and run the project. If you don't have Maven on your server, you may install it using +``` +sudo apt-get install maven +``` + +## Install TDengine Client and TaosAdapter +Make sure you have already installed a tdengine client on your current develop environment. +Download the tdengine package on our website: ``https://www.taosdata.com/cn/all-downloads/`` and install the client. + +## Run Consumer Demo using mvn plugin +run command: +``` +mvn clean compile exec:java -Dexec.mainClass="com.taosdata.ConsumerDemo" +``` + +## Custom configuration +```shell +# the host of TDengine server +export TAOS_HOST="127.0.0.1" + +# the port of TDengine server +export TAOS_PORT="6041" + +# the consumer type, can be "ws" or "jni" +export TAOS_TYPE="ws" + +# the number of consumers +export TAOS_JDBC_CONSUMER_NUM="1" + +# the number of processors to consume +export TAOS_JDBC_PROCESSOR_NUM="2" + +# the number of records to be consumed per processor per second +export TAOS_JDBC_RATE_PER_PROCESSOR="1000" + +# poll wait time in ms +export TAOS_JDBC_POLL_SLEEP="100" +``` + +## Run Consumer Demo using jar + +To compile the demo project, go to the source directory ``TDengine/tests/examples/JDBC/consumer-demo`` and execute +``` +mvn clean package assembly:single +``` + +To run ConsumerDemo.jar, go to ``TDengine/tests/examples/JDBC/consumer-demo`` and execute +``` +java -jar target/ConsumerDemo-jar-with-dependencies.jar +``` + diff --git a/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Bean.java b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Bean.java new file mode 100644 index 0000000000000000000000000000000000000000..2f2467b3713b2d6bcedd708f3ea4b20d3db53c23 --- /dev/null +++ b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Bean.java @@ -0,0 +1,43 @@ +package com.taosdata; + +import java.sql.Timestamp; + +public class Bean { + private Timestamp ts; + private Integer c1; + private String c2; + + public Timestamp getTs() { + return ts; + } + + public void setTs(Timestamp ts) { + this.ts = ts; + } + + public Integer getC1() { + return c1; + } + + public void setC1(Integer c1) { + this.c1 = c1; + } + + public String getC2() { + return c2; + } + + public void setC2(String c2) { + this.c2 = c2; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Bean {"); + sb.append("ts=").append(ts); + sb.append(", c1=").append(c1); + sb.append(", c2='").append(c2).append('\''); + sb.append('}'); + return sb.toString(); + } +} diff --git a/examples/JDBC/consumer-demo/src/main/java/com/taosdata/BeanDeserializer.java b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/BeanDeserializer.java new file mode 100644 index 0000000000000000000000000000000000000000..478af9e70da51657f9d2f7a49e7b919bf5399d55 --- /dev/null +++ b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/BeanDeserializer.java @@ -0,0 +1,6 @@ +package com.taosdata; + +import com.taosdata.jdbc.tmq.ReferenceDeserializer; + +public class BeanDeserializer extends ReferenceDeserializer { +} diff --git a/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Config.java b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Config.java new file mode 100644 index 0000000000000000000000000000000000000000..08579926e3cbc1dcaa4c8c01027340d8c2635cb2 --- /dev/null +++ b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Config.java @@ -0,0 +1,78 @@ +package com.taosdata; + +public class Config { + public static final String TOPIC = "test_consumer"; + public static final String TAOS_HOST = "127.0.0.1"; + public static final String TAOS_PORT = "6041"; + public static final String TAOS_TYPE = "ws"; + public static final int TAOS_JDBC_CONSUMER_NUM = 1; + public static final int TAOS_JDBC_PROCESSOR_NUM = 2; + public static final int TAOS_JDBC_RATE_PER_PROCESSOR = 1000; + public static final int TAOS_JDBC_POLL_SLEEP = 100; + + private final int consumerNum; + private final int processCapacity; + private final int rate; + private final int pollSleep; + private final String type; + private final String host; + private final String port; + + public Config(String type, String host, String port, int consumerNum, int processCapacity, int rate, int pollSleep) { + this.type = type; + this.consumerNum = consumerNum; + this.processCapacity = processCapacity; + this.rate = rate; + this.pollSleep = pollSleep; + this.host = host; + this.port = port; + } + + public int getConsumerNum() { + return consumerNum; + } + + public int getProcessCapacity() { + return processCapacity; + } + + public int getRate() { + return rate; + } + + public int getPollSleep() { + return pollSleep; + } + + public String getHost() { + return host; + } + + public String getPort() { + return port; + } + + public String getType() { + return type; + } + + public static Config getFromENV() { + String host = System.getenv("TAOS_HOST") != null ? System.getenv("TAOS_HOST") : TAOS_HOST; + String port = System.getenv("TAOS_PORT") != null ? System.getenv("TAOS_PORT") : TAOS_PORT; + String type = System.getenv("TAOS_TYPE") != null ? System.getenv("TAOS_TYPE") : TAOS_TYPE; + + String c = System.getenv("TAOS_JDBC_CONSUMER_NUM"); + int num = c != null ? Integer.parseInt(c) : TAOS_JDBC_CONSUMER_NUM; + + String p = System.getenv("TAOS_JDBC_PROCESSOR_NUM"); + int capacity = p != null ? Integer.parseInt(p) : TAOS_JDBC_PROCESSOR_NUM; + + String r = System.getenv("TAOS_JDBC_RATE_PER_PROCESSOR"); + int rate = r != null ? Integer.parseInt(r) : TAOS_JDBC_RATE_PER_PROCESSOR; + + String s = System.getenv("TAOS_JDBC_POLL_SLEEP"); + int sleep = s != null ? Integer.parseInt(s) : TAOS_JDBC_POLL_SLEEP; + + return new Config(type, host, port, num, capacity, rate, sleep); + } +} diff --git a/examples/JDBC/consumer-demo/src/main/java/com/taosdata/ConsumerDemo.java b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/ConsumerDemo.java new file mode 100644 index 0000000000000000000000000000000000000000..7c7719c63986cc309363c13fecb5fafe0243cdba --- /dev/null +++ b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/ConsumerDemo.java @@ -0,0 +1,65 @@ +package com.taosdata; + +import com.taosdata.jdbc.tmq.TMQConstants; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.taosdata.Config.*; + +public class ConsumerDemo { + public static void main(String[] args) throws SQLException { + // Config + Config config = Config.getFromENV(); + // Generated data + mockData(); + + Properties prop = new Properties(); + prop.setProperty(TMQConstants.CONNECT_TYPE, config.getType()); + prop.setProperty(TMQConstants.BOOTSTRAP_SERVERS, config.getHost() + ":" + config.getPort()); + prop.setProperty(TMQConstants.CONNECT_USER, "root"); + prop.setProperty(TMQConstants.CONNECT_PASS, "taosdata"); + prop.setProperty(TMQConstants.MSG_WITH_TABLE_NAME, "true"); + prop.setProperty(TMQConstants.ENABLE_AUTO_COMMIT, "true"); + prop.setProperty(TMQConstants.GROUP_ID, "gId"); + prop.setProperty(TMQConstants.VALUE_DESERIALIZER, "com.taosdata.BeanDeserializer"); + for (int i = 0; i < config.getConsumerNum() - 1; i++) { + new Thread(new Worker(prop, config)).start(); + } + new Worker(prop, config).run(); + } + + public static void mockData() throws SQLException { + String dbName = "test_consumer"; + String tableName = "st"; + String url = "jdbc:TAOS-RS://" + TAOS_HOST + ":" + TAOS_PORT + "/?user=root&password=taosdata&batchfetch=true"; + Connection connection = DriverManager.getConnection(url); + Statement statement = connection.createStatement(); + statement.executeUpdate("create database if not exists " + dbName + " WAL_RETENTION_PERIOD 3650"); + statement.executeUpdate("use " + dbName); + statement.executeUpdate("create table if not exists " + tableName + " (ts timestamp, c1 int, c2 nchar(100)) "); + statement.executeUpdate("create topic if not exists " + TOPIC + " as select ts, c1, c2 from " + tableName); + + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r); + t.setName("mock-data-thread-" + t.getId()); + return t; + }); + AtomicInteger atomic = new AtomicInteger(); + scheduledExecutorService.scheduleWithFixedDelay(() -> { + int i = atomic.getAndIncrement(); + try { + statement.executeUpdate("insert into " + tableName + " values(now, " + i + ",'" + i + "')"); + } catch (SQLException e) { + // ignore + } + }, 0, 10, TimeUnit.MILLISECONDS); + } +} diff --git a/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java new file mode 100644 index 0000000000000000000000000000000000000000..f6e21cd7294333aeb96a6e99f0cafe326073f5fa --- /dev/null +++ b/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java @@ -0,0 +1,60 @@ +package com.taosdata; + +import com.google.common.util.concurrent.RateLimiter; +import com.taosdata.jdbc.tmq.ConsumerRecord; +import com.taosdata.jdbc.tmq.ConsumerRecords; +import com.taosdata.jdbc.tmq.TaosConsumer; + +import java.sql.SQLException; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.Collections; +import java.util.Properties; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.Semaphore; + +public class Worker implements Runnable { + + int sleepTime; + int rate; + + ForkJoinPool pool = new ForkJoinPool(); + Semaphore semaphore; + + TaosConsumer consumer; + + public Worker(Properties prop, Config config) throws SQLException { + consumer = new TaosConsumer<>(prop); + consumer.subscribe(Collections.singletonList(Config.TOPIC)); + semaphore = new Semaphore(config.getProcessCapacity()); + sleepTime = config.getPollSleep(); + rate = config.getRate(); + } + + @Override + public void run() { + while (!Thread.interrupted()) { + try { + // 控制请求频率 + if (semaphore.tryAcquire()) { + ConsumerRecords records = consumer.poll(Duration.ofMillis(sleepTime)); + pool.submit(() -> { + RateLimiter limiter = RateLimiter.create(rate); + try { + for (ConsumerRecord record : records) { + // 流量控制 + limiter.acquire(); + // 业务处理数据 + System.out.println("[" + LocalDateTime.now() + "] Thread id:" + Thread.currentThread().getId() + " -> " + record.value()); + } + } finally { + semaphore.release(); + } + }); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + } +} diff --git a/examples/JDBC/mybatisplus-demo/pom.xml b/examples/JDBC/mybatisplus-demo/pom.xml index 5555145958de67fdf03eb744426afcfc13b6fcb3..f792946c9653ebffa9c78f7380764a71ab76e364 100644 --- a/examples/JDBC/mybatisplus-demo/pom.xml +++ b/examples/JDBC/mybatisplus-demo/pom.xml @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.0.0 + 3.2.4 diff --git a/examples/JDBC/springbootdemo/src/main/resources/application.properties b/examples/JDBC/springbootdemo/src/main/resources/application.properties index bf21047395ed534e4c7d9db919bb371fab45ec16..c523952fb641f90358210ce40f88d4c23273f2c7 100644 --- a/examples/JDBC/springbootdemo/src/main/resources/application.properties +++ b/examples/JDBC/springbootdemo/src/main/resources/application.properties @@ -5,7 +5,7 @@ #spring.datasource.password=taosdata # datasource config - JDBC-RESTful spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver -spring.datasource.url=jdbc:TAOS-RS://localhost:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +spring.datasource.url=jdbc:TAOS-RS://localhost:6041/test spring.datasource.username=root spring.datasource.password=taosdata spring.datasource.druid.initial-size=5 diff --git a/examples/c/CMakeLists.txt b/examples/c/CMakeLists.txt index e14c4e60d9d49a660643c89f4642af7b12c8d18a..07fc2fd71b54255535e8002aa1075dfbb6b38b89 100644 --- a/examples/c/CMakeLists.txt +++ b/examples/c/CMakeLists.txt @@ -42,27 +42,27 @@ IF (TD_LINUX) ) target_link_libraries(tmq - taos_static + taos ) target_link_libraries(stream_demo - taos_static + taos ) target_link_libraries(schemaless - taos_static + taos ) target_link_libraries(prepare - taos_static + taos ) target_link_libraries(demo - taos_static + taos ) target_link_libraries(asyncdemo - taos_static + taos ) SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 94545dfaad943a3ec5e35ae6859213835d8c4540..e1133c109e72873ddcf4b89a818d878d56a628f1 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -73,7 +73,7 @@ static int32_t init_env() { taos_free_result(pRes); // create database - pRes = taos_query(pConn, "create database tmqdb precision 'ns'"); + pRes = taos_query(pConn, "create database tmqdb precision 'ns' WAL_RETENTION_PERIOD 3600"); if (taos_errno(pRes) != 0) { printf("error in create tmqdb, reason:%s\n", taos_errstr(pRes)); goto END; @@ -289,7 +289,7 @@ void consume_repeatly(tmq_t* tmq) { } } - free(pAssign); + tmq_free_assignment(pAssign); // let's do it again basic_consume_loop(tmq); diff --git a/examples/lua/lua51/lua_connector51.c b/examples/lua/lua51/lua_connector51.c index c5ef9126a644cae83a730f5d39d501713aff9db5..a8f051a7904967544f25d6718ec1e787fa8017f8 100644 --- a/examples/lua/lua51/lua_connector51.c +++ b/examples/lua/lua51/lua_connector51.c @@ -162,6 +162,7 @@ static int l_query(lua_State *L){ case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: //printf("type:%d, max len:%d, current len:%d\n",fields[i].type, fields[i].bytes, length[i]); lua_pushlstring(L,(char *)row[i], length[i]); break; diff --git a/examples/lua/lua_connector.c b/examples/lua/lua_connector.c index 5d8226f2623f6451ef77b71dac1a6583ba6c460b..1422d1eb032ba3eea8e19797d911510289439e3f 100644 --- a/examples/lua/lua_connector.c +++ b/examples/lua/lua_connector.c @@ -161,6 +161,7 @@ static int l_query(lua_State *L){ case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: //printf("type:%d, max len:%d, current len:%d\n",fields[i].type, fields[i].bytes, length[i]); lua_pushlstring(L,(char *)row[i], length[i]); break; diff --git a/include/client/taos.h b/include/client/taos.h index 8811c4ab64e3ae65085c26ccd791705a98541423..3cc2d907ab5ca18d16a9553d336672d67e4f974c 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -51,7 +51,8 @@ typedef void TAOS_SUB; #define TSDB_DATA_TYPE_BLOB 18 // binary #define TSDB_DATA_TYPE_MEDIUMBLOB 19 #define TSDB_DATA_TYPE_BINARY TSDB_DATA_TYPE_VARCHAR // string -#define TSDB_DATA_TYPE_MAX 20 +#define TSDB_DATA_TYPE_GEOMETRY 20 // geometry +#define TSDB_DATA_TYPE_MAX 21 typedef enum { TSDB_OPTION_LOCALE, @@ -286,10 +287,20 @@ DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout); DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq); DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param); +DLL_EXPORT int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset); +DLL_EXPORT void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param); DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment, int32_t *numOfAssignment); +DLL_EXPORT void tmq_free_assignment(tmq_topic_assignment* pAssignment); DLL_EXPORT int32_t tmq_offset_seek(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset); +DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); +DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); +DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); +DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res); +DLL_EXPORT int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId); +DLL_EXPORT int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId); + /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ enum tmq_conf_res_t { @@ -307,11 +318,6 @@ DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_comm /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */ -DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); -DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); -DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); -DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res); - /* ------------------------------ TAOSX -----------------------------------*/ // note: following apis are unstable enum tmq_res_t { diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 0544247da239693e5c7ca78f95fece01b3291a0c..bdfb1d32b4642e265b468e4f4e3a7bd476b07ff9 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -37,6 +37,13 @@ extern "C" { ) // clang-format on +typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2); + +typedef struct STableKeyInfo { + uint64_t uid; + uint64_t groupId; +} STableKeyInfo; + typedef struct SWinKey { uint64_t groupId; TSKEY ts; @@ -47,6 +54,11 @@ typedef struct SSessionKey { uint64_t groupId; } SSessionKey; +typedef struct SVersionRange { + uint64_t minVer; + uint64_t maxVer; +} SVersionRange; + static inline int winKeyCmprImpl(const void* pKey1, const void* pKey2) { SWinKey* pWin1 = (SWinKey*)pKey1; SWinKey* pWin2 = (SWinKey*)pKey2; @@ -82,7 +94,7 @@ typedef struct STuplePos { int32_t pageId; int32_t offset; }; - STupleKey streamTupleKey; + SWinKey streamTupleKey; }; } STuplePos; @@ -124,10 +136,10 @@ static inline int STupleKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, enum { TMQ_MSG_TYPE__DUMMY = 0, - TMQ_MSG_TYPE__POLL_RSP, + TMQ_MSG_TYPE__POLL_DATA_RSP, TMQ_MSG_TYPE__POLL_META_RSP, TMQ_MSG_TYPE__EP_RSP, - TMQ_MSG_TYPE__TAOSX_RSP, + TMQ_MSG_TYPE__POLL_DATA_META_RSP, TMQ_MSG_TYPE__WALINFO_RSP, TMQ_MSG_TYPE__END_RSP, }; @@ -191,6 +203,7 @@ typedef struct SDataBlockInfo { SBlockID id; int16_t hasVarCol; int16_t dataLoad; // denote if the data is loaded or not + uint8_t scanFlag; // TODO: optimize and remove following int64_t version; // used for stream, and need serialization @@ -208,11 +221,6 @@ typedef struct SSDataBlock { SDataBlockInfo info; } SSDataBlock; -enum { - FETCH_TYPE__DATA = 0, - FETCH_TYPE__NONE, -}; - typedef struct SVarColAttr { int32_t* offset; // start position for each entry in the list uint32_t length; // used buffer size that contain the valid data @@ -229,6 +237,7 @@ typedef struct SColumnInfoData { }; SColumnInfo info; // column info bool hasNull; // if current column data has null value. + bool reassigned; // if current column data is reassigned. } SColumnInfoData; typedef struct SQueryTableDataCond { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 99fffa2cf1d6f28697fea0a96562707aeb3e3a71..c0412d2617500196d2d4d31f9ff49f7bede9a08d 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -177,7 +177,7 @@ static FORCE_INLINE void colDataSetDouble(SColumnInfoData* pColumnInfoData, uint int32_t getJsonValueLen(const char* data); int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); -int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); +int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData); int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, bool trimValue); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, int32_t numOfRow2); @@ -186,6 +186,8 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); + +int32_t colDataGetRowLength(const SColumnInfoData* pColumnInfoData, int32_t rowIdx); void colDataTrim(SColumnInfoData* pColumnInfoData); size_t blockDataGetNumOfCols(const SSDataBlock* pBlock); @@ -206,7 +208,6 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock); size_t blockDataGetSerialMetaSize(uint32_t numOfCols); int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo); -int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); @@ -215,7 +216,7 @@ void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); void blockDataCleanup(SSDataBlock* pDataBlock); void blockDataEmpty(SSDataBlock* pDataBlock); -size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); +size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize, int32_t extraSize); int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n); int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n); @@ -235,11 +236,10 @@ int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColIn SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); +int32_t blockGetEncodeSize(const SSDataBlock* pBlock); int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols); const char* blockDecode(SSDataBlock* pBlock, const char* pData); -void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag); -void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); @@ -247,10 +247,9 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData tb_uid_t suid); char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); +int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf); -static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { - return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock); -} +void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList); #ifdef __cplusplus } diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 8be5cb4d413b9392702c628d72f27cb0662687a3..e04bdd1b07c00be375a86056976a1ee8aad7d558 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -145,7 +145,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull); // for stmt bind -int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind); +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen); void tColDataSortMerge(SArray *colDataArr); // for raw block diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 8509d39793fee311e90f8036669c45be38705995..0d3852cbab009968133320f1c00899bbf4233b21 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -29,12 +29,12 @@ extern "C" { #define SLOW_LOG_TYPE_OTHERS 0x4 #define SLOW_LOG_TYPE_ALL 0xFFFFFFFF - // cluster extern char tsFirst[]; extern char tsSecond[]; extern char tsLocalFqdn[]; extern char tsLocalEp[]; +extern char tsVersionName[]; extern uint16_t tsServerPort; extern int32_t tsVersion; extern int32_t tsStatusInterval; @@ -49,6 +49,7 @@ extern int32_t tsMaxNumOfDistinctResults; extern int32_t tsCompatibleModel; extern bool tsPrintAuth; extern int64_t tsTickPerMin[3]; +extern int64_t tsTickPerHour[3]; extern int32_t tsCountAlwaysReturnValue; extern float tsSelectivityRatio; extern int32_t tsTagFilterResCacheSize; @@ -57,6 +58,7 @@ extern int32_t tsTagFilterResCacheSize; extern int32_t tsNumOfRpcThreads; extern int32_t tsNumOfRpcSessions; extern int32_t tsTimeToGetAvailableConn; +extern int32_t tsKeepAliveIdle; extern int32_t tsNumOfCommitThreads; extern int32_t tsNumOfTaskQueueThreads; extern int32_t tsNumOfMnodeQueryThreads; @@ -83,6 +85,14 @@ extern int64_t tsVndCommitMaxIntervalMs; // mnode extern int64_t tsMndSdbWriteDelta; extern int64_t tsMndLogRetention; +extern int8_t tsGrant; +extern int32_t tsMndGrantMode; +extern bool tsMndSkipGrant; + +// dnode +extern int64_t tsDndStart; +extern int64_t tsDndStartOsUptime; +extern int64_t tsDndUpTime; // monitor extern bool tsEnableMonitor; @@ -119,18 +129,20 @@ extern bool tsQueryUseNodeAllocator; extern bool tsKeepColumnName; extern bool tsEnableQueryHb; extern bool tsEnableScience; +extern bool tsTtlChangeOnWrite; extern int32_t tsRedirectPeriod; extern int32_t tsRedirectFactor; extern int32_t tsRedirectMaxPeriod; extern int32_t tsMaxRetryWaitTime; extern bool tsUseAdapter; +extern int32_t tsMetaCacheMaxSize; extern int32_t tsSlowLogThreshold; extern int32_t tsSlowLogScope; // client extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; -extern int32_t tsMaxMemUsedByInsert; +extern int32_t tsMaxInsertBatchRows; // build info extern char version[]; @@ -158,9 +170,13 @@ extern char tsUdfdLdLibPath[]; // schemaless extern char tsSmlChildTableName[]; extern char tsSmlTagName[]; +extern bool tsSmlDot2Underline; +extern char tsSmlTsDefaultName[]; // extern bool tsSmlDataFormat; // extern int32_t tsSmlBatchSize; +extern int32_t tmqMaxTopicNum; + // wal extern int64_t tsWalFsyncDataSizeLimit; @@ -179,6 +195,10 @@ extern int32_t tsRpcRetryInterval; extern bool tsDisableStream; extern int64_t tsStreamBufferSize; extern int64_t tsCheckpointInterval; +extern bool tsFilterScalarMode; +extern int32_t tsKeepTimeOffset; +extern int32_t tsMaxStreamBackendCache; +extern int32_t tsPQSortMemThreshold; // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) @@ -193,8 +213,9 @@ struct SConfig *taosGetCfg(); void taosSetAllDebugFlag(int32_t flag, bool rewrite); void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, bool rewrite); -int32_t taosSetCfg(SConfig *pCfg, char *name); +int32_t taosApplyLocalCfg(SConfig *pCfg, char *name); void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); +int8_t taosGranted(); #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 2daa37d4530767628b40eca71a1c3f9cbe83a42d..01923d2b30af0b4f4de5eddac607503261f32fc1 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -106,7 +106,6 @@ enum { HEARTBEAT_KEY_DBINFO, HEARTBEAT_KEY_STBINFO, HEARTBEAT_KEY_TMQ, - HEARTBEAT_KEY_USER_PASSINFO, }; typedef enum _mgmt_table { @@ -636,6 +635,7 @@ typedef struct { SEpSet epSet; int32_t svrTimestamp; int32_t passVer; + int32_t authVer; char sVer[TSDB_VERSION_LEN]; char sDetailVer[128]; } SConnectRsp; @@ -703,6 +703,7 @@ int32_t tDeserializeSGetUserAuthReq(void* buf, int32_t bufLen, SGetUserAuthReq* typedef struct { char user[TSDB_USER_LEN]; int32_t version; + int32_t passVer; int8_t superAuth; int8_t sysInfo; int8_t enable; @@ -719,14 +720,6 @@ int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pR int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); void tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp); -typedef struct SUserPassVersion { - char user[TSDB_USER_LEN]; - int32_t version; -} SUserPassVersion; - -typedef SGetUserAuthReq SGetUserPassReq; -typedef SUserPassVersion SGetUserPassRsp; - /* * for client side struct, only column id, type, bytes are necessary * But for data in vnode side, we need all the following information. @@ -945,13 +938,16 @@ int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq); int32_t tDeserializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq); typedef struct { - int32_t timestamp; + int32_t timestampSec; } SVDropTtlTableReq; int32_t tSerializeSVDropTtlTableReq(void* buf, int32_t bufLen, SVDropTtlTableReq* pReq); int32_t tDeserializeSVDropTtlTableReq(void* buf, int32_t bufLen, SVDropTtlTableReq* pReq); typedef struct { + char db[TSDB_DB_FNAME_LEN]; + int64_t dbId; + int32_t cfgVersion; int32_t numOfVgroups; int32_t numOfStables; int32_t buffer; @@ -984,8 +980,13 @@ typedef struct { int16_t sstTrigger; } SDbCfgRsp; +typedef SDbCfgRsp SDbCfgInfo; + +int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp); int32_t tSerializeSDbCfgRsp(void* buf, int32_t bufLen, const SDbCfgRsp* pRsp); int32_t tDeserializeSDbCfgRsp(void* buf, int32_t bufLen, SDbCfgRsp* pRsp); +int32_t tDeserializeSDbCfgRspImpl(SDecoder* decoder, SDbCfgRsp *pRsp); +void tFreeSDbCfgRsp(SDbCfgRsp *pRsp); typedef struct { int32_t rowNum; @@ -1042,12 +1043,17 @@ int32_t tDeserializeSDnodeListRsp(void* buf, int32_t bufLen, SDnodeListRsp* pRsp void tFreeSDnodeListRsp(SDnodeListRsp* pRsp); typedef struct { - SArray* pArray; // Array of SUseDbRsp -} SUseDbBatchRsp; + SUseDbRsp *useDbRsp; + SDbCfgRsp *cfgRsp; +} SDbHbRsp; + +typedef struct { + SArray* pArray; // Array of SDbHbRsp +} SDbHbBatchRsp; -int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); -int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); -void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); +int32_t tSerializeSDbHbBatchRsp(void* buf, int32_t bufLen, SDbHbBatchRsp* pRsp); +int32_t tDeserializeSDbHbBatchRsp(void* buf, int32_t bufLen, SDbHbBatchRsp* pRsp); +void tFreeSDbHbBatchRsp(SDbHbBatchRsp* pRsp); typedef struct { SArray* pArray; // Array of SGetUserAuthRsp @@ -1057,14 +1063,6 @@ int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp); -typedef struct { - SArray* pArray; // Array of SGetUserPassRsp -} SUserPassBatchRsp; - -int32_t tSerializeSUserPassBatchRsp(void* buf, int32_t bufLen, SUserPassBatchRsp* pRsp); -int32_t tDeserializeSUserPassBatchRsp(void* buf, int32_t bufLen, SUserPassBatchRsp* pRsp); -void tFreeSUserPassBatchRsp(SUserPassBatchRsp* pRsp); - typedef struct { char db[TSDB_DB_FNAME_LEN]; STimeWindow timeRange; @@ -1146,6 +1144,7 @@ typedef struct { char timezone[TD_TIMEZONE_LEN]; // tsTimezone char locale[TD_LOCALE_LEN]; // tsLocale char charset[TD_LOCALE_LEN]; // tsCharset + int8_t ttlChangeOnWrite; } SClusterCfg; typedef struct { @@ -1182,6 +1181,8 @@ typedef struct { typedef struct { int8_t syncState; int8_t syncRestore; + int64_t syncTerm; + int64_t roleTimeMs; } SMnodeLoad; typedef struct { @@ -1497,6 +1498,7 @@ int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesR typedef struct { char name[TSDB_CONFIG_OPTION_LEN + 1]; char value[TSDB_CONFIG_VALUE_LEN + 1]; + char scope[TSDB_CONFIG_SCOPE_LEN + 1]; } SVariablesInfo; typedef struct { @@ -1629,6 +1631,7 @@ typedef struct { char fqdn[TSDB_FQDN_LEN]; int32_t port; int8_t force; + int8_t unsafe; } SDropDnodeReq; int32_t tSerializeSDropDnodeReq(void* buf, int32_t bufLen, SDropDnodeReq* pReq); @@ -1961,6 +1964,7 @@ typedef struct { SArray* fillNullCols; // array of SColLocation int64_t deleteMark; int8_t igUpdate; + int64_t lastTs; } SCMCreateStreamReq; typedef struct { @@ -1995,10 +1999,8 @@ typedef struct { int8_t withMeta; char* sql; char subDbName[TSDB_DB_FNAME_LEN]; - union { - char* ast; - char subStbName[TSDB_TABLE_FNAME_LEN]; - }; + char* ast; + char subStbName[TSDB_TABLE_FNAME_LEN]; } SCMCreateTopicReq; int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq); @@ -2021,6 +2023,11 @@ typedef struct { char cgroup[TSDB_CGROUP_LEN]; char clientId[256]; SArray* topicNames; // SArray + + int8_t withTbName; + int8_t autoCommit; + int32_t autoCommitInterval; + int8_t resetOffsetCfg; } SCMSubscribeReq; static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { @@ -2035,6 +2042,12 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc for (int32_t i = 0; i < topicNum; i++) { tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i)); } + + tlen += taosEncodeFixedI8(buf, pReq->withTbName); + tlen += taosEncodeFixedI8(buf, pReq->autoCommit); + tlen += taosEncodeFixedI32(buf, pReq->autoCommitInterval); + tlen += taosEncodeFixedI8(buf, pReq->resetOffsetCfg); + return tlen; } @@ -2052,6 +2065,11 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq buf = taosDecodeString(buf, &name); taosArrayPush(pReq->topicNames, &name); } + + buf = taosDecodeFixedI8(buf, &pReq->withTbName); + buf = taosDecodeFixedI8(buf, &pReq->autoCommit); + buf = taosDecodeFixedI32(buf, &pReq->autoCommitInterval); + buf = taosDecodeFixedI8(buf, &pReq->resetOffsetCfg); return buf; } @@ -2249,7 +2267,7 @@ typedef struct SVCreateTbReq { int32_t flags; char* name; tb_uid_t uid; - int64_t ctime; + int64_t btime; int32_t ttl; int32_t commentLen; char* comment; @@ -2386,10 +2404,12 @@ typedef struct { int32_t newTTL; int32_t newCommentLen; char* newComment; + int64_t ctimeMs; // fill by vnode } SVAlterTbReq; int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq); int32_t tDecodeSVAlterTbReq(SDecoder* pDecoder, SVAlterTbReq* pReq); +int32_t tDecodeSVAlterTbReqSetCtime(SDecoder* pDecoder, SVAlterTbReq* pReq, int64_t ctimeMs); typedef struct { int32_t code; @@ -2443,15 +2463,6 @@ typedef struct { char cgroup[TSDB_CGROUP_LEN]; } SMqAskEpReq; -typedef struct { - int64_t consumerId; - int32_t epoch; -} SMqHbReq; - -typedef struct { - int8_t reserved; -} SMqHbRsp; - typedef struct { int32_t key; int32_t valueLen; @@ -2475,6 +2486,7 @@ typedef struct { int64_t stime; // timestamp precision ms int64_t reqRid; bool stableQuery; + bool isSubQuery; char fqdn[TSDB_FQDN_LEN]; int32_t subPlanNum; SArray* subDesc; // SArray @@ -2755,6 +2767,7 @@ typedef struct { typedef struct { SMsgHead head; int64_t leftForVer; + int64_t streamId; int32_t taskId; } SVDropStreamTaskReq; @@ -2795,37 +2808,49 @@ typedef struct { int64_t suid; } SMqRebVgReq; -static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pReq->leftForVer); - tlen += taosEncodeFixedI32(buf, pReq->vgId); - tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); - tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); - tlen += taosEncodeString(buf, pReq->subKey); - tlen += taosEncodeFixedI8(buf, pReq->subType); - tlen += taosEncodeFixedI8(buf, pReq->withMeta); +static FORCE_INLINE int tEncodeSMqRebVgReq(SEncoder *pCoder, const SMqRebVgReq* pReq) { + if (tStartEncode(pCoder) < 0) return -1; + if (tEncodeI64(pCoder, pReq->leftForVer) < 0) return -1; + if (tEncodeI32(pCoder, pReq->vgId) < 0) return -1; + if (tEncodeI64(pCoder, pReq->oldConsumerId) < 0) return -1; + if (tEncodeI64(pCoder, pReq->newConsumerId) < 0) return -1; + if (tEncodeCStr(pCoder, pReq->subKey) < 0) return -1; + if (tEncodeI8(pCoder, pReq->subType) < 0) return -1; + if (tEncodeI8(pCoder, pReq->withMeta) < 0) return -1; + if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { - tlen += taosEncodeString(buf, pReq->qmsg); + if (tEncodeCStr(pCoder, pReq->qmsg) < 0) return -1; } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { - tlen += taosEncodeFixedI64(buf, pReq->suid); + if (tEncodeI64(pCoder, pReq->suid) < 0) return -1; + if (tEncodeCStr(pCoder, pReq->qmsg) < 0) return -1; } - return tlen; + tEndEncode(pCoder); + return 0; } -static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) { - buf = taosDecodeFixedI64(buf, &pReq->leftForVer); - buf = taosDecodeFixedI32(buf, &pReq->vgId); - buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); - buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); - buf = taosDecodeStringTo(buf, pReq->subKey); - buf = taosDecodeFixedI8(buf, &pReq->subType); - buf = taosDecodeFixedI8(buf, &pReq->withMeta); +static FORCE_INLINE int tDecodeSMqRebVgReq(SDecoder *pCoder, SMqRebVgReq* pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pReq->leftForVer) < 0) return -1; + + if (tDecodeI32(pCoder, &pReq->vgId) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->oldConsumerId) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->newConsumerId) < 0) return -1; + if (tDecodeCStrTo(pCoder, pReq->subKey) < 0) return -1; + if (tDecodeI8(pCoder, &pReq->subType) < 0) return -1; + if (tDecodeI8(pCoder, &pReq->withMeta) < 0) return -1; + if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { - buf = taosDecodeString(buf, &pReq->qmsg); + if (tDecodeCStr(pCoder, &pReq->qmsg) < 0) return -1; } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { - buf = taosDecodeFixedI64(buf, &pReq->suid); + if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1; + if (!tDecodeIsEnd(pCoder)){ + if (tDecodeCStr(pCoder, &pReq->qmsg) < 0) return -1; + } } - return (void*)buf; + + tEndDecode(pCoder); + return 0; } typedef struct { @@ -2867,13 +2892,19 @@ int32_t tDecodeSMqCMCommitOffsetReq(SDecoder* decoder, SMqCMCommitOffsetReq* pRe // tqOffset enum { TMQ_OFFSET__RESET_NONE = -3, - TMQ_OFFSET__RESET_EARLIEAST = -2, + TMQ_OFFSET__RESET_EARLIEST = -2, TMQ_OFFSET__RESET_LATEST = -1, TMQ_OFFSET__LOG = 1, TMQ_OFFSET__SNAPSHOT_DATA = 2, TMQ_OFFSET__SNAPSHOT_META = 3, }; +enum { + WITH_DATA = 0, + WITH_META = 1, + ONLY_META = 2, +}; + typedef struct { int8_t type; union { @@ -2928,6 +2959,7 @@ int32_t tDecodeMqVgOffset(SDecoder* pDecoder, SMqVgOffset* pOffset); typedef struct { SMsgHead head; + int64_t streamId; int32_t taskId; } SVPauseStreamTaskReq; @@ -2946,6 +2978,7 @@ int32_t tDeserializeSMPauseStreamReq(void* buf, int32_t bufLen, SMPauseStreamReq typedef struct { SMsgHead head; int32_t taskId; + int64_t streamId; int8_t igUntreated; } SVResumeStreamTaskReq; @@ -2984,6 +3017,7 @@ typedef struct { char* sql; char* ast; int64_t deleteMark; + int64_t lastTs; } SMCreateSmaReq; int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq); @@ -3177,7 +3211,8 @@ typedef struct { char dbFName[TSDB_DB_FNAME_LEN]; uint64_t suid; int32_t version; - SArray* pIndex; + int32_t indexSize; + SArray* pIndex; // STableIndexInfo } STableIndexRsp; int32_t tSerializeSTableIndexRsp(void* buf, int32_t bufLen, const STableIndexRsp* pRsp); @@ -3329,6 +3364,33 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { taosArrayDestroyEx(pRsp->topics, (FDelete)tDeleteMqSubTopicEp); } +typedef struct { + int32_t vgId; + STqOffsetVal offset; + int64_t rows; +}OffsetRows; + +typedef struct{ + char topicName[TSDB_TOPIC_FNAME_LEN]; + SArray* offsetRows; +}TopicOffsetRows; + +typedef struct { + int64_t consumerId; + int32_t epoch; + SArray* topics; +} SMqHbReq; + +typedef struct { + int8_t reserved; +} SMqHbRsp; + +typedef struct { + SMsgHead head; + int64_t consumerId; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; +} SMqSeekReq; + #define TD_AUTO_CREATE_TABLE 0x1 typedef struct { int64_t suid; @@ -3380,6 +3442,7 @@ typedef struct SDeleteRes { int64_t affectedRows; char tableFName[TSDB_TABLE_NAME_LEN]; char tsColName[TSDB_COL_NAME_LEN]; + int64_t ctimeMs; // fill by vnode } SDeleteRes; int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes); @@ -3398,10 +3461,12 @@ int32_t tDecodeSSingleDeleteReq(SDecoder* pCoder, SSingleDeleteReq* pReq); typedef struct { int64_t suid; SArray* deleteReqs; // SArray + int64_t ctimeMs; // fill by vnode } SBatchDeleteReq; int32_t tEncodeSBatchDeleteReq(SEncoder* pCoder, const SBatchDeleteReq* pReq); int32_t tDecodeSBatchDeleteReq(SDecoder* pCoder, SBatchDeleteReq* pReq); +int32_t tDecodeSBatchDeleteReqSetCtime(SDecoder* pDecoder, SBatchDeleteReq* pReq, int64_t ctimeMs); typedef struct { int32_t msgIdx; @@ -3453,10 +3518,10 @@ int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); -int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); -int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); -int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); -int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); +int32_t tDeatroySMqHbReq(SMqHbReq* pReq); + +int32_t tSerializeSMqSeekReq(void *buf, int32_t bufLen, SMqSeekReq *pReq); +int32_t tDeserializeSMqSeekReq(void *buf, int32_t bufLen, SMqSeekReq *pReq); #define SUBMIT_REQ_AUTO_CREATE_TABLE 0x1 #define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2 @@ -3471,6 +3536,7 @@ typedef struct { SArray* aRowP; SArray* aCol; }; + int64_t ctimeMs; } SSubmitTbData; typedef struct { diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 1f2d59749692d23deed0e115efc6e87c9c815e0b..232551007d2c7ec9c038b9d31420606b95bd5cb9 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -145,12 +145,12 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_TMQ_DROP_TOPIC, "drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_SUBSCRIBE, "subscribe", SCMSubscribeReq, SCMSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_ASK_EP, "ask-ep", SMqAskEpReq, SMqAskEpRsp) - TD_DEF_MSG_TYPE(TDMT_MND_TMQ_CONSUMER_LOST, "consumer-lost", SMqConsumerLostMsg, NULL) +// TD_DEF_MSG_TYPE(TDMT_MND_TMQ_CONSUMER_LOST, "consumer-lost", SMqConsumerLostMsg, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_CONSUMER_RECOVER, "consumer-recover", SMqConsumerRecoverMsg, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_HB, "consumer-hb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_DO_REBALANCE, "do-rebalance", SMqDoRebalanceMsg, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_DROP_CGROUP, "drop-cgroup", SMqDropCGroupReq, SMqDropCGroupRsp) - TD_DEF_MSG_TYPE(TDMT_MND_UNUSED2, "unused2", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VG, "create-vg", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_TIMER, "tmq-tmr", SMTimerReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "telem-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "trans-tmr", NULL, NULL) @@ -252,7 +252,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "stream-recover-finish", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_SCAN_HISTORY, "stream-scan-history", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_SCAN_HISTORY_FINISH, "stream-scan-history-finish", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_TRANSFER_STATE, "stream-transfer-state", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECK, "stream-task-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECKPOINT, "stream-checkpoint", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_REPORT_CHECKPOINT, "stream-report-checkpoint", NULL, NULL) @@ -297,8 +299,7 @@ enum { TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE, "vnode-stream-recover1", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, "vnode-stream-recover2", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_SCAN_HISTORY, "vnode-stream-scan-history", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_CHECK_POINT_SOURCE, "vnode-stream-checkpoint-source", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) @@ -306,11 +307,13 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DELETE_SUB, "vnode-tmq-delete-sub", SMqVDeleteReq, SMqVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_COMMIT_OFFSET, "vnode-tmq-commit-offset", STqOffset, STqOffset) - TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SEEK_TO_OFFSET, "vnode-tmq-seekto-offset", STqOffset, STqOffset) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SEEK, "vnode-tmq-seek", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_ADD_CHECKINFO, "vnode-tmq-add-checkinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DEL_CHECKINFO, "vnode-del-checkinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_CONSUME, "vnode-tmq-consume", SMqPollReq, SMqDataBlkRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_CONSUME_PUSH, "vnode-tmq-consume-push", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_WALINFO, "vnode-tmq-vg-walinfo", SMqPollReq, SMqDataBlkRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_COMMITTEDINFO, "vnode-tmq-committedinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL) diff --git a/include/common/ttime.h b/include/common/ttime.h index f189959f22ca4bee4b07518aff00e7bbc527fb87..de74e48100e337a516d6fa3848317074be56ec1f 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -23,7 +23,7 @@ extern "C" { #endif -#define TIME_IS_VAR_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') +#define IS_CALENDAR_TIME_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') #define TIME_UNIT_NANOSECOND 'b' #define TIME_UNIT_MICROSECOND 'u' @@ -74,7 +74,7 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); -int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); +int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval); int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 79e8cc1bf15969aae1516904bc04496892db00c0..8a6b7b50209e0d35247cc3211f71ad43d991fa14 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -70,286 +70,289 @@ #define TK_RESTORE 52 #define TK_NK_IPTOKEN 53 #define TK_FORCE 54 -#define TK_LOCAL 55 -#define TK_QNODE 56 -#define TK_BNODE 57 -#define TK_SNODE 58 -#define TK_MNODE 59 -#define TK_VNODE 60 -#define TK_DATABASE 61 -#define TK_USE 62 -#define TK_FLUSH 63 -#define TK_TRIM 64 -#define TK_COMPACT 65 -#define TK_IF 66 -#define TK_NOT 67 -#define TK_EXISTS 68 -#define TK_BUFFER 69 -#define TK_CACHEMODEL 70 -#define TK_CACHESIZE 71 -#define TK_COMP 72 -#define TK_DURATION 73 -#define TK_NK_VARIABLE 74 -#define TK_MAXROWS 75 -#define TK_MINROWS 76 -#define TK_KEEP 77 -#define TK_PAGES 78 -#define TK_PAGESIZE 79 -#define TK_TSDB_PAGESIZE 80 -#define TK_PRECISION 81 -#define TK_REPLICA 82 -#define TK_VGROUPS 83 -#define TK_SINGLE_STABLE 84 -#define TK_RETENTIONS 85 -#define TK_SCHEMALESS 86 -#define TK_WAL_LEVEL 87 -#define TK_WAL_FSYNC_PERIOD 88 -#define TK_WAL_RETENTION_PERIOD 89 -#define TK_WAL_RETENTION_SIZE 90 -#define TK_WAL_ROLL_PERIOD 91 -#define TK_WAL_SEGMENT_SIZE 92 -#define TK_STT_TRIGGER 93 -#define TK_TABLE_PREFIX 94 -#define TK_TABLE_SUFFIX 95 -#define TK_NK_COLON 96 -#define TK_MAX_SPEED 97 -#define TK_START 98 -#define TK_TIMESTAMP 99 -#define TK_END 100 -#define TK_TABLE 101 -#define TK_NK_LP 102 -#define TK_NK_RP 103 -#define TK_STABLE 104 -#define TK_ADD 105 -#define TK_COLUMN 106 -#define TK_MODIFY 107 -#define TK_RENAME 108 -#define TK_TAG 109 -#define TK_SET 110 -#define TK_NK_EQ 111 -#define TK_USING 112 -#define TK_TAGS 113 -#define TK_BOOL 114 -#define TK_TINYINT 115 -#define TK_SMALLINT 116 -#define TK_INT 117 -#define TK_INTEGER 118 -#define TK_BIGINT 119 -#define TK_FLOAT 120 -#define TK_DOUBLE 121 -#define TK_BINARY 122 -#define TK_NCHAR 123 -#define TK_UNSIGNED 124 -#define TK_JSON 125 -#define TK_VARCHAR 126 -#define TK_MEDIUMBLOB 127 -#define TK_BLOB 128 -#define TK_VARBINARY 129 -#define TK_DECIMAL 130 -#define TK_COMMENT 131 -#define TK_MAX_DELAY 132 -#define TK_WATERMARK 133 -#define TK_ROLLUP 134 -#define TK_TTL 135 -#define TK_SMA 136 -#define TK_DELETE_MARK 137 -#define TK_FIRST 138 -#define TK_LAST 139 -#define TK_SHOW 140 -#define TK_PRIVILEGES 141 -#define TK_DATABASES 142 -#define TK_TABLES 143 -#define TK_STABLES 144 -#define TK_MNODES 145 -#define TK_QNODES 146 -#define TK_FUNCTIONS 147 -#define TK_INDEXES 148 -#define TK_ACCOUNTS 149 -#define TK_APPS 150 -#define TK_CONNECTIONS 151 -#define TK_LICENCES 152 -#define TK_GRANTS 153 -#define TK_QUERIES 154 -#define TK_SCORES 155 -#define TK_TOPICS 156 -#define TK_VARIABLES 157 -#define TK_CLUSTER 158 -#define TK_BNODES 159 -#define TK_SNODES 160 -#define TK_TRANSACTIONS 161 -#define TK_DISTRIBUTED 162 -#define TK_CONSUMERS 163 -#define TK_SUBSCRIPTIONS 164 -#define TK_VNODES 165 -#define TK_ALIVE 166 -#define TK_LIKE 167 -#define TK_TBNAME 168 -#define TK_QTAGS 169 -#define TK_AS 170 -#define TK_INDEX 171 -#define TK_FUNCTION 172 -#define TK_INTERVAL 173 -#define TK_COUNT 174 -#define TK_LAST_ROW 175 -#define TK_TOPIC 176 -#define TK_META 177 -#define TK_CONSUMER 178 -#define TK_GROUP 179 -#define TK_DESC 180 -#define TK_DESCRIBE 181 -#define TK_RESET 182 -#define TK_QUERY 183 -#define TK_CACHE 184 -#define TK_EXPLAIN 185 -#define TK_ANALYZE 186 -#define TK_VERBOSE 187 -#define TK_NK_BOOL 188 -#define TK_RATIO 189 -#define TK_NK_FLOAT 190 -#define TK_OUTPUTTYPE 191 -#define TK_AGGREGATE 192 -#define TK_BUFSIZE 193 -#define TK_LANGUAGE 194 -#define TK_REPLACE 195 -#define TK_STREAM 196 -#define TK_INTO 197 -#define TK_PAUSE 198 -#define TK_RESUME 199 -#define TK_TRIGGER 200 -#define TK_AT_ONCE 201 -#define TK_WINDOW_CLOSE 202 -#define TK_IGNORE 203 -#define TK_EXPIRED 204 -#define TK_FILL_HISTORY 205 -#define TK_UPDATE 206 -#define TK_SUBTABLE 207 -#define TK_UNTREATED 208 -#define TK_KILL 209 -#define TK_CONNECTION 210 -#define TK_TRANSACTION 211 -#define TK_BALANCE 212 -#define TK_VGROUP 213 -#define TK_LEADER 214 -#define TK_MERGE 215 -#define TK_REDISTRIBUTE 216 -#define TK_SPLIT 217 -#define TK_DELETE 218 -#define TK_INSERT 219 -#define TK_NULL 220 -#define TK_NK_QUESTION 221 -#define TK_NK_ARROW 222 -#define TK_ROWTS 223 -#define TK_QSTART 224 -#define TK_QEND 225 -#define TK_QDURATION 226 -#define TK_WSTART 227 -#define TK_WEND 228 -#define TK_WDURATION 229 -#define TK_IROWTS 230 -#define TK_ISFILLED 231 -#define TK_CAST 232 -#define TK_NOW 233 -#define TK_TODAY 234 -#define TK_TIMEZONE 235 -#define TK_CLIENT_VERSION 236 -#define TK_SERVER_VERSION 237 -#define TK_SERVER_STATUS 238 -#define TK_CURRENT_USER 239 -#define TK_CASE 240 -#define TK_WHEN 241 -#define TK_THEN 242 -#define TK_ELSE 243 -#define TK_BETWEEN 244 -#define TK_IS 245 -#define TK_NK_LT 246 -#define TK_NK_GT 247 -#define TK_NK_LE 248 -#define TK_NK_GE 249 -#define TK_NK_NE 250 -#define TK_MATCH 251 -#define TK_NMATCH 252 -#define TK_CONTAINS 253 -#define TK_IN 254 -#define TK_JOIN 255 -#define TK_INNER 256 -#define TK_SELECT 257 -#define TK_DISTINCT 258 -#define TK_WHERE 259 -#define TK_PARTITION 260 -#define TK_BY 261 -#define TK_SESSION 262 -#define TK_STATE_WINDOW 263 -#define TK_EVENT_WINDOW 264 -#define TK_SLIDING 265 -#define TK_FILL 266 -#define TK_VALUE 267 -#define TK_VALUE_F 268 -#define TK_NONE 269 -#define TK_PREV 270 -#define TK_NULL_F 271 -#define TK_LINEAR 272 -#define TK_NEXT 273 -#define TK_HAVING 274 -#define TK_RANGE 275 -#define TK_EVERY 276 -#define TK_ORDER 277 -#define TK_SLIMIT 278 -#define TK_SOFFSET 279 -#define TK_LIMIT 280 -#define TK_OFFSET 281 -#define TK_ASC 282 -#define TK_NULLS 283 -#define TK_ABORT 284 -#define TK_AFTER 285 -#define TK_ATTACH 286 -#define TK_BEFORE 287 -#define TK_BEGIN 288 -#define TK_BITAND 289 -#define TK_BITNOT 290 -#define TK_BITOR 291 -#define TK_BLOCKS 292 -#define TK_CHANGE 293 -#define TK_COMMA 294 -#define TK_CONCAT 295 -#define TK_CONFLICT 296 -#define TK_COPY 297 -#define TK_DEFERRED 298 -#define TK_DELIMITERS 299 -#define TK_DETACH 300 -#define TK_DIVIDE 301 -#define TK_DOT 302 -#define TK_EACH 303 -#define TK_FAIL 304 -#define TK_FILE 305 -#define TK_FOR 306 -#define TK_GLOB 307 -#define TK_ID 308 -#define TK_IMMEDIATE 309 -#define TK_IMPORT 310 -#define TK_INITIALLY 311 -#define TK_INSTEAD 312 -#define TK_ISNULL 313 -#define TK_KEY 314 -#define TK_MODULES 315 -#define TK_NK_BITNOT 316 -#define TK_NK_SEMI 317 -#define TK_NOTNULL 318 -#define TK_OF 319 -#define TK_PLUS 320 -#define TK_PRIVILEGE 321 -#define TK_RAISE 322 -#define TK_RESTRICT 323 -#define TK_ROW 324 -#define TK_SEMI 325 -#define TK_STAR 326 -#define TK_STATEMENT 327 -#define TK_STRICT 328 -#define TK_STRING 329 -#define TK_TIMES 330 -#define TK_VALUES 331 -#define TK_VARIABLE 332 -#define TK_VIEW 333 -#define TK_WAL 334 +#define TK_UNSAFE 55 +#define TK_LOCAL 56 +#define TK_QNODE 57 +#define TK_BNODE 58 +#define TK_SNODE 59 +#define TK_MNODE 60 +#define TK_VNODE 61 +#define TK_DATABASE 62 +#define TK_USE 63 +#define TK_FLUSH 64 +#define TK_TRIM 65 +#define TK_COMPACT 66 +#define TK_IF 67 +#define TK_NOT 68 +#define TK_EXISTS 69 +#define TK_BUFFER 70 +#define TK_CACHEMODEL 71 +#define TK_CACHESIZE 72 +#define TK_COMP 73 +#define TK_DURATION 74 +#define TK_NK_VARIABLE 75 +#define TK_MAXROWS 76 +#define TK_MINROWS 77 +#define TK_KEEP 78 +#define TK_PAGES 79 +#define TK_PAGESIZE 80 +#define TK_TSDB_PAGESIZE 81 +#define TK_PRECISION 82 +#define TK_REPLICA 83 +#define TK_VGROUPS 84 +#define TK_SINGLE_STABLE 85 +#define TK_RETENTIONS 86 +#define TK_SCHEMALESS 87 +#define TK_WAL_LEVEL 88 +#define TK_WAL_FSYNC_PERIOD 89 +#define TK_WAL_RETENTION_PERIOD 90 +#define TK_WAL_RETENTION_SIZE 91 +#define TK_WAL_ROLL_PERIOD 92 +#define TK_WAL_SEGMENT_SIZE 93 +#define TK_STT_TRIGGER 94 +#define TK_TABLE_PREFIX 95 +#define TK_TABLE_SUFFIX 96 +#define TK_NK_COLON 97 +#define TK_MAX_SPEED 98 +#define TK_START 99 +#define TK_TIMESTAMP 100 +#define TK_END 101 +#define TK_TABLE 102 +#define TK_NK_LP 103 +#define TK_NK_RP 104 +#define TK_STABLE 105 +#define TK_ADD 106 +#define TK_COLUMN 107 +#define TK_MODIFY 108 +#define TK_RENAME 109 +#define TK_TAG 110 +#define TK_SET 111 +#define TK_NK_EQ 112 +#define TK_USING 113 +#define TK_TAGS 114 +#define TK_BOOL 115 +#define TK_TINYINT 116 +#define TK_SMALLINT 117 +#define TK_INT 118 +#define TK_INTEGER 119 +#define TK_BIGINT 120 +#define TK_FLOAT 121 +#define TK_DOUBLE 122 +#define TK_BINARY 123 +#define TK_NCHAR 124 +#define TK_UNSIGNED 125 +#define TK_JSON 126 +#define TK_VARCHAR 127 +#define TK_MEDIUMBLOB 128 +#define TK_BLOB 129 +#define TK_VARBINARY 130 +#define TK_GEOMETRY 131 +#define TK_DECIMAL 132 +#define TK_COMMENT 133 +#define TK_MAX_DELAY 134 +#define TK_WATERMARK 135 +#define TK_ROLLUP 136 +#define TK_TTL 137 +#define TK_SMA 138 +#define TK_DELETE_MARK 139 +#define TK_FIRST 140 +#define TK_LAST 141 +#define TK_SHOW 142 +#define TK_PRIVILEGES 143 +#define TK_DATABASES 144 +#define TK_TABLES 145 +#define TK_STABLES 146 +#define TK_MNODES 147 +#define TK_QNODES 148 +#define TK_FUNCTIONS 149 +#define TK_INDEXES 150 +#define TK_ACCOUNTS 151 +#define TK_APPS 152 +#define TK_CONNECTIONS 153 +#define TK_LICENCES 154 +#define TK_GRANTS 155 +#define TK_QUERIES 156 +#define TK_SCORES 157 +#define TK_TOPICS 158 +#define TK_VARIABLES 159 +#define TK_CLUSTER 160 +#define TK_BNODES 161 +#define TK_SNODES 162 +#define TK_TRANSACTIONS 163 +#define TK_DISTRIBUTED 164 +#define TK_CONSUMERS 165 +#define TK_SUBSCRIPTIONS 166 +#define TK_VNODES 167 +#define TK_ALIVE 168 +#define TK_LIKE 169 +#define TK_TBNAME 170 +#define TK_QTAGS 171 +#define TK_AS 172 +#define TK_INDEX 173 +#define TK_FUNCTION 174 +#define TK_INTERVAL 175 +#define TK_COUNT 176 +#define TK_LAST_ROW 177 +#define TK_META 178 +#define TK_ONLY 179 +#define TK_TOPIC 180 +#define TK_CONSUMER 181 +#define TK_GROUP 182 +#define TK_DESC 183 +#define TK_DESCRIBE 184 +#define TK_RESET 185 +#define TK_QUERY 186 +#define TK_CACHE 187 +#define TK_EXPLAIN 188 +#define TK_ANALYZE 189 +#define TK_VERBOSE 190 +#define TK_NK_BOOL 191 +#define TK_RATIO 192 +#define TK_NK_FLOAT 193 +#define TK_OUTPUTTYPE 194 +#define TK_AGGREGATE 195 +#define TK_BUFSIZE 196 +#define TK_LANGUAGE 197 +#define TK_REPLACE 198 +#define TK_STREAM 199 +#define TK_INTO 200 +#define TK_PAUSE 201 +#define TK_RESUME 202 +#define TK_TRIGGER 203 +#define TK_AT_ONCE 204 +#define TK_WINDOW_CLOSE 205 +#define TK_IGNORE 206 +#define TK_EXPIRED 207 +#define TK_FILL_HISTORY 208 +#define TK_UPDATE 209 +#define TK_SUBTABLE 210 +#define TK_UNTREATED 211 +#define TK_KILL 212 +#define TK_CONNECTION 213 +#define TK_TRANSACTION 214 +#define TK_BALANCE 215 +#define TK_VGROUP 216 +#define TK_LEADER 217 +#define TK_MERGE 218 +#define TK_REDISTRIBUTE 219 +#define TK_SPLIT 220 +#define TK_DELETE 221 +#define TK_INSERT 222 +#define TK_NULL 223 +#define TK_NK_QUESTION 224 +#define TK_NK_ARROW 225 +#define TK_ROWTS 226 +#define TK_QSTART 227 +#define TK_QEND 228 +#define TK_QDURATION 229 +#define TK_WSTART 230 +#define TK_WEND 231 +#define TK_WDURATION 232 +#define TK_IROWTS 233 +#define TK_ISFILLED 234 +#define TK_CAST 235 +#define TK_NOW 236 +#define TK_TODAY 237 +#define TK_TIMEZONE 238 +#define TK_CLIENT_VERSION 239 +#define TK_SERVER_VERSION 240 +#define TK_SERVER_STATUS 241 +#define TK_CURRENT_USER 242 +#define TK_CASE 243 +#define TK_WHEN 244 +#define TK_THEN 245 +#define TK_ELSE 246 +#define TK_BETWEEN 247 +#define TK_IS 248 +#define TK_NK_LT 249 +#define TK_NK_GT 250 +#define TK_NK_LE 251 +#define TK_NK_GE 252 +#define TK_NK_NE 253 +#define TK_MATCH 254 +#define TK_NMATCH 255 +#define TK_CONTAINS 256 +#define TK_IN 257 +#define TK_JOIN 258 +#define TK_INNER 259 +#define TK_SELECT 260 +#define TK_DISTINCT 261 +#define TK_WHERE 262 +#define TK_PARTITION 263 +#define TK_BY 264 +#define TK_SESSION 265 +#define TK_STATE_WINDOW 266 +#define TK_EVENT_WINDOW 267 +#define TK_SLIDING 268 +#define TK_FILL 269 +#define TK_VALUE 270 +#define TK_VALUE_F 271 +#define TK_NONE 272 +#define TK_PREV 273 +#define TK_NULL_F 274 +#define TK_LINEAR 275 +#define TK_NEXT 276 +#define TK_HAVING 277 +#define TK_RANGE 278 +#define TK_EVERY 279 +#define TK_ORDER 280 +#define TK_SLIMIT 281 +#define TK_SOFFSET 282 +#define TK_LIMIT 283 +#define TK_OFFSET 284 +#define TK_ASC 285 +#define TK_NULLS 286 +#define TK_ABORT 287 +#define TK_AFTER 288 +#define TK_ATTACH 289 +#define TK_BEFORE 290 +#define TK_BEGIN 291 +#define TK_BITAND 292 +#define TK_BITNOT 293 +#define TK_BITOR 294 +#define TK_BLOCKS 295 +#define TK_CHANGE 296 +#define TK_COMMA 297 +#define TK_CONCAT 298 +#define TK_CONFLICT 299 +#define TK_COPY 300 +#define TK_DEFERRED 301 +#define TK_DELIMITERS 302 +#define TK_DETACH 303 +#define TK_DIVIDE 304 +#define TK_DOT 305 +#define TK_EACH 306 +#define TK_FAIL 307 +#define TK_FILE 308 +#define TK_FOR 309 +#define TK_GLOB 310 +#define TK_ID 311 +#define TK_IMMEDIATE 312 +#define TK_IMPORT 313 +#define TK_INITIALLY 314 +#define TK_INSTEAD 315 +#define TK_ISNULL 316 +#define TK_KEY 317 +#define TK_MODULES 318 +#define TK_NK_BITNOT 319 +#define TK_NK_SEMI 320 +#define TK_NOTNULL 321 +#define TK_OF 322 +#define TK_PLUS 323 +#define TK_PRIVILEGE 324 +#define TK_RAISE 325 +#define TK_RESTRICT 326 +#define TK_ROW 327 +#define TK_SEMI 328 +#define TK_STAR 329 +#define TK_STATEMENT 330 +#define TK_STRICT 331 +#define TK_STRING 332 +#define TK_TIMES 333 +#define TK_VALUES 334 +#define TK_VARIABLE 335 +#define TK_VIEW 336 +#define TK_WAL 337 #define TK_NK_SPACE 600 diff --git a/include/common/ttypes.h b/include/common/ttypes.h index f8a85ee1b0a86e268046f813ccb1721d97c3c029..dd8033eb436bdc6695ac252187f099cb3e55b301 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -269,7 +269,7 @@ typedef struct { (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) #define IS_VAR_DATA_TYPE(t) \ - (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON)) + (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY)) #define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) #define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) @@ -316,6 +316,8 @@ static FORCE_INLINE bool isNull(const void *val, int32_t type) { return *(uint32_t *)val == TSDB_DATA_UINT_NULL; case TSDB_DATA_TYPE_UBIGINT: return *(uint64_t *)val == TSDB_DATA_UBIGINT_NULL; + case TSDB_DATA_TYPE_GEOMETRY: + return varDataLen(val) == sizeof(int8_t) && *(uint8_t *)varDataVal(val) == TSDB_DATA_GEOMETRY_NULL; default: return false; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index d7084cfac46c6aa49552a55682ba6a31ad9ffc30..7a7a13b28523be078f3f62d82c7e527681c54a83 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -87,6 +87,7 @@ typedef struct SCatalogReq { bool dNodeRequired; // valid dnode bool svrVerRequired; bool forceUpdate; + bool cloned; } SCatalogReq; typedef struct SMetaRes { @@ -129,13 +130,14 @@ typedef struct SSTableVersion { int32_t smaVer; } SSTableVersion; -typedef struct SDbVgVersion { +typedef struct SDbCacheInfo { char dbFName[TSDB_DB_FNAME_LEN]; int64_t dbId; int32_t vgVersion; + int32_t cfgVersion; int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT int64_t stateTs; -} SDbVgVersion; +} SDbCacheInfo; typedef struct STbSVersion { char* tbFName; @@ -148,7 +150,6 @@ typedef struct SUserAuthVersion { int32_t version; } SUserAuthVersion; -typedef SDbCfgRsp SDbCfgInfo; typedef SUserIndexRsp SIndexInfo; typedef void (*catalogCallback)(SMetaData* pResult, void* param, int32_t code); @@ -180,6 +181,8 @@ int32_t catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgInfo* dbInfo); +int32_t catalogUpdateDbCfg(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDbCfgInfo* cfgInfo); + int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId); int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName); @@ -212,7 +215,7 @@ int32_t catalogGetSTableMeta(SCatalog* pCatalog, SRequestConnInfo* pConn, const int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg); -int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg); +int32_t catalogAsyncUpdateTableMeta(SCatalog* pCtg, STableMetaRsp* pMsg); int32_t catalogGetCachedTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta); @@ -304,7 +307,7 @@ int32_t catalogGetDnodeList(SCatalog* pCatalog, SRequestConnInfo* pConn, SArray* int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion** stables, uint32_t* num); -int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion** dbs, uint32_t* num); +int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbCacheInfo** dbs, uint32_t* num); int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num); diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index ce7d038d42c8f37bec730111d6d26d946c47a168..0a9037d21cff5e87e5f0da3b0948651e291ff781 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -59,7 +59,7 @@ typedef struct SDataSinkMgtCfg { uint32_t maxDataBlockNumPerQuery; } SDataSinkMgtCfg; -int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg); +int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI); typedef struct SInputData { const struct SSDataBlock* pData; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index b7e6c42e3b7ede77237aa7e8e910b45f236b5cc9..634d708260790fb26ec5c6fb28987fedbecbc75f 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -23,6 +23,7 @@ extern "C" { #include "query.h" #include "tcommon.h" #include "tmsgcb.h" +#include "storageapi.h" typedef void* qTaskInfo_t; typedef void* DataSinkHandle; @@ -41,7 +42,6 @@ typedef struct { typedef struct { void* tqReader; - void* meta; void* config; void* vnode; void* mnd; @@ -51,10 +51,13 @@ typedef struct { bool initTableReader; bool initTqReader; int32_t numOfVgroups; + void* sContext; // SSnapContext* - void* sContext; // SSnapContext* + void* pStateBackend; + struct SStorageAPI api; - void* pStateBackend; + int8_t fillHistory; + STimeWindow winRange; } SReadHandle; // in queue mode, data streams are seperated by msg @@ -71,7 +74,7 @@ typedef enum { * @param vgId * @return */ -qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t vgId); +qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t vgId, int32_t taskId); /** * Create the exec task for queue mode @@ -82,6 +85,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id); +int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList, void* pTaskInfo); + /** * set the task Id, usually used by message queue process * @param tinfo @@ -186,12 +191,12 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len); int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len); -STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key); +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); +STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key); SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo); -void verifyOffset(void *pWalReader, STqOffsetVal* pOffset); - int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType); void qStreamSetOpen(qTaskInfo_t tinfo); @@ -208,15 +213,18 @@ void* qExtractReaderFromStreamScanner(void* scanner); int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner); -int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo); -int32_t qStreamSourceRecoverStep1(qTaskInfo_t tinfo, int64_t ver); -int32_t qStreamSourceRecoverStep2(qTaskInfo_t tinfo, int64_t ver); +int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo); +int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow); +int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow); int32_t qStreamRecoverFinish(qTaskInfo_t tinfo); -int32_t qStreamRestoreParam(qTaskInfo_t tinfo); +int32_t qRestoreStreamOperatorOption(qTaskInfo_t tinfo); bool qStreamRecoverScanFinished(qTaskInfo_t tinfo); -void qStreamCloseTsdbReader(void* task); +int32_t qStreamInfoResetTimewindowFilter(qTaskInfo_t tinfo); void resetTaskInfo(qTaskInfo_t tinfo); +int32_t qStreamOperatorReleaseState(qTaskInfo_t tInfo); +int32_t qStreamOperatorReloadState(qTaskInfo_t tInfo); + #ifdef __cplusplus } #endif diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h new file mode 100644 index 0000000000000000000000000000000000000000..773f373a2d174121a7396bed1f5827e7bb514987 --- /dev/null +++ b/include/libs/executor/storageapi.h @@ -0,0 +1,416 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_STORAGEAPI_H +#define TDENGINE_STORAGEAPI_H + +#include "function.h" +#include "index.h" +#include "taosdef.h" +#include "tcommon.h" +#include "tmsg.h" +#include "tscalablebf.h" +#include "tsimplehash.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define TIMEWINDOW_RANGE_CONTAINED 1 +#define TIMEWINDOW_RANGE_EXTERNAL 2 + +#define CACHESCAN_RETRIEVE_TYPE_ALL 0x1 +#define CACHESCAN_RETRIEVE_TYPE_SINGLE 0x2 +#define CACHESCAN_RETRIEVE_LAST_ROW 0x4 +#define CACHESCAN_RETRIEVE_LAST 0x8 + +#define META_READER_NOLOCK 0x1 + +typedef struct SMeta SMeta; +typedef TSKEY (*GetTsFun)(void*); + +typedef struct SMetaEntry { + int64_t version; + int8_t type; + int8_t flags; // TODO: need refactor? + tb_uid_t uid; + char* name; + union { + struct { + SSchemaWrapper schemaRow; + SSchemaWrapper schemaTag; + SRSmaParam rsmaParam; + } stbEntry; + struct { + int64_t btime; + int32_t ttlDays; + int32_t commentLen; + char* comment; + tb_uid_t suid; + uint8_t* pTags; + } ctbEntry; + struct { + int64_t btime; + int32_t ttlDays; + int32_t commentLen; + char* comment; + int32_t ncid; // next column id + SSchemaWrapper schemaRow; + } ntbEntry; + struct { + STSma* tsma; + } smaEntry; + }; + + uint8_t* pBuf; +} SMetaEntry; + +typedef struct SMetaReader { + int32_t flags; + void* pMeta; + SDecoder coder; + SMetaEntry me; + void* pBuf; + int32_t szBuf; + struct SStoreMeta* pAPI; +} SMetaReader; + +typedef struct SMTbCursor { + void* pMeta; + void* pDbc; + void* pKey; + void* pVal; + int32_t kLen; + int32_t vLen; + SMetaReader mr; + int8_t paused; +} SMTbCursor; + +typedef struct SRowBuffPos { + void* pRowBuff; + void* pKey; + bool beFlushed; + bool beUsed; +} SRowBuffPos; + +// tq +typedef struct SMetaTableInfo { + int64_t suid; + int64_t uid; + SSchemaWrapper* schema; + char tbName[TSDB_TABLE_NAME_LEN]; +} SMetaTableInfo; + +typedef struct SSnapContext { + SMeta* pMeta; // todo remove it + int64_t snapVersion; + void* pCur; + int64_t suid; + int8_t subType; + SHashObj* idVersion; + SHashObj* suidInfo; + SArray* idList; + int32_t index; + int8_t withMeta; + int8_t queryMeta; // true-get meta, false-get data +} SSnapContext; + +typedef struct { + int64_t uid; + int64_t ctbNum; +} SMetaStbStats; + +// void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); +// int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); +// int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); +// int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); +// bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); +// bool tqCurrentBlockConsumed(const STqReader* pReader); +// int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); +// bool tqNextBlockInWal(STqReader* pReader, const char* idstr); +// bool tqNextBlockImpl(STqReader *pReader, const char* idstr); +// int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t +// *uid); SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx); int32_t setForSnapShot(SSnapContext +// *ctx, int64_t uid); int32_t destroySnapContext(SSnapContext *ctx); + +// clang-format off +/*-------------------------------------------------new api format---------------------------------------------------*/ +typedef struct TsdReader { + int32_t (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, + SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly, + SHashObj** pIgnoreTables); + void (*tsdReaderClose)(); + void (*tsdSetReaderTaskId)(void *pReader, const char *pId); + int32_t (*tsdSetQueryTableList)(); + int32_t (*tsdNextDataBlock)(); + + int32_t (*tsdReaderRetrieveBlockSMAInfo)(); + SSDataBlock *(*tsdReaderRetrieveDataBlock)(); + + void (*tsdReaderReleaseDataBlock)(); + + int32_t (*tsdReaderResetStatus)(); + int32_t (*tsdReaderGetDataBlockDistInfo)(); + int64_t (*tsdReaderGetNumOfInMemRows)(); + void (*tsdReaderNotifyClosing)(); +} TsdReader; + +typedef struct SStoreCacheReader { + int32_t (*openReader)(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, + SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); + void *(*closeReader)(void *pReader); + int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, + SArray *pTableUidList); + int32_t (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables); +} SStoreCacheReader; + +// clang-format on + +/*------------------------------------------------------------------------------------------------------------------*/ +/* +void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); +int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); +int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); +bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); +bool tqCurrentBlockConsumed(const STqReader* pReader); + +int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); +bool tqNextBlockInWal(STqReader* pReader, const char* idstr); +bool tqNextBlockImpl(STqReader *pReader, const char* idstr); + + int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char* idstr); +STqReader *tqReaderOpen(void *pVnode); +void tqReaderClose(STqReader *); + +int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); +bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); +SWalReader* tqGetWalReader(STqReader* pReader); +int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); +*/ +// todo rename +typedef struct SStoreTqReader { + struct STqReader* (*tqReaderOpen)(); + void (*tqReaderClose)(); + + int32_t (*tqReaderSeek)(); + int32_t (*tqRetrieveBlock)(); + bool (*tqReaderNextBlockInWal)(); + bool (*tqNextBlockImpl)(); // todo remove it + SSDataBlock* (*tqGetResultBlock)(); + + void (*tqReaderSetColIdList)(); + int32_t (*tqReaderSetQueryTableList)(); + + int32_t (*tqReaderAddTables)(); + int32_t (*tqReaderRemoveTables)(); + + bool (*tqReaderIsQueriedTable)(); + bool (*tqReaderCurrentBlockConsumed)(); + + struct SWalReader* (*tqReaderGetWalReader)(); // todo remove it + int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it + + int32_t (*tqReaderSetSubmitMsg)(); // todo remove it + bool (*tqReaderNextBlockFilterOut)(); +} SStoreTqReader; + +typedef struct SStoreSnapshotFn { + int32_t (*setForSnapShot)(SSnapContext* ctx, int64_t uid); + int32_t (*destroySnapshot)(SSnapContext* ctx); + SMetaTableInfo (*getMetaTableInfoFromSnapshot)(SSnapContext* ctx); + int32_t (*getTableInfoFromSnapshot)(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid); +} SStoreSnapshotFn; + +typedef struct SStoreMeta { + SMTbCursor* (*openTableMetaCursor)(void* pVnode); // metaOpenTbCursor + void (*closeTableMetaCursor)(SMTbCursor* pTbCur); // metaCloseTbCursor + void (*pauseTableMetaCursor)(SMTbCursor* pTbCur); // metaPauseTbCursor + void (*resumeTableMetaCursor)(SMTbCursor* pTbCur, int8_t first); // metaResumeTbCursor + int32_t (*cursorNext)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorNext + int32_t (*cursorPrev)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorPrev + + int32_t (*getTableTags)(void* pVnode, uint64_t suid, SArray* uidList); + int32_t (*getTableTagsByUid)(void* pVnode, int64_t suid, SArray* uidList); + const void* (*extractTagVal)(const void* tag, int16_t type, STagVal* tagVal); // todo remove it + + int32_t (*getTableUidByName)(void* pVnode, char* tbName, uint64_t* uid); + int32_t (*getTableTypeByName)(void* pVnode, char* tbName, ETableType* tbType); + int32_t (*getTableNameByUid)(void* pVnode, uint64_t uid, char* tbName); + bool (*isTableExisted)(void* pVnode, tb_uid_t uid); + + int32_t (*metaGetCachedTbGroup)(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); + int32_t (*metaPutTbGroupToCache)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen); + + int32_t (*getCachedTableList)(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, + bool* acquireRes); + int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen, double selectivityRatio); + + void* (*storeGetIndexInfo)(); + void* (*getInvertIndex)(void* pVnode); + int32_t (*getChildTableList)( + void* pVnode, int64_t suid, + SArray* list); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] + int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList); // vnodeGetStbIdList & vnodeGetAllTableList + void* storeGetVersionRange; + void* storeGetLastTimestamp; + + int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid); // tsdbGetTableSchema + + // db name, vgId, numOfTables, numOfSTables + int32_t (*getNumOfChildTables)( + void* pVnode, int64_t uid, + int64_t* numOfTables); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); + void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables, + int64_t* numOfNormalTables); // vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & + // metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); + + int64_t (*getNumOfRowsInMem)(void* pVnode); + /** +int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); +int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); +int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); + */ +} SStoreMeta; + +typedef struct SStoreMetaReader { + void (*initReader)(SMetaReader* pReader, void* pVnode, int32_t flags, SStoreMeta* pAPI); + void (*clearReader)(SMetaReader* pReader); + void (*readerReleaseLock)(SMetaReader* pReader); + int32_t (*getTableEntryByUid)(SMetaReader* pReader, tb_uid_t uid); + int32_t (*getTableEntryByName)(SMetaReader* pReader, const char* name); + int32_t (*getEntryGetUidCache)(SMetaReader* pReader, tb_uid_t uid); +} SStoreMetaReader; + +typedef struct SUpdateInfo { + SArray* pTsBuckets; + uint64_t numBuckets; + SArray* pTsSBFs; + uint64_t numSBFs; + int64_t interval; + int64_t watermark; + TSKEY minTS; + SScalableBf* pCloseWinSBF; + SHashObj* pMap; + uint64_t maxDataVersion; +} SUpdateInfo; + +typedef struct { + void* iter; // rocksdb_iterator_t* iter; + void* snapshot; // rocksdb_snapshot_t* snapshot; + void* readOpt; // rocksdb_readoptions_t* readOpt; + void* db; // rocksdb_t* db; + void* pCur; + int64_t number; +} SStreamStateCur; + +typedef struct SStateStore { + int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname); + int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal); + + int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); + int32_t (*streamStateReleaseBuf)(SStreamState* pState, const SWinKey* key, void* pVal); + void (*streamStateFreeVal)(void* val); + + int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); + int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); + bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key); + int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal); + int32_t (*streamStateDel)(SStreamState* pState, const SWinKey* key); + int32_t (*streamStateClear)(SStreamState* pState); + void (*streamStateSetNumber)(SStreamState* pState, int32_t number); + int32_t (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); + int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); + + int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); + int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); + int32_t (*streamStateFillDel)(SStreamState* pState, const SWinKey* key); + + int32_t (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur); + int32_t (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur); + + SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); + SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); + SStreamStateCur* (*streamStateFillSeekKeyNext)(SStreamState* pState, const SWinKey* key); + SStreamStateCur* (*streamStateFillSeekKeyPrev)(SStreamState* pState, const SWinKey* key); + void (*streamStateFreeCur)(SStreamStateCur* pCur); + + int32_t (*streamStateGetGroupKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); + int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); + + int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, + int32_t* pVLen); + int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen); + int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); + int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); + int32_t (*streamStateSessionClear)(SStreamState* pState); + int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); + int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); + int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); + + SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark); + TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol); + bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts); + bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); + void (*updateInfoDestroy)(SUpdateInfo* pInfo); + void (*windowSBfDelete)(SUpdateInfo *pInfo, uint64_t count); + void (*windowSBfAdd)(SUpdateInfo *pInfo, uint64_t count); + + SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark); + void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); + void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); + int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); + int32_t (*updateInfoDeserialize)(void* buf, int32_t bufLen, SUpdateInfo* pInfo); + + SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); + SStreamStateCur* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key); + SStreamStateCur* (*streamStateSessionSeekKeyCurrentNext)(SStreamState* pState, const SSessionKey* key); + + struct SStreamFileState* (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, + uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark, const char*id); + + void (*streamFileStateDestroy)(struct SStreamFileState* pFileState); + void (*streamFileStateClear)(struct SStreamFileState* pFileState); + bool (*needClearDiskBuff)(struct SStreamFileState* pFileState); + + SStreamState* (*streamStateOpen)(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); + void (*streamStateClose)(SStreamState* pState, bool remove); + int32_t (*streamStateBegin)(SStreamState* pState); + int32_t (*streamStateCommit)(SStreamState* pState); + void (*streamStateDestroy)(SStreamState* pState, bool remove); + int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); + void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts); +} SStateStore; + +typedef struct SStorageAPI { + SStoreMeta metaFn; // todo: refactor + TsdReader tsdReader; + SStoreMetaReader metaReaderFn; + SStoreCacheReader cacheFn; + SStoreSnapshotFn snapshotFn; + SStoreTqReader tqReaderFn; + SStateStore stateStore; + SMetaDataFilterAPI metaFilter; + SFunctionStateStore functionStore; +} SStorageAPI; + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_STORAGEAPI_H diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 85f7cf7e2c1e2d8de3970029ed9bffe9fb4c0563..2e3cd670d73651494aafc148a474dc8d2f48ad3c 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "tcommon.h" +#include "tsimplehash.h" #include "tvariant.h" struct SqlFunctionCtx; @@ -76,7 +77,7 @@ enum { enum { MAIN_SCAN = 0x0u, REVERSE_SCAN = 0x1u, // todo remove it - PRE_SCAN = 0x2u, // pre-scan belongs to the main scan and occurs before main scan + PRE_SCAN = 0x2u, // pre-scan belongs to the main scan and occurs before main scan }; typedef struct SPoint1 { @@ -127,16 +128,67 @@ typedef struct SSerializeDataHandle { void *pState; } SSerializeDataHandle; +// incremental state storage + +typedef struct SBackendCfWrapper { + void *rocksdb; + void **pHandle; + void *writeOpts; + void *readOpts; + void **cfOpts; + void *dbOpt; + void *param; + void *env; + SListNode *pComparNode; + void *pBackend; + void *compactFactory; + TdThreadRwlock rwLock; + bool remove; + int64_t backendId; + char idstr[64]; +} SBackendCfWrapper; +typedef struct STdbState { + SBackendCfWrapper *pBackendCfWrapper; + int64_t backendCfWrapperId; + char idstr[64]; + + struct SStreamTask *pOwner; + void *db; + void *pStateDb; + void *pFuncStateDb; + void *pFillStateDb; // todo refactor + void *pSessionStateDb; + void *pParNameDb; + void *pParTagDb; + void *txn; +} STdbState; + +typedef struct { + STdbState *pTdbState; + struct SStreamFileState *pFileState; + int32_t number; + SSHashObj *parNameMap; + int64_t checkPointId; + int32_t taskId; + int64_t streamId; + int64_t streamBackendRid; +} SStreamState; + +typedef struct SFunctionStateStore { + int32_t (*streamStateFuncPut)(SStreamState *pState, const SWinKey *key, const void *value, int32_t vLen); + int32_t (*streamStateFuncGet)(SStreamState *pState, const SWinKey *key, void **ppVal, int32_t *pVLen); +} SFunctionStateStore; + // sql function runtime context typedef struct SqlFunctionCtx { SInputColumnInfoData input; SResultDataInfo resDataInfo; - uint32_t order; // data block scanner order: asc|desc - uint8_t isPseudoFunc;// denote current function is pseudo function or not [added for perf reason] - uint8_t isNotNullFunc;// not return null value. - uint8_t scanFlag; // record current running step, default: 0 - int16_t functionId; // function id - char *pOutput; // final result output buffer, point to sdata->data + uint32_t order; // data block scanner order: asc|desc + uint8_t isPseudoFunc; // denote current function is pseudo function or not [added for perf reason] + uint8_t isNotNullFunc; // not return null value. + uint8_t scanFlag; // record current running step, default: 0 + int16_t functionId; // function id + char *pOutput; // final result output buffer, point to sdata->data // input parameter, e.g., top(k, 20), the number of results of top query is kept in param SFunctParam *param; // corresponding output buffer for timestamp of each result, e.g., diff/csum @@ -155,6 +207,7 @@ typedef struct SqlFunctionCtx { SSerializeDataHandle saveHandle; int32_t exprIdx; char *udfName; + SFunctionStateStore *pStore; } SqlFunctionCtx; typedef struct tExprNode { diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 42bc89f0b71aeba488e0e27f899c754caf8f1df9..402b8f03092789b2ead705f3bd93b1bc235e78bc 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -158,6 +158,17 @@ typedef enum EFunctionType { FUNCTION_TYPE_STDDEV_PARTIAL, FUNCTION_TYPE_STDDEV_MERGE, + // geometry functions + FUNCTION_TYPE_GEOM_FROM_TEXT = 4250, + FUNCTION_TYPE_AS_TEXT, + FUNCTION_TYPE_MAKE_POINT, + FUNCTION_TYPE_INTERSECTS, + FUNCTION_TYPE_EQUALS, + FUNCTION_TYPE_TOUCHES, + FUNCTION_TYPE_COVERS, + FUNCTION_TYPE_CONTAINS, + FUNCTION_TYPE_CONTAINS_PROPERLY, + // user defined funcion FUNCTION_TYPE_UDF = 10000 } EFunctionType; @@ -205,7 +216,7 @@ bool fmIsUserDefinedFunc(int32_t funcId); bool fmIsDistExecFunc(int32_t funcId); bool fmIsForbidFillFunc(int32_t funcId); bool fmIsForbidStreamFunc(int32_t funcId); -bool fmIsForbidSuperTableFunc(int32_t funcId); +bool fmIsForbidSysTableFunc(int32_t funcId); bool fmIsIntervalInterpoFunc(int32_t funcId); bool fmIsInterpFunc(int32_t funcId); bool fmIsLastRowFunc(int32_t funcId); @@ -222,6 +233,7 @@ bool fmIsGroupKeyFunc(int32_t funcId); bool fmIsBlockDistFunc(int32_t funcId); void getLastCacheDataType(SDataType* pType); +SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList); int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc); @@ -230,6 +242,7 @@ typedef enum EFuncDataRequired { FUNC_DATA_REQUIRED_SMA_LOAD, FUNC_DATA_REQUIRED_NOT_LOAD, FUNC_DATA_REQUIRED_FILTEROUT, + FUNC_DATA_REQUIRED_ALL_FILTEROUT, } EFuncDataRequired; EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h index c825574fa6e3aea8936a3a9e5cd16224956a94a6..995198bc9ea5713f1b5d41070c0fe5583beebaf4 100644 --- a/include/libs/function/taosudf.h +++ b/include/libs/function/taosudf.h @@ -109,7 +109,7 @@ typedef uint16_t VarDataLenT; // maxVarDataLen: 65535 #define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) #define IS_VAR_DATA_TYPE(t) \ - (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON)) + (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY)) #define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) static FORCE_INLINE char *udfColDataGetData(const SUdfColumn *pColumn, int32_t row) { diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index b71d50d43cc59988407576c1c1e0b9c2bce8fa3b..6b15833917cb9bf9fde78363f57740dd4e061647 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -111,6 +111,12 @@ int32_t udfStartUdfd(int32_t startDnodeId); */ int32_t udfStopUdfd(); +/** + * get udfd pid + * + */ + int32_t udfGetUdfdPid(int32_t* pUdfdPid); + #ifdef __cplusplus } #endif diff --git a/include/libs/geometry/geomFunc.h b/include/libs/geometry/geomFunc.h new file mode 100644 index 0000000000000000000000000000000000000000..05627e71f58c7a99e0b444b80c4a4a5ecf4dfd6a --- /dev/null +++ b/include/libs/geometry/geomFunc.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_GEOM_FUNC_H +#define TDENGINE_GEOM_FUNC_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "function.h" + +int32_t makePointFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + +int32_t geomFromTextFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t asTextFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + +int32_t intersectsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t equalsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t touchesFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t coversFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t containsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t containsProperlyFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_GEOM_FUNC_H diff --git a/include/libs/geometry/geosWrapper.h b/include/libs/geometry/geosWrapper.h new file mode 100644 index 0000000000000000000000000000000000000000..a5bc0cec17b5386acb35793abc33e68d8a68be2f --- /dev/null +++ b/include/libs/geometry/geosWrapper.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_GEOS_WRAPPER_H +#define TDENGINE_GEOS_WRAPPER_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include "os.h" + +#include "tgeosctx.h" + +void geosFreeBuffer(void *buffer); + +int32_t initCtxMakePoint(); +int32_t doMakePoint(double x, double y, unsigned char **outputGeom, size_t *size); + +int32_t initCtxGeomFromText(); +int32_t doGeomFromText(const char *inputWKT, unsigned char **outputGeom, size_t *size); + +int32_t initCtxAsText(); +int32_t doAsText(const unsigned char *inputGeom, size_t size, char **outputWKT); + +int32_t initCtxRelationFunc(); +int32_t doIntersects(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, + bool swapped, char *res); +int32_t doEquals(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, + bool swapped, char *res); +int32_t doTouches(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, + bool swapped, char *res); +int32_t doCovers(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, + bool swapped, char *res); +int32_t doContains(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, + bool swapped, char *res); +int32_t doContainsProperly(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, + bool swapped, char *res); + +int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, const GEOSPreparedGeometry **outputPreparedGeom); +void destroyGeometry(GEOSGeometry **geom, const GEOSPreparedGeometry **preparedGeom); + +#ifdef __cplusplus +} +#endif + +#endif /*TDENGINE_GEOS_WRAPPER_H*/ diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 0d31ca2f683ff52b440a131ddf7602b11e53233b..cfcc9993cffb54942fe43b492b1a84f63f9a9d37 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -212,13 +212,38 @@ typedef struct SIndexMetaArg { void* idx; void* ivtIdx; uint64_t suid; + int (*metaFilterFunc)(void* metaEx, void* param, SArray* result); } SIndexMetaArg; +/** + * the underlying storage module must implement this API to employ the index functions. + * @param pMeta + * @param param + * @param results + * @return + */ +typedef struct SMetaFltParam { + uint64_t suid; + int16_t cid; + int16_t type; + void *val; + bool reverse; + bool equal; + int (*filterFunc)(void *a, void *b, int16_t type); +} SMetaFltParam; + +typedef struct SMetaDataFilterAPI { + int32_t (*metaFilterTableIds)(void *pVnode, SMetaFltParam *arg, SArray *pUids); + int32_t (*metaFilterCreateTime)(void *pVnode, SMetaFltParam *arg, SArray *pUids); + int32_t (*metaFilterTableName)(void *pVnode, SMetaFltParam *arg, SArray *pUids); + int32_t (*metaFilterTtl)(void *pVnode, SMetaFltParam *arg, SArray *pUids); +} SMetaDataFilterAPI; + typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus; -SIdxFltStatus idxGetFltStatus(SNode* pFilterNode); +SIdxFltStatus idxGetFltStatus(SNode* pFilterNode, SMetaDataFilterAPI* pAPI); -int32_t doFilterTag(SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* result, SIdxFltStatus* status); +int32_t doFilterTag(SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* result, SIdxFltStatus* status, SMetaDataFilterAPI* pAPI); /* * init index env diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index d899307d74649b9b337a2f42cfe6ee05741ea225..f0a715e6511af931b7e580dae9a83cf7d447e371 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -36,9 +36,10 @@ extern "C" { #define SHOW_CREATE_TB_RESULT_FIELD1_LEN (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) #define SHOW_CREATE_TB_RESULT_FIELD2_LEN (TSDB_MAX_ALLOWED_SQL_LEN * 3) -#define SHOW_LOCAL_VARIABLES_RESULT_COLS 2 +#define SHOW_LOCAL_VARIABLES_RESULT_COLS 3 #define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) +#define SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE) #define SHOW_ALIVE_RESULT_COLS 1 @@ -249,6 +250,7 @@ typedef struct SDropDnodeStmt { char fqdn[TSDB_FQDN_LEN]; int32_t port; bool force; + bool unsafe; } SDropDnodeStmt; typedef struct SAlterDnodeStmt { @@ -318,19 +320,22 @@ typedef struct SIndexOptions { SNode* pInterval; SNode* pOffset; SNode* pSliding; + int8_t tsPrecision; SNode* pStreamOptions; } SIndexOptions; typedef struct SCreateIndexStmt { - ENodeType type; - EIndexType indexType; - bool ignoreExists; - char indexDbName[TSDB_DB_NAME_LEN]; - char indexName[TSDB_INDEX_NAME_LEN]; - char dbName[TSDB_DB_NAME_LEN]; - char tableName[TSDB_TABLE_NAME_LEN]; - SNodeList* pCols; - SIndexOptions* pOptions; + ENodeType type; + EIndexType indexType; + bool ignoreExists; + char indexDbName[TSDB_DB_NAME_LEN]; + char indexName[TSDB_INDEX_NAME_LEN]; + char dbName[TSDB_DB_NAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; + SNodeList* pCols; + SIndexOptions* pOptions; + SNode* pPrevQuery; + SMCreateSmaReq* pReq; } SCreateIndexStmt; typedef struct SDropIndexStmt { @@ -357,23 +362,24 @@ typedef struct SRestoreComponentNodeStmt { typedef struct SCreateTopicStmt { ENodeType type; - char topicName[TSDB_TABLE_NAME_LEN]; + char topicName[TSDB_TOPIC_NAME_LEN]; char subDbName[TSDB_DB_NAME_LEN]; char subSTbName[TSDB_TABLE_NAME_LEN]; bool ignoreExists; - bool withMeta; + int8_t withMeta; SNode* pQuery; + SNode* pWhere; } SCreateTopicStmt; typedef struct SDropTopicStmt { ENodeType type; - char topicName[TSDB_TABLE_NAME_LEN]; + char topicName[TSDB_TOPIC_NAME_LEN]; bool ignoreNotExists; } SDropTopicStmt; typedef struct SDropCGroupStmt { ENodeType type; - char topicName[TSDB_TABLE_NAME_LEN]; + char topicName[TSDB_TOPIC_NAME_LEN]; char cgroup[TSDB_CGROUP_LEN]; bool ignoreNotExists; } SDropCGroupStmt; @@ -423,16 +429,18 @@ typedef struct SStreamOptions { } SStreamOptions; typedef struct SCreateStreamStmt { - ENodeType type; - char streamName[TSDB_TABLE_NAME_LEN]; - char targetDbName[TSDB_DB_NAME_LEN]; - char targetTabName[TSDB_TABLE_NAME_LEN]; - bool ignoreExists; - SStreamOptions* pOptions; - SNode* pQuery; - SNodeList* pTags; - SNode* pSubtable; - SNodeList* pCols; + ENodeType type; + char streamName[TSDB_TABLE_NAME_LEN]; + char targetDbName[TSDB_DB_NAME_LEN]; + char targetTabName[TSDB_TABLE_NAME_LEN]; + bool ignoreExists; + SStreamOptions* pOptions; + SNode* pQuery; + SNode* pPrevQuery; + SNodeList* pTags; + SNode* pSubtable; + SNodeList* pCols; + SCMCreateStreamReq* pReq; } SCreateStreamStmt; typedef struct SDropStreamStmt { diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 8eda9457c6b77027476ce997606a811d5a507c8f..2319643b09f6a93084a527f5f5d6ceb02e6b891e 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -313,6 +313,7 @@ void nodesDestroyAllocator(int64_t allocatorId); SNode* nodesMakeNode(ENodeType type); void nodesDestroyNode(SNode* pNode); +void nodesFree(void* p); SNodeList* nodesMakeList(); int32_t nodesListAppend(SNodeList* pList, SNode* pNode); @@ -327,6 +328,8 @@ void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* p SNode* nodesListGetNode(SNodeList* pList, int32_t index); SListCell* nodesListGetCell(SNodeList* pList, int32_t index); void nodesDestroyList(SNodeList* pList); +bool nodesListMatch(const SNodeList* pList, const SNodeList* pSubList); + // Only clear the linked list structure, without releasing the elements inside void nodesClearList(SNodeList* pList); @@ -345,6 +348,7 @@ void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pCon void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext); bool nodesEqualNode(const SNode* a, const SNode* b); +bool nodesMatchNode(const SNode* pSub, const SNode* pNode); SNode* nodesCloneNode(const SNode* pNode); SNodeList* nodesCloneList(const SNodeList* pList); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 197a5ecaf9b1e13af8d19dfde550f04c83183711..063318332a7cdc0a66c983fdfa464ca8f1bf9a8e 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -53,6 +53,9 @@ typedef struct SLogicNode { EDataOrderLevel requireDataOrder; // requirements for input data EDataOrderLevel resultDataOrder; // properties of the output data EGroupAction groupAction; + EOrder inputTsOrder; + EOrder outputTsOrder; + bool forceCreateNonBlockingOptr; // true if the operator can use non-blocking(pipeline) mode } SLogicNode; typedef enum EScanType { @@ -103,6 +106,7 @@ typedef struct SScanLogicNode { bool hasNormalCols; // neither tag column nor primary key tag column bool sortPrimaryKey; bool igLastNull; + bool groupOrderScan; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -111,7 +115,7 @@ typedef struct SJoinLogicNode { SNode* pMergeCondition; SNode* pOnConditions; bool isSingleTableJoin; - EOrder inputTsOrder; + SNode* pColEqualOnConditions; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -228,8 +232,6 @@ typedef struct SWindowLogicNode { int8_t igExpired; int8_t igCheckUpdate; EWindowAlgorithm windowAlgo; - EOrder inputTsOrder; - EOrder outputTsOrder; } SWindowLogicNode; typedef struct SFillLogicNode { @@ -240,13 +242,14 @@ typedef struct SFillLogicNode { SNode* pWStartTs; SNode* pValues; // SNodeListNode STimeWindow timeRange; - EOrder inputTsOrder; } SFillLogicNode; typedef struct SSortLogicNode { SLogicNode node; SNodeList* pSortKeys; bool groupSort; + int64_t maxRows; + bool skipPKSortOpt; } SSortLogicNode; typedef struct SPartitionLogicNode { @@ -309,12 +312,15 @@ typedef struct SDataBlockDescNode { typedef struct SPhysiNode { ENodeType type; + EOrder inputTsOrder; + EOrder outputTsOrder; SDataBlockDescNode* pOutputDataBlockDesc; SNode* pConditions; SNodeList* pChildren; struct SPhysiNode* pParent; SNode* pLimit; SNode* pSlimit; + bool forceCreateNonBlockingOptr; } SPhysiNode; typedef struct SScanPhysiNode { @@ -325,6 +331,7 @@ typedef struct SScanPhysiNode { uint64_t suid; int8_t tableType; SName tableName; + bool groupOrderScan; } SScanPhysiNode; typedef SScanPhysiNode STagScanPhysiNode; @@ -405,7 +412,7 @@ typedef struct SSortMergeJoinPhysiNode { SNode* pMergeCondition; SNode* pOnConditions; SNodeList* pTargets; - EOrder inputTsOrder; + SNode* pColEqualOnConditions; } SSortMergeJoinPhysiNode; typedef struct SAggPhysiNode { @@ -448,7 +455,7 @@ typedef struct SMergePhysiNode { bool ignoreGroupId; } SMergePhysiNode; -typedef struct SWinodwPhysiNode { +typedef struct SWindowPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of parameter expression of function SNodeList* pFuncs; @@ -458,13 +465,11 @@ typedef struct SWinodwPhysiNode { int64_t watermark; int64_t deleteMark; int8_t igExpired; - EOrder inputTsOrder; - EOrder outputTsOrder; bool mergeDataBlock; -} SWinodwPhysiNode; +} SWindowPhysiNode; typedef struct SIntervalPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; int64_t interval; int64_t offset; int64_t sliding; @@ -486,7 +491,6 @@ typedef struct SFillPhysiNode { SNode* pWStartTs; // SColumnNode SNode* pValues; // SNodeListNode STimeWindow timeRange; - EOrder inputTsOrder; } SFillPhysiNode; typedef SFillPhysiNode SStreamFillPhysiNode; @@ -497,7 +501,7 @@ typedef struct SMultiTableIntervalPhysiNode { } SMultiTableIntervalPhysiNode; typedef struct SSessionWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; int64_t gap; } SSessionWinodwPhysiNode; @@ -506,14 +510,14 @@ typedef SSessionWinodwPhysiNode SStreamSemiSessionWinodwPhysiNode; typedef SSessionWinodwPhysiNode SStreamFinalSessionWinodwPhysiNode; typedef struct SStateWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; SNode* pStateKey; } SStateWinodwPhysiNode; typedef SStateWinodwPhysiNode SStreamStateWinodwPhysiNode; typedef struct SEventWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; SNode* pStartCond; SNode* pEndCond; } SEventWinodwPhysiNode; @@ -615,6 +619,7 @@ typedef struct SQueryPlan { int32_t numOfSubplans; SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. SExplainInfo explainInfo; + void* pPostPlan; } SQueryPlan; const char* dataOrderStr(EDataOrderLevel order); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 9569cfe055fee6c715b3dc921518cc15feb439ad..dc312a762ee9f7b396d6926e685e9e4b46fc15f1 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -52,6 +52,7 @@ typedef struct SExprNode { SArray* pAssociation; bool orderAlias; bool asAlias; + bool asParam; } SExprNode; typedef enum EColumnType { @@ -69,6 +70,7 @@ typedef struct SColumnNode { uint64_t tableId; int8_t tableType; col_id_t colId; + uint16_t projIdx; // the idx in project list, start from 1 EColumnType colType; // column or tag bool hasIndex; char dbName[TSDB_DB_NAME_LEN]; @@ -241,6 +243,12 @@ typedef enum EFillMode { FILL_MODE_NEXT } EFillMode; +typedef enum ETimeLineMode { + TIME_LINE_NONE = 1, + TIME_LINE_MULTI, + TIME_LINE_GLOBAL, +} ETimeLineMode; + typedef struct SFillNode { ENodeType type; // QUERY_NODE_FILL EFillMode mode; @@ -263,50 +271,50 @@ typedef struct SCaseWhenNode { } SCaseWhenNode; typedef struct SSelectStmt { - ENodeType type; // QUERY_NODE_SELECT_STMT - bool isDistinct; - SNodeList* pProjectionList; - SNode* pFromTable; - SNode* pWhere; - SNodeList* pPartitionByList; - SNodeList* pTags; // for create stream - SNode* pSubtable; // for create stream - SNode* pWindow; - SNodeList* pGroupByList; // SGroupingSetNode - SNode* pHaving; - SNode* pRange; - SNode* pEvery; - SNode* pFill; - SNodeList* pOrderByList; // SOrderByExprNode - SLimitNode* pLimit; - SLimitNode* pSlimit; - STimeWindow timeRange; - char stmtName[TSDB_TABLE_NAME_LEN]; - uint8_t precision; - int32_t selectFuncNum; - int32_t returnRows; // EFuncReturnRows - bool isEmptyResult; - bool isTimeLineResult; - bool isSubquery; - bool hasAggFuncs; - bool hasRepeatScanFuncs; - bool hasIndefiniteRowsFunc; - bool hasMultiRowsFunc; - bool hasSelectFunc; - bool hasSelectValFunc; - bool hasOtherVectorFunc; - bool hasUniqueFunc; - bool hasTailFunc; - bool hasInterpFunc; - bool hasInterpPseudoColFunc; - bool hasLastRowFunc; - bool hasLastFunc; - bool hasTimeLineFunc; - bool hasUdaf; - bool hasStateKey; - bool onlyHasKeepOrderFunc; - bool groupSort; - bool tagScan; + ENodeType type; // QUERY_NODE_SELECT_STMT + bool isDistinct; + SNodeList* pProjectionList; + SNode* pFromTable; + SNode* pWhere; + SNodeList* pPartitionByList; + SNodeList* pTags; // for create stream + SNode* pSubtable; // for create stream + SNode* pWindow; + SNodeList* pGroupByList; // SGroupingSetNode + SNode* pHaving; + SNode* pRange; + SNode* pEvery; + SNode* pFill; + SNodeList* pOrderByList; // SOrderByExprNode + SLimitNode* pLimit; + SLimitNode* pSlimit; + STimeWindow timeRange; + char stmtName[TSDB_TABLE_NAME_LEN]; + uint8_t precision; + int32_t selectFuncNum; + int32_t returnRows; // EFuncReturnRows + ETimeLineMode timeLineResMode; + bool isEmptyResult; + bool isSubquery; + bool hasAggFuncs; + bool hasRepeatScanFuncs; + bool hasIndefiniteRowsFunc; + bool hasMultiRowsFunc; + bool hasSelectFunc; + bool hasSelectValFunc; + bool hasOtherVectorFunc; + bool hasUniqueFunc; + bool hasTailFunc; + bool hasInterpFunc; + bool hasInterpPseudoColFunc; + bool hasLastRowFunc; + bool hasLastFunc; + bool hasTimeLineFunc; + bool hasUdaf; + bool hasStateKey; + bool onlyHasKeepOrderFunc; + bool groupSort; + bool tagScan; } SSelectStmt; typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType; @@ -321,6 +329,7 @@ typedef struct SSetOperator { SNode* pLimit; char stmtName[TSDB_TABLE_NAME_LEN]; uint8_t precision; + ETimeLineMode timeLineResMode; } SSetOperator; typedef enum ESqlClause { @@ -434,7 +443,9 @@ typedef struct SQuery { EQueryExecStage execStage; EQueryExecMode execMode; bool haveResultSet; + SNode* pPrevRoot; SNode* pRoot; + SNode* pPostRoot; int32_t numOfResCols; SSchema* pResSchema; int8_t precision; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 94fb6824d2843d3f5f7a13bc3ccdbf96428f8cf3..58bdb77df3d3f6cde847fa1cd7dddfd2096d275f 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -58,6 +58,7 @@ typedef struct SParseContext { bool isSuperUser; bool enableSysInfo; bool async; + bool hasInvisibleCol; const char* svrVer; bool nodeOffline; SArray* pTableMetaPos; // sql table pos => catalog data pos @@ -74,6 +75,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata const struct SMetaData* pMetaData, SQuery* pQuery); int32_t qContinueParseSql(SParseContext* pCxt, struct SCatalogReq* pCatalogReq, const struct SMetaData* pMetaData, SQuery* pQuery); +int32_t qContinueParsePostQuery(SParseContext* pCxt, SQuery* pQuery, void** pResRow); void qDestroyParseContext(SParseContext* pCxt); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 41c0e98084a95f34d3ef6fffb094d47c6a9976dc..1b523c0323915f75757ab739020bc94e7b533167 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -52,6 +52,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo // @groupId id of a group of datasource subplans of this @pSubplan // @pSource one execution location of this group of datasource subplans int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource); +int32_t qContinuePlanPostQuery(void *pPostPlan); void qClearSubplanExecutionNode(SSubplan* pSubplan); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 3841210076bccf0a5f6e552f32892a4502e3b1d7..6a1091e658cdb297c7153357fa020880acb34c4e 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -51,6 +51,12 @@ typedef enum { TARGET_TYPE_OTHER, } ETargetType; +typedef enum { + TCOL_TYPE_COLUMN = 1, + TCOL_TYPE_TAG, + TCOL_TYPE_NONE, +} ETableColumnType; + #define QUERY_POLICY_VNODE 1 #define QUERY_POLICY_HYBRID 2 #define QUERY_POLICY_QNODE 3 @@ -90,28 +96,23 @@ typedef struct STbVerInfo { int32_t tversion; } STbVerInfo; -/* - * ASSERT(sizeof(SCTableMeta) == 24) - * ASSERT(tableType == TSDB_CHILD_TABLE) - * The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info. - */ +#pragma pack(push, 1) typedef struct SCTableMeta { - int32_t vgId : 24; - int8_t tableType; uint64_t uid; uint64_t suid; + int32_t vgId; + int8_t tableType; } SCTableMeta; +#pragma pack(pop) -/* - * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a - * SCTableMeta. - */ + +#pragma pack(push, 1) typedef struct STableMeta { // BEGIN: KEEP THIS PART SAME WITH SCTableMeta - int32_t vgId : 24; - int8_t tableType; uint64_t uid; uint64_t suid; + int32_t vgId; + int8_t tableType; // END: KEEP THIS PART SAME WITH SCTableMeta // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta @@ -121,6 +122,7 @@ typedef struct STableMeta { STableComInfo tableInfo; SSchema schema[]; } STableMeta; +#pragma pack(pop) typedef struct SDBVgInfo { int32_t vgVersion; @@ -130,7 +132,7 @@ typedef struct SDBVgInfo { int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT int64_t stateTs; SHashObj* vgHash; // key:vgId, value:SVgroupInfo - SArray* vgArray; + SArray* vgArray; // SVgroupInfo } SDBVgInfo; typedef struct SUseDbOutput { @@ -257,9 +259,11 @@ void destroyQueryExecRes(SExecResult* pRes); int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len); char* parseTagDatatoJson(void* p); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); +void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst); void freeVgInfo(SDBVgInfo* vgInfo); +void freeDbCfgInfo(SDbCfgInfo *pInfo); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, void* (*mallocFp)(int64_t)); @@ -277,7 +281,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t (_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID)) #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \ ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) -#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER) +#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) \ + ((_code) == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || (_code) == TSDB_CODE_MND_INVALID_SCHEMA_VER) #define NEED_CLIENT_HANDLE_ERROR(_code) \ (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index f20ba287de0ac2ec429ad44107418c8bfe58e0d7..adabe6d67c16953f2204becbf6da123587cb8058 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -41,7 +41,7 @@ typedef struct SFilterColumnParam { } SFilterColumnParam; extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options); -extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, SColumnDataAgg *statis, +extern int32_t filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, SColumnDataAgg *statis, int16_t numOfCols, int32_t *pFilterResStatus); extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param); extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param); diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 5b125b42d4bc67ed6625d38f9baf4fb356ad041e..7747df85956798f6271e06822a4812d03829f89e 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -27,68 +27,65 @@ extern "C" { #endif +#include "storageapi.h" + // void* streamBackendInit(const char* path); // void streamBackendCleanup(void* arg); // SListNode* streamBackendAddCompare(void* backend, void* arg); // void streamBackendDelCompare(void* backend, void* arg); -typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2); - -typedef struct STdbState { - rocksdb_t* rocksdb; - rocksdb_column_family_handle_t** pHandle; - rocksdb_writeoptions_t* writeOpts; - rocksdb_readoptions_t* readOpts; - rocksdb_options_t** cfOpts; - rocksdb_options_t* dbOpt; - struct SStreamTask* pOwner; - void* param; - void* env; - SListNode* pComparNode; - void* pBackendHandle; - char idstr[64]; - void* compactFactory; - - TDB* db; - TTB* pStateDb; - TTB* pFuncStateDb; - TTB* pFillStateDb; // todo refactor - TTB* pSessionStateDb; - TTB* pParNameDb; - TTB* pParTagDb; - TXN* txn; -} STdbState; - -// incremental state storage -typedef struct { - STdbState* pTdbState; - SStreamFileState* pFileState; - int32_t number; - SSHashObj* parNameMap; - int64_t checkPointId; - int32_t taskId; - int64_t streamId; -} SStreamState; - -SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages); + +// <<<<<<< HEAD +// typedef struct STdbState { +// rocksdb_t* rocksdb; +// rocksdb_column_family_handle_t** pHandle; +// rocksdb_writeoptions_t* writeOpts; +// rocksdb_readoptions_t* readOpts; +// rocksdb_options_t** cfOpts; +// rocksdb_options_t* dbOpt; +// struct SStreamTask* pOwner; +// void* param; +// void* env; +// SListNode* pComparNode; +// void* pBackend; +// char idstr[64]; +// void* compactFactory; +// TdThreadRwlock rwLock; +// ======= +// typedef struct STdbState { +// rocksdb_t* rocksdb; +// rocksdb_column_family_handle_t** pHandle; +// rocksdb_writeoptions_t* writeOpts; +// rocksdb_readoptions_t* readOpts; +// rocksdb_options_t** cfOpts; +// rocksdb_options_t* dbOpt; +// struct SStreamTask* pOwner; +// void* param; +// void* env; +// SListNode* pComparNode; +// void* pBackendHandle; +// char idstr[64]; +// void* compactFactory; +// +// TDB* db; +// TTB* pStateDb; +// TTB* pFuncStateDb; +// TTB* pFillStateDb; // todo refactor +// TTB* pSessionStateDb; +// TTB* pParNameDb; +// TTB* pParTagDb; +// TXN* txn; +//} STdbState; +//>>>>>>> enh/dev3.0 + +SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); void streamStateClose(SStreamState* pState, bool remove); int32_t streamStateBegin(SStreamState* pState); int32_t streamStateCommit(SStreamState* pState); void streamStateDestroy(SStreamState* pState, bool remove); int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark); -typedef struct { - rocksdb_iterator_t* iter; - rocksdb_snapshot_t* snapshot; - rocksdb_readoptions_t* readOpt; - rocksdb_t* db; - - TBC* pCur; - int64_t number; -} SStreamStateCur; - -int32_t streamStateFuncPut(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen); -int32_t streamStateFuncGet(SStreamState* pState, const STupleKey* key, void** pVal, int32_t* pVLen); -int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key); +int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); +int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); @@ -120,7 +117,7 @@ int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key); int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal); -void streamFreeVal(void* val); +void streamStateFreeVal(void* val); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key); SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key); @@ -141,6 +138,8 @@ int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname); int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal); +void streamStateReloadInfo(SStreamState* pState, TSKEY ts); + /***compare func **/ typedef struct SStateChekpoint { diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index c7e55650cd7a422b2af9680a79660e64328deeb0..b9b24917f3dcf3a2a31780e1f9980b93da74efd4 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -30,6 +30,7 @@ extern "C" { typedef struct SStreamTask SStreamTask; +#define SSTREAM_TASK_VER 1 enum { STREAM_STATUS__NORMAL = 0, STREAM_STATUS__STOP, @@ -44,11 +45,9 @@ enum { TASK_STATUS__DROPPING, TASK_STATUS__FAIL, TASK_STATUS__STOP, - TASK_STATUS__WAIT_DOWNSTREAM, - TASK_STATUS__RECOVER_PREPARE, - TASK_STATUS__RECOVER1, - TASK_STATUS__RECOVER2, - TASK_STATUS__PAUSE, + TASK_STATUS__SCAN_HISTORY, // stream task scan history data by using tsdbread in the stream scanner + TASK_STATUS__HALT, // pause, but not be manipulated by user command + TASK_STATUS__PAUSE, // pause }; enum { @@ -78,11 +77,11 @@ enum { TASK_TRIGGER_STATUS__ACTIVE, }; -enum { +typedef enum { TASK_LEVEL__SOURCE = 1, TASK_LEVEL__AGG, TASK_LEVEL__SINK, -}; +} ETASK_LEVEL; enum { TASK_OUTPUT__FIXED_DISPATCH = 1, @@ -110,14 +109,14 @@ typedef struct { int64_t ver; int32_t* dataRef; SPackedData submit; -} SStreamDataSubmit2; +} SStreamDataSubmit; typedef struct { int8_t type; int64_t ver; SArray* dataRefs; // SArray SArray* submits; // SArray -} SStreamMergedSubmit2; +} SStreamMergedSubmit; typedef struct { int8_t type; @@ -133,8 +132,6 @@ typedef struct { // ref data block, for delete typedef struct { int8_t type; - int64_t ver; - int32_t* dataRef; SSDataBlock* pBlock; } SStreamRefDataBlock; @@ -204,12 +201,10 @@ static FORCE_INLINE void streamQueueProcessFail(SStreamQueue* queue) { atomic_store_8(&queue->status, STREAM_QUEUE__FAILED); } -void* streamQueueNextItem(SStreamQueue* queue); +void* streamQueueNextItem(SStreamQueue* pQueue); -SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type); -void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit); - -SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit); +SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type); +void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit); typedef struct { char* qmsg; @@ -252,7 +247,7 @@ typedef struct { int8_t reserved; } STaskSinkFetch; -typedef struct { +typedef struct SStreamChildEpInfo { int32_t nodeId; int32_t childId; int32_t taskId; @@ -273,31 +268,63 @@ typedef struct SCheckpointInfo { typedef struct SStreamStatus { int8_t taskStatus; + int8_t downstreamReady; // downstream tasks are all ready now, if this flag is set int8_t schedStatus; int8_t keepTaskStatus; + bool transferState; + int8_t timerActive; // timer is active + int8_t pauseAllowed; // allowed task status to be set to be paused } SStreamStatus; -struct SStreamTask { - SStreamId id; - int32_t totalLevel; - int8_t taskLevel; - int8_t outputType; - int16_t dispatchMsgType; - SStreamStatus status; - int32_t selfChildId; - int32_t nodeId; - SEpSet epSet; - SCheckpointInfo chkInfo; - STaskExec exec; - - // fill history - int8_t fillHistory; - - // children info - SArray* childEpInfo; // SArray - int32_t nextCheckId; - SArray* checkpointInfo; // SArray +typedef struct SHistDataRange { + SVersionRange range; + STimeWindow window; +} SHistDataRange; + +typedef struct SSTaskBasicInfo { + int32_t nodeId; // vgroup id or snode id + SEpSet epSet; + int32_t selfChildId; + int32_t totalLevel; + int8_t taskLevel; + int8_t fillHistory; // is fill history task or not +} SSTaskBasicInfo; + +typedef struct SDispatchMsgInfo { + void* pData; // current dispatch data + int16_t msgType; // dispatch msg type + int32_t retryCount; // retry send data count + int64_t blockingTs; // output blocking timestamp +} SDispatchMsgInfo; +typedef struct { + int8_t type; + int8_t status; + SStreamQueue* queue; +} STaskOutputInfo; + +typedef struct { + int64_t init; + int64_t step1Start; + int64_t step2Start; +} STaskTimestamp; + +struct SStreamTask { + int64_t ver; + SStreamId id; + SSTaskBasicInfo info; + STaskOutputInfo outputInfo; + SDispatchMsgInfo msgInfo; + SStreamStatus status; + SCheckpointInfo chkInfo; + STaskExec exec; + SHistDataRange dataRange; + SStreamId historyTaskId; + SStreamId streamTaskId; + SArray* pUpstreamEpInfoList; // SArray, // children info + int32_t nextCheckId; + SArray* checkpointInfo; // SArray + STaskTimestamp tsInfo; // output union { STaskDispatcherFixedEp fixedEpDispatcher; @@ -308,51 +335,56 @@ struct SStreamTask { }; int8_t inputStatus; - int8_t outputStatus; SStreamQueue* inputQueue; - SStreamQueue* outputQueue; // trigger int8_t triggerStatus; int64_t triggerParam; - void* timer; + void* schedTimer; + void* launchTaskTimer; SMsgCb* pMsgCb; // msg handle SStreamState* pState; // state backend + SArray* pRspMsgList; + TdThreadMutex lock; // the followings attributes don't be serialized - int32_t recoverTryingDownstream; - int32_t recoverWaitingUpstream; + int32_t notReadyTasks; + int32_t numOfWaitingUpstream; int64_t checkReqId; SArray* checkReqIds; // shuffle int32_t refCnt; int64_t checkpointingId; int32_t checkpointAlignCnt; + int32_t transferStateAlignCnt; struct SStreamMeta* pMeta; + SSHashObj* pNameMap; }; // meta typedef struct SStreamMeta { - char* path; - TDB* db; - TTB* pTaskDb; - TTB* pCheckpointDb; - SHashObj* pTasks; - SArray* pTaskList; // SArray - void* ahandle; - TXN* txn; - FTaskExpand* expandFunc; - int32_t vgId; - SRWLatch lock; - int32_t walScanCounter; - void* streamBackend; - int32_t streamBackendId; - int64_t streamBackendRid; + char* path; + TDB* db; + TTB* pTaskDb; + TTB* pCheckpointDb; + SHashObj* pTasks; + SArray* pTaskList; // SArray + void* ahandle; + TXN* txn; + FTaskExpand* expandFunc; + int32_t vgId; + SRWLatch lock; + int32_t walScanCounter; + void* streamBackend; + int64_t streamBackendRid; + SHashObj* pTaskBackendUnique; + TdThreadMutex backendMutex; } SStreamMeta; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo); -SStreamTask* tNewStreamTask(int64_t streamId); +SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory, int64_t triggerParam, + SArray* pTaskList); int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); void tFreeStreamTask(SStreamTask* pTask); @@ -373,6 +405,7 @@ typedef struct { int32_t upstreamChildId; int32_t upstreamNodeId; int32_t blockNum; + int64_t totalLen; SArray* dataLen; // SArray SArray* data; // SArray } SStreamDispatchReq; @@ -429,16 +462,19 @@ typedef struct { SMsgHead msgHead; int64_t streamId; int32_t taskId; -} SStreamRecoverStep1Req, SStreamRecoverStep2Req; + int8_t igUntreated; +} SStreamScanHistoryReq; typedef struct { int64_t streamId; - int32_t taskId; + int32_t upstreamTaskId; + int32_t downstreamTaskId; + int32_t upstreamNodeId; int32_t childId; -} SStreamRecoverFinishReq; +} SStreamScanHistoryFinishReq, SStreamTransferReq; -int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq); -int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishReq* pReq); +int32_t tEncodeStreamScanHistoryFinishReq(SEncoder* pEncoder, const SStreamScanHistoryFinishReq* pReq); +int32_t tDecodeStreamScanHistoryFinishReq(SDecoder* pDecoder, SStreamScanHistoryFinishReq* pReq); typedef struct { int64_t streamId; @@ -494,6 +530,17 @@ int32_t tDecodeSStreamCheckpointReq(SDecoder* pDecoder, SStreamCheckpointReq* pR int32_t tEncodeSStreamCheckpointRsp(SEncoder* pEncoder, const SStreamCheckpointRsp* pRsp); int32_t tDecodeSStreamCheckpointRsp(SDecoder* pDecoder, SStreamCheckpointRsp* pRsp); +typedef struct { + int64_t streamId; + int32_t upstreamTaskId; + int32_t upstreamNodeId; + int32_t downstreamId; + int32_t downstreamNode; +} SStreamCompleteHistoryMsg; + +int32_t tEncodeCompleteHistoryDataMsg(SEncoder* pEncoder, const SStreamCompleteHistoryMsg* pReq); +int32_t tDecodeCompleteHistoryDataMsg(SDecoder* pDecoder, SStreamCompleteHistoryMsg* pReq); + typedef struct { int64_t streamId; int32_t downstreamTaskId; @@ -507,14 +554,14 @@ typedef struct { SArray* checkpointVer; // SArray } SStreamRecoverDownstreamRsp; -int32_t tEncodeSStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq); -int32_t tDecodeSStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq); +int32_t tEncodeStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq); +int32_t tDecodeStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq); -int32_t tEncodeSStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp); -int32_t tDecodeSStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp); +int32_t tEncodeStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp); +int32_t tDecodeStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp); -int32_t tEncodeSStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq); -int32_t tDecodeSStreamTaskRecoverReq(SDecoder* pDecoder, SStreamRecoverDownstreamReq* pReq); +int32_t tEncodeSStreamTaskScanHistoryReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq); +int32_t tDecodeSStreamTaskScanHistoryReq(SDecoder* pDecoder, SStreamRecoverDownstreamReq* pReq); int32_t tEncodeSStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamRecoverDownstreamRsp* pRsp); int32_t tDecodeSStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamRecoverDownstreamRsp* pRsp); @@ -523,65 +570,87 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq); void tDeleteStreamRetrieveReq(SStreamRetrieveReq* pReq); -void tDeleteStreamDispatchReq(SStreamDispatchReq* pReq); +int32_t tInitStreamDispatchReq(SStreamDispatchReq* pReq, const SStreamTask* pTask, int32_t vgId, int32_t numOfBlocks, + int64_t dstTaskId); +void tDeleteStreamDispatchReq(SStreamDispatchReq* pReq); -int32_t streamSetupTrigger(SStreamTask* pTask); +int32_t streamSetupScheduleTrigger(SStreamTask* pTask); int32_t streamProcessRunReq(SStreamTask* pTask); -int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg, bool exec); +int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg, bool exec); int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code); int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg); -// int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); void streamTaskInputFail(SStreamTask* pTask); int32_t streamTryExec(SStreamTask* pTask); int32_t streamSchedExec(SStreamTask* pTask); -int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock); +int32_t streamTaskOutputResultBlock(SStreamTask* pTask, SStreamDataBlock* pBlock); bool streamTaskShouldStop(const SStreamStatus* pStatus); bool streamTaskShouldPause(const SStreamStatus* pStatus); +bool streamTaskIsIdle(const SStreamTask* pTask); +int32_t streamTaskEndScanWAL(SStreamTask* pTask); + +SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId); +int32_t streamScanExec(SStreamTask* pTask, int32_t batchSize); -int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz); +char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); // recover and fill history -int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version); -int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version); +void streamTaskCheckDownstreamTasks(SStreamTask* pTask); +int32_t streamTaskDoCheckDownstreamTasks(SStreamTask* pTask); +int32_t streamTaskLaunchScanHistory(SStreamTask* pTask); int32_t streamTaskCheckStatus(SStreamTask* pTask); -int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version); +int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp, + SRpcHandleInfo* pRpcInfo, int32_t taskId); +int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp); +int32_t streamLaunchFillHistoryTask(SStreamTask* pTask); +int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask); +int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated); +bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer); // common -int32_t streamSetParamForRecover(SStreamTask* pTask); -int32_t streamRestoreParam(SStreamTask* pTask); -int32_t streamSetStatusNormal(SStreamTask* pTask); +int32_t streamRestoreParam(SStreamTask* pTask); +int32_t streamSetStatusNormal(SStreamTask* pTask); +const char* streamGetTaskStatusStr(int32_t status); +void streamTaskPause(SStreamTask* pTask); +void streamTaskResume(SStreamTask* pTask); +void streamTaskHalt(SStreamTask* pTask); +void streamTaskResumeFromHalt(SStreamTask* pTask); +void streamTaskDisablePause(SStreamTask* pTask); +void streamTaskEnablePause(SStreamTask* pTask); + // source level -int32_t streamSourceRecoverPrepareStep1(SStreamTask* pTask, int64_t ver); -int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamRecoverStep1Req* pReq); -int32_t streamSourceRecoverScanStep1(SStreamTask* pTask); -int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req* pReq); -int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver); -int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask); +int32_t streamSetParamForStreamScannerStep1(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow); +int32_t streamSetParamForStreamScannerStep2(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow); +int32_t streamSourceScanHistoryData(SStreamTask* pTask); +int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask); + +int32_t streamDispatchTransferStateMsg(SStreamTask* pTask); + // agg level -int32_t streamAggRecoverPrepare(SStreamTask* pTask); -// int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask); -int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId); +int32_t streamTaskScanHistoryPrepare(SStreamTask* pTask); +int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, + SRpcHandleInfo* pRpcInfo); +int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask); +// stream task meta void streamMetaInit(); void streamMetaCleanup(); SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId); void streamMetaClose(SStreamMeta* streamMeta); -int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); -int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); -int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen); -int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta); - -SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId); +// save to b-tree meta store +int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); +int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); +int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded); +int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); +int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); // todo remove it +SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); -void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); int32_t streamMetaBegin(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta); -int32_t streamMetaRollBack(SStreamMeta* pMeta); int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver); // checkpoint @@ -589,6 +658,10 @@ int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, int32_t streamProcessCheckpointReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointReq* pReq); int32_t streamProcessCheckpointRsp(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointRsp* pRsp); +int32_t streamTaskReleaseState(SStreamTask* pTask); +int32_t streamTaskReloadState(SStreamTask* pTask); +int32_t streamAlignTransferState(SStreamTask* pTask); + #ifdef __cplusplus } #endif diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index d50f0e0a312017cd76ee2b9aee52c74b0f7c5f65..b2255013ca5de8246f94eecd13dae4e382c1cd32 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -21,25 +21,17 @@ #include "tarray.h" #include "tdef.h" #include "tlist.h" +#include "storageapi.h" #ifdef __cplusplus extern "C" { #endif typedef struct SStreamFileState SStreamFileState; -typedef struct SRowBuffPos { - void* pRowBuff; - void* pKey; - bool beFlushed; - bool beUsed; -} SRowBuffPos; - typedef SList SStreamSnapshot; -typedef TSKEY (*GetTsFun)(void*); - -SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, GetTsFun fp, void* pFile, - TSKEY delMark); +SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, + GetTsFun fp, void* pFile, TSKEY delMark, const char* id); void streamFileStateDestroy(SStreamFileState* pFileState); void streamFileStateClear(SStreamFileState* pFileState); bool needClearDiskBuff(SStreamFileState* pFileState); @@ -56,6 +48,8 @@ int32_t recoverSnapshot(SStreamFileState* pFileState); int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list); int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark); +int32_t streamFileStateGeSelectRowSize(SStreamFileState* pFileState); +void streamFileStateReloadInfo(SStreamFileState* pFileState, TSKEY ts); #ifdef __cplusplus } diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 4678aa0bd9630beff4eb53111dbe5221325d1ac9..7bb1d027c9da6539708a755b52f23b87a10beea8 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -19,7 +19,7 @@ #include "tarray.h" #include "tcommon.h" #include "tmsg.h" -#include "tscalablebf.h" +#include "storageapi.h" #ifdef __cplusplus extern "C" { @@ -30,18 +30,18 @@ typedef struct SUpdateKey { TSKEY ts; } SUpdateKey; -typedef struct SUpdateInfo { - SArray *pTsBuckets; - uint64_t numBuckets; - SArray *pTsSBFs; - uint64_t numSBFs; - int64_t interval; - int64_t watermark; - TSKEY minTS; - SScalableBf *pCloseWinSBF; - SHashObj *pMap; - uint64_t maxDataVersion; -} SUpdateInfo; +//typedef struct SUpdateInfo { +// SArray *pTsBuckets; +// uint64_t numBuckets; +// SArray *pTsSBFs; +// uint64_t numSBFs; +// int64_t interval; +// int64_t watermark; +// TSKEY minTS; +// SScalableBf *pCloseWinSBF; +// SHashObj *pMap; +// uint64_t maxDataVersion; +//} SUpdateInfo; SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark); SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark); @@ -53,6 +53,8 @@ void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo); void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo); +void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count); +void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count); #ifdef __cplusplus } diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index e86a4f96905512dd08af6acc4b9563206b93f985..88ccf562c72eb346fd7901865f9d44229e082c74 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -154,14 +154,14 @@ typedef struct SSnapshotMeta { typedef struct SSyncFSM { void* data; - int32_t (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); + int32_t (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, SFsmCbMeta* pMeta); SyncIndex (*FpAppliedIndexCb)(const struct SSyncFSM* pFsm); - int32_t (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); - void (*FpRollBackCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); + int32_t (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, SFsmCbMeta* pMeta); + void (*FpRollBackCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, SFsmCbMeta* pMeta); void (*FpRestoreFinishCb)(const struct SSyncFSM* pFsm, const SyncIndex commitIdx); - void (*FpReConfigCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SReConfigCbMeta* pMeta); - void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); + void (*FpReConfigCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, SReConfigCbMeta* pMeta); + void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, SFsmCbMeta* pMeta); bool (*FpApplyQueueEmptyCb)(const struct SSyncFSM* pFsm); int32_t (*FpApplyQueueItems)(const struct SSyncFSM* pFsm); @@ -239,29 +239,31 @@ typedef struct SSyncState { ESyncState state; bool restored; bool canRead; + SyncTerm term; + int64_t roleTimeMs; } SSyncState; -int32_t syncInit(); -void syncCleanUp(); -int64_t syncOpen(SSyncInfo* pSyncInfo); -int32_t syncStart(int64_t rid); -void syncStop(int64_t rid); -void syncPreStop(int64_t rid); -void syncPostStop(int64_t rid); -int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq); -int32_t syncIsCatchUp(int64_t rid); +int32_t syncInit(); +void syncCleanUp(); +int64_t syncOpen(SSyncInfo* pSyncInfo); +int32_t syncStart(int64_t rid); +void syncStop(int64_t rid); +void syncPreStop(int64_t rid); +void syncPostStop(int64_t rid); +int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq); +int32_t syncIsCatchUp(int64_t rid); ESyncRole syncGetRole(int64_t rid); -int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); -int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg); -int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex); -int32_t syncEndSnapshot(int64_t rid); -int32_t syncLeaderTransfer(int64_t rid); -int32_t syncStepDown(int64_t rid, SyncTerm newTerm); -bool syncIsReadyForRead(int64_t rid); -bool syncSnapshotSending(int64_t rid); -bool syncSnapshotRecving(int64_t rid); -int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq); -int32_t syncForceBecomeFollower(SSyncNode* ths, const SRpcMsg* pRpcMsg); +int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); +int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg); +int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex); +int32_t syncEndSnapshot(int64_t rid); +int32_t syncLeaderTransfer(int64_t rid); +int32_t syncStepDown(int64_t rid, SyncTerm newTerm); +bool syncIsReadyForRead(int64_t rid); +bool syncSnapshotSending(int64_t rid); +bool syncSnapshotRecving(int64_t rid); +int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq); +int32_t syncForceBecomeFollower(SSyncNode* ths, const SRpcMsg* pRpcMsg); SSyncState syncGetState(int64_t rid); void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 622cd615b8e14a82d1eb940986cb70ad9a086992..2b90e3226c8b5e9519590873d280952aca0ec4a4 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -69,6 +69,13 @@ void tfsUpdateSize(STfs *pTfs); */ SDiskSize tfsGetSize(STfs *pTfs); +/** + * @brief Get the number of disks at level of multi-tier storage. + * + * @param pTfs + * @return int32_t + */ +int32_t tfsGetDisksAtLevel(STfs *pTfs, int32_t level); /** * @brief Get level of multi-tier storage. * @@ -123,6 +130,15 @@ int32_t tfsMkdir(STfs *pTfs, const char *rname); */ int32_t tfsMkdirAt(STfs *pTfs, const char *rname, SDiskID diskId); +/** + * @brief Recursive make directory at all levels in tfs. + * + * @param pTfs The fs object. + * @param rname The rel name of directory. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t tfsMkdirRecur(STfs *pTfs, const char *rname); + /** * @brief Recursive create directories in tfs. * @@ -160,7 +176,17 @@ int32_t tfsRmdir(STfs *pTfs, const char *rname); * @param nrname The rel name of new file. * @return int32_t 0 for success, -1 for failure. */ -int32_t tfsRename(STfs *pTfs, const char *orname, const char *nrname); +int32_t tfsRename(STfs *pTfs, int32_t diskPrimary, const char *orname, const char *nrname); + +/** + * @brief Search fname in level of tfs + * + * @param pTfs The fs object. + * @param level The level to search on + * @param fname The relative file name to be searched + * @param int32_t diskId for successs, -1 for failure + */ +int32_t tfsSearch(STfs *pTfs, int32_t level, const char *fname); /** * @brief Init file object in tfs. @@ -274,6 +300,25 @@ void tfsClosedir(STfsDir *pDir); */ int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo); +/** + * @brief Check if disk space available at level + * + * @param pTfs The fs object. + * #param level the level + * @return bool + */ +bool tfsDiskSpaceAvailable(STfs *pTfs, int32_t level); + +/** + * @brief Check if disk space sufficient at disk of level + * + * @param pTfs The fs object. + * @param level the level + * @param disk the disk + * @return bool + */ +bool tfsDiskSpaceSufficient(STfs *pTfs, int32_t level, int32_t disk); + #ifdef __cplusplus } #endif diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index c73e5c127af03cdc07e47c1ca95dd775a38599f2..e5955aad5449b955d3d3b3a7470692f3ad5b7feb 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -46,6 +46,7 @@ typedef struct SRpcHandleInfo { int8_t noResp; // has response or not(default 0, 0: resp, 1: no resp) int8_t persistHandle; // persist handle or not int8_t hasEpSet; + int32_t cliVer; // app info void *ahandle; // app handle set by client @@ -83,11 +84,12 @@ typedef struct SRpcInit { int32_t sessions; // number of sessions allowed int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS int32_t idleTime; // milliseconds, 0 means idle timer is disabled + int32_t compatibilityVer; int32_t retryMinInterval; // retry init interval int32_t retryStepFactor; // retry interval factor int32_t retryMaxInterval; // retry max interval - int64_t retryMaxTimouet; + int64_t retryMaxTimeout; int32_t failFastThreshold; int32_t failFastInterval; diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 1e2e22b0a531081016df0b5a68dd3ca408bc5eb5..b19a0d783d2df9e70b9b21d0b5321ec98df35880 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -135,25 +135,27 @@ typedef struct { int8_t scanUncommited; int8_t scanNotApplied; int8_t scanMeta; + int8_t deleteMsg; int8_t enableRef; } SWalFilterCond; typedef struct SWalReader SWalReader; // todo hide this struct -typedef struct SWalReader { +struct SWalReader { SWal *pWal; int64_t readerId; TdFilePtr pLogFile; TdFilePtr pIdxFile; int64_t curFileFirstVer; int64_t curVersion; + int64_t skipToVersion; // skip data and jump to destination version, usually used by stream resume ignoring untreated data int64_t capacity; TdThreadMutex mutex; SWalFilterCond cond; // TODO remove it SWalCkHead *pHead; -} SWalReader; +}; // module initialization int32_t walInit(); @@ -199,7 +201,10 @@ int32_t walReaderSeekVer(SWalReader *pRead, int64_t ver); int32_t walNextValidMsg(SWalReader *pRead); int64_t walReaderGetCurrentVer(const SWalReader *pReader); int64_t walReaderGetValidFirstVer(const SWalReader *pReader); +int64_t walReaderGetSkipToVersion(SWalReader *pReader); +void walReaderSetSkipToVersion(SWalReader *pReader, int64_t ver); void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64_t *ever); +void walReaderVerifyOffset(SWalReader *pWalReader, STqOffsetVal* pOffset); // only for tq usage void walSetReaderCapacity(SWalReader *pRead, int32_t capacity); @@ -207,8 +212,8 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead); int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead); int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead); -SWalRef *walRefFirstVer(SWal *, SWalRef *); -SWalRef *walRefCommittedVer(SWal *); +void walRefFirstVer(SWal *, SWalRef *); +void walRefLastVer(SWal *, SWalRef *); SWalRef *walOpenRef(SWal *); void walCloseRef(SWal *pWal, int64_t refId); diff --git a/include/os/os.h b/include/os/os.h index 309a977ff6b7e6d500d93cba0ee487cc7befbd9e..ac1a750b78cc01109821a39f24da5d4453abf8d8 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -53,6 +53,7 @@ extern "C" { #else #include #include +#include #if defined(_TD_X86_) #include #endif diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 18cd0d9cc60e7f1affc51bf2b98bfb23f7ad8b7e..683d10e926d88a196cf91354e99764b3e0bc92b9 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -22,21 +22,20 @@ extern "C" { // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following sectio -// #if !defined(WINDOWS) +#if !defined(WINDOWS) -// #ifndef ALLOW_FORBID_FUNC -// #define malloc MALLOC_FUNC_TAOS_FORBID -// #define calloc CALLOC_FUNC_TAOS_FORBID -// #define realloc REALLOC_FUNC_TAOS_FORBID -// #define free FREE_FUNC_TAOS_FORBID -// #ifdef strdup -// #undef strdup -// #define strdup STRDUP_FUNC_TAOS_FORBID -// #endif -// #endif // ifndef ALLOW_FORBID_FUNC -// #endif // if !defined(WINDOWS) +#ifndef ALLOW_FORBID_FUNC +#define malloc MALLOC_FUNC_TAOS_FORBID +#define calloc CALLOC_FUNC_TAOS_FORBID +#define realloc REALLOC_FUNC_TAOS_FORBID +#define free FREE_FUNC_TAOS_FORBID +#ifdef strdup +#undef strdup +#define strdup STRDUP_FUNC_TAOS_FORBID +#endif +#endif // ifndef ALLOW_FORBID_FUNC +#endif // if !defined(WINDOWS) -// // #define taosMemoryFree malloc // #define taosMemoryMalloc malloc // #define taosMemoryCalloc calloc // #define taosMemoryRealloc realloc diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index a8ccb67bfb1bb345150a2d7ee90d776e7b251aa3..a6a3655a5516748a8cb75ac453c21948636888a9 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -35,8 +35,9 @@ typedef struct { bool taosCheckSystemIsLittleEnd(); void taosGetSystemInfo(); +int64_t taosGetOsUptime(); int32_t taosGetEmail(char *email, int32_t maxLen); -int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen); +int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t maxLen); int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores); int32_t taosGetCpuCores(float *numOfCores); void taosGetCpuUsage(double *cpu_system, double *cpu_engine); diff --git a/include/os/osTime.h b/include/os/osTime.h index 51a285a1393171ea59940498a4c8134afa270366..87df3a26503b2ff710f32e79a658742ace24badd 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -95,6 +95,8 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf); struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst); time_t taosTime(time_t *t); time_t taosMktime(struct tm *timep); +int64_t user_mktime64(const uint32_t year, const uint32_t mon, const uint32_t day, const uint32_t hour, + const uint32_t min, const uint32_t sec, int64_t time_zone); #ifdef __cplusplus } diff --git a/include/util/talgo.h b/include/util/talgo.h index f9d51c4b5bc68b988bf562bcdf7452a3dd1208a5..7c92c0fe87c5a198441db4412519755c8cdd528d 100644 --- a/include/util/talgo.h +++ b/include/util/talgo.h @@ -31,7 +31,7 @@ typedef void *(*__array_item_dup_fn_t)(void *); typedef void (*FDelete)(void *); typedef int32_t (*FEncode)(void **buf, const void *dst); -typedef void *(*FDecode)(const void *buf, void *dst); +typedef void *(*FDecode)(const void *buf, void *dst, int8_t sver); #define TD_EQ 0x1 #define TD_GT 0x2 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 8bc6ed2ac87c461b1dc291a1ea88fc7f3d6185b4..b43985074c7d88673f80dac9b02607a66d114b20 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -66,8 +66,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018) // #define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019) // #define TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED TAOS_DEF_ERROR_CODE(0, 0x0020) // "Vgroup could not be connected" -#define TSDB_CODE_RPC_SOMENODE_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0021) // -#define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x0022) // +#define TSDB_CODE_RPC_SOMENODE_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0021) // +#define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x0022) // @@ -277,7 +277,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x0378) #define TSDB_CODE_MND_INVALID_FUNC_RETRIEVE TAOS_DEF_ERROR_CODE(0, 0x0379) - + // mnode-db #define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) @@ -288,9 +288,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOO_MANY_DATABASES TAOS_DEF_ERROR_CODE(0, 0x0385) #define TSDB_CODE_MND_DB_IN_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0386) // // #define TSDB_CODE_MND_VGROUP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0387) // 2.x -#define TSDB_CODE_MND_DB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0388) // +#define TSDB_CODE_MND_DB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0388) // #define TSDB_CODE_MND_INVALID_DB_ACCT TAOS_DEF_ERROR_CODE(0, 0x0389) // internal -#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) // +#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) // #define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x038B) #define TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO TAOS_DEF_ERROR_CODE(0, 0x038C) // #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) // 2.x @@ -345,7 +345,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_CLOG_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x03D4) #define TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL TAOS_DEF_ERROR_CODE(0, 0x03D5) #define TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x03D6) //internal -#define TSDB_CODE_MND_TRNAS_SYNC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x03D7) +#define TSDB_CODE_MND_TRANS_SYNC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x03D7) #define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03DF) // mnode-mq @@ -408,13 +408,15 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MNODE_NOT_CATCH_UP TAOS_DEF_ERROR_CODE(0, 0x0412) // internal #define TSDB_CODE_MNODE_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0413) // internal #define TSDB_CODE_MNODE_ONLY_TWO_MNODE TAOS_DEF_ERROR_CODE(0, 0x0414) // internal +#define TSDB_CODE_MNODE_NO_NEED_RESTORE TAOS_DEF_ERROR_CODE(0, 0x0415) // internal +#define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0416) // vnode // #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x // #define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) // 2.x // #define TSDB_CODE_VND_ACTION_NEED_REPROCESS. TAOS_DEF_ERROR_CODE(0, 0x0502) // 2.x #define TSDB_CODE_VND_INVALID_VGROUP_ID TAOS_DEF_ERROR_CODE(0, 0x0503) -// #define TSDB_CODE_VND_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0504) // 2.x +#define TSDB_CODE_VND_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0504) // #define TSDB_CODE_VND_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0505) // 2.x // #define TSDB_CODE_VND_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0506) // 2.x // #define TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR TAOS_DEF_ERROR_CODE(0, 0x0507) // 2.x @@ -445,6 +447,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_NOT_CATCH_UP TAOS_DEF_ERROR_CODE(0, 0x0532) // internal #define TSDB_CODE_VND_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0533) // internal #define TSDB_CODE_VND_DIR_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0534) +#define TSDB_CODE_VND_META_DATA_UNSAFE_DELETE TAOS_DEF_ERROR_CODE(0, 0x0535) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -513,6 +516,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_JSON_IN_GROUP_ERROR TAOS_DEF_ERROR_CODE(0, 0x072E) #define TSDB_CODE_QRY_JOB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x072F) #define TSDB_CODE_QRY_QWORKER_QUIT TAOS_DEF_ERROR_CODE(0, 0x0730) +#define TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR TAOS_DEF_ERROR_CODE(0, 0x0731) // grant #define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) @@ -701,6 +705,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_TAGS_PC TAOS_DEF_ERROR_CODE(0, 0x2665) #define TSDB_CODE_PAR_INVALID_TIMELINE_QUERY TAOS_DEF_ERROR_CODE(0, 0x2666) #define TSDB_CODE_PAR_INVALID_OPTR_USAGE TAOS_DEF_ERROR_CODE(0, 0x2667) +#define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2668) +#define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED TAOS_DEF_ERROR_CODE(0, 0x2669) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner @@ -751,19 +757,30 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RSMA_INVALID_SCHEMA TAOS_DEF_ERROR_CODE(0, 0x3153) #define TSDB_CODE_RSMA_STREAM_STATE_OPEN TAOS_DEF_ERROR_CODE(0, 0x3154) #define TSDB_CODE_RSMA_STREAM_STATE_COMMIT TAOS_DEF_ERROR_CODE(0, 0x3155) -#define TSDB_CODE_RSMA_FS_REF TAOS_DEF_ERROR_CODE(0, 0x3156) -#define TSDB_CODE_RSMA_FS_SYNC TAOS_DEF_ERROR_CODE(0, 0x3157) -#define TSDB_CODE_RSMA_FS_UPDATE TAOS_DEF_ERROR_CODE(0, 0x3158) +#define TSDB_CODE_RSMA_FS_SYNC TAOS_DEF_ERROR_CODE(0, 0x3156) +#define TSDB_CODE_RSMA_RESULT TAOS_DEF_ERROR_CODE(0, 0x3157) //index #define TSDB_CODE_INDEX_REBUILDING TAOS_DEF_ERROR_CODE(0, 0x3200) #define TSDB_CODE_INDEX_INVALID_FILE TAOS_DEF_ERROR_CODE(0, 0x3201) +//scalar +#define TSDB_CODE_SCALAR_CONVERT_ERROR TAOS_DEF_ERROR_CODE(0, 0x3250) + //tmq #define TSDB_CODE_TMQ_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x4000) #define TSDB_CODE_TMQ_CONSUMER_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x4001) #define TSDB_CODE_TMQ_CONSUMER_CLOSED TAOS_DEF_ERROR_CODE(0, 0x4002) #define TSDB_CODE_TMQ_CONSUMER_ERROR TAOS_DEF_ERROR_CODE(0, 0x4003) +#define TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x4004) +#define TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x4005) +#define TSDB_CODE_TMQ_SNAPSHOT_ERROR TAOS_DEF_ERROR_CODE(0, 0x4006) +#define TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x4007) +#define TSDB_CODE_TMQ_INVALID_VGID TAOS_DEF_ERROR_CODE(0, 0x4008) +#define TSDB_CODE_TMQ_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x4009) +#define TSDB_CODE_TMQ_NEED_INITIALIZED TAOS_DEF_ERROR_CODE(0, 0x4010) +#define TSDB_CODE_TMQ_NO_COMMITTED TAOS_DEF_ERROR_CODE(0, 0x4011) +#define TSDB_CODE_TMQ_SAME_COMMITTED_VALUE TAOS_DEF_ERROR_CODE(0, 0x4012) // stream #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) @@ -774,7 +791,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDLITE_IVLD_OPEN_DIR TAOS_DEF_ERROR_CODE(0, 0x5101) // UTIL -#define TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x6000) +#define TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x6000) #ifdef __cplusplus } diff --git a/include/util/tarray.h b/include/util/tarray.h index 4bf24b46b9d913c00d11b5ef53f8630f3ad4dbf5..f56c9e3a1761303961aa9e946e7063049a48baf9 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -22,7 +22,7 @@ extern "C" { #endif -#define TARRAY_MIN_SIZE 8 +#define TARRAY_MIN_SIZE 4 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) #define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) @@ -138,7 +138,7 @@ size_t taosArrayGetSize(const SArray* pArray); * @param index * @param pData */ -void* taosArrayInsert(SArray* pArray, size_t index, void* pData); +void* taosArrayInsert(SArray* pArray, size_t index, const void* pData); /** * set data in array @@ -204,9 +204,9 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)); void* taosArrayDestroy(SArray* pArray); -void taosArrayDestroyP(SArray* pArray, FDelete fp); +void taosArrayDestroyP(SArray* pArray, FDelete fp); -void taosArrayDestroyEx(SArray* pArray, FDelete fp); +void taosArrayDestroyEx(SArray* pArray, FDelete fp); void taosArraySwap(SArray* a, SArray* b); @@ -244,7 +244,7 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param); int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode); -void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz); +void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz, int8_t sver); #ifdef __cplusplus } diff --git a/include/util/tarray2.h b/include/util/tarray2.h new file mode 100644 index 0000000000000000000000000000000000000000..cd49e647895e1571b6b70ad46408420d6972a4b7 --- /dev/null +++ b/include/util/tarray2.h @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "talgo.h" + +#ifndef _TD_UTIL_TARRAY2_H_ +#define _TD_UTIL_TARRAY2_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// a: a +// e: element +// ep: element pointer +// cmp: compare function +// idx: index +// cb: callback function + +#define TARRAY2(TYPE) \ + struct { \ + int32_t size; \ + int32_t capacity; \ + TYPE *data; \ + } + +typedef void (*TArray2Cb)(void *); + +#define TARRAY2_SIZE(a) ((a)->size) +#define TARRAY2_CAPACITY(a) ((a)->capacity) +#define TARRAY2_DATA(a) ((a)->data) +#define TARRAY2_GET(a, i) ((a)->data[i]) +#define TARRAY2_GET_PTR(a, i) ((a)->data + i) +#define TARRAY2_FIRST(a) ((a)->data[0]) +#define TARRAY2_LAST(a) ((a)->data[(a)->size - 1]) +#define TARRAY2_DATA_LEN(a) ((a)->size * sizeof(((a)->data[0]))) + +static FORCE_INLINE int32_t tarray2_make_room(void *arr, int32_t expSize, int32_t eleSize) { + TARRAY2(void) *a = arr; + + int32_t capacity = (a->capacity > 0) ? (a->capacity << 1) : 32; + while (capacity < expSize) { + capacity <<= 1; + } + void *p = taosMemoryRealloc(a->data, capacity * eleSize); + if (p == NULL) return TSDB_CODE_OUT_OF_MEMORY; + a->capacity = capacity; + a->data = p; + return 0; +} + +static FORCE_INLINE int32_t tarray2InsertBatch(void *arr, int32_t idx, const void *elePtr, int32_t numEle, + int32_t eleSize) { + TARRAY2(uint8_t) *a = arr; + + int32_t ret = 0; + if (a->size + numEle > a->capacity) { + ret = tarray2_make_room(a, a->size + numEle, eleSize); + } + if (ret == 0) { + if (idx < a->size) { + memmove(a->data + (idx + numEle) * eleSize, a->data + idx * eleSize, (a->size - idx) * eleSize); + } + memcpy(a->data + idx * eleSize, elePtr, numEle * eleSize); + a->size += numEle; + } + return ret; +} + +static FORCE_INLINE void *tarray2Search(void *arr, const void *elePtr, int32_t eleSize, __compar_fn_t compar, + int32_t flag) { + TARRAY2(void) *a = arr; + return taosbsearch(elePtr, a->data, a->size, eleSize, compar, flag); +} + +static FORCE_INLINE int32_t tarray2SearchIdx(void *arr, const void *elePtr, int32_t eleSize, __compar_fn_t compar, + int32_t flag) { + TARRAY2(void) *a = arr; + void *p = taosbsearch(elePtr, a->data, a->size, eleSize, compar, flag); + if (p == NULL) { + return -1; + } else { + return (int32_t)(((uint8_t *)p - (uint8_t *)a->data) / eleSize); + } +} + +static FORCE_INLINE int32_t tarray2SortInsert(void *arr, const void *elePtr, int32_t eleSize, __compar_fn_t compar) { + TARRAY2(void) *a = arr; + int32_t idx = tarray2SearchIdx(arr, elePtr, eleSize, compar, TD_GT); + return tarray2InsertBatch(arr, idx < 0 ? a->size : idx, elePtr, 1, eleSize); +} + +#define TARRAY2_INIT_EX(a, size_, capacity_, data_) \ + do { \ + (a)->size = (size_); \ + (a)->capacity = (capacity_); \ + (a)->data = (data_); \ + } while (0) + +#define TARRAY2_INIT(a) TARRAY2_INIT_EX(a, 0, 0, NULL) + +#define TARRAY2_CLEAR(a, cb) \ + do { \ + if ((cb) && (a)->size > 0) { \ + TArray2Cb cb_ = (TArray2Cb)(cb); \ + for (int32_t i = 0; i < (a)->size; ++i) { \ + cb_((a)->data + i); \ + } \ + } \ + (a)->size = 0; \ + } while (0) + +#define TARRAY2_DESTROY(a, cb) \ + do { \ + TARRAY2_CLEAR(a, cb); \ + if ((a)->data) { \ + taosMemoryFree((a)->data); \ + (a)->data = NULL; \ + } \ + (a)->capacity = 0; \ + } while (0) + +#define TARRAY2_INSERT_PTR(a, idx, ep) tarray2InsertBatch(a, idx, ep, 1, sizeof((a)->data[0])) +#define TARRAY2_APPEND_PTR(a, ep) tarray2InsertBatch(a, (a)->size, ep, 1, sizeof((a)->data[0])) +#define TARRAY2_APPEND_BATCH(a, ep, n) tarray2InsertBatch(a, (a)->size, ep, n, sizeof((a)->data[0])) +#define TARRAY2_APPEND(a, e) TARRAY2_APPEND_PTR(a, &(e)) + +// return (TYPE *) +#define TARRAY2_SEARCH(a, ep, cmp, flag) tarray2Search(a, ep, sizeof(((a)->data[0])), (__compar_fn_t)cmp, flag) + +#define TARRAY2_SEARCH_IDX(a, ep, cmp, flag) tarray2SearchIdx(a, ep, sizeof(((a)->data[0])), (__compar_fn_t)cmp, flag) + +#define TARRAY2_SORT_INSERT(a, e, cmp) tarray2SortInsert(a, &(e), sizeof(((a)->data[0])), (__compar_fn_t)cmp) +#define TARRAY2_SORT_INSERT_P(a, ep, cmp) tarray2SortInsert(a, ep, sizeof(((a)->data[0])), (__compar_fn_t)cmp) + +#define TARRAY2_REMOVE(a, idx, cb) \ + do { \ + if ((idx) < (a)->size) { \ + if (cb) { \ + TArray2Cb cb_ = (TArray2Cb)(cb); \ + cb_((a)->data + (idx)); \ + } \ + if ((idx) < (a)->size - 1) { \ + memmove((a)->data + (idx), (a)->data + (idx) + 1, sizeof((*(a)->data)) * ((a)->size - (idx)-1)); \ + } \ + (a)->size--; \ + } \ + } while (0) + +#define TARRAY2_FOREACH(a, e) for (int32_t __i = 0; __i < (a)->size && ((e) = (a)->data[__i], 1); __i++) +#define TARRAY2_FOREACH_REVERSE(a, e) for (int32_t __i = (a)->size - 1; __i >= 0 && ((e) = (a)->data[__i], 1); __i--) +#define TARRAY2_FOREACH_PTR(a, ep) for (int32_t __i = 0; __i < (a)->size && ((ep) = &(a)->data[__i], 1); __i++) +#define TARRAY2_FOREACH_PTR_REVERSE(a, ep) \ + for (int32_t __i = (a)->size - 1; __i >= 0 && ((ep) = &(a)->data[__i], 1); __i--) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_TARRAY2_H_*/ diff --git a/include/util/tcompare.h b/include/util/tcompare.h index f92e1c3970a828fdfe109ee51a8e1f52f1ae0389..2fa736f4df73b0e8661484151e9b22c929e62b9c 100644 --- a/include/util/tcompare.h +++ b/include/util/tcompare.h @@ -79,6 +79,7 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight); int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight); int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight); +int32_t compareLenBinaryVal(const void *pLeft, const void *pRight); int32_t comparestrRegexMatch(const void *pLeft, const void *pRight); int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight); diff --git a/include/util/tconfig.h b/include/util/tconfig.h index ca3c5eb0ebd35867464c5c02851a9761e111d250..ed8eedb7e574fd7f5e682f45eebb57df3328ee31 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -50,11 +50,17 @@ typedef enum { CFG_DTYPE_TIMEZONE } ECfgDataType; +typedef enum { + CFG_SCOPE_SERVER, + CFG_SCOPE_CLIENT, + CFG_SCOPE_BOTH +} ECfgScopeType; + typedef struct SConfigItem { ECfgSrcType stype; ECfgDataType dtype; - bool tsc; - char *name; + int8_t scope; + char *name; union { bool bval; float fval; @@ -92,20 +98,21 @@ int32_t cfgGetSize(SConfig *pCfg); SConfigItem *cfgGetItem(SConfig *pCfg, const char *name); int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype); -int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, bool tsc); -int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, bool tsc); -int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, bool tsc); -int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, bool tsc); -int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc); -int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc); -int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal); -int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal); -int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope); +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope); +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope); +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, int8_t scope); +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); const char *cfgStypeStr(ECfgSrcType type); const char *cfgDtypeStr(ECfgDataType type); void cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); +void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); diff --git a/include/util/tdef.h b/include/util/tdef.h index 427a49fd4e039c596051db64e7990f13bd69f004..e4af88bf10291235e07ae7b18674fa064b054683 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -32,7 +32,7 @@ extern "C" { #define TD_VER_MAX UINT64_MAX // TODO: use the real max version from query handle // Bytes for each type. -extern const int32_t TYPE_BYTES[16]; +extern const int32_t TYPE_BYTES[21]; // TODO: replace and remove code below #define CHAR_BYTES sizeof(char) @@ -53,10 +53,11 @@ extern const int32_t TYPE_BYTES[16]; #define TSDB_DATA_BIGINT_NULL 0x8000000000000000LL #define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL -#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN -#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000LL // an NAN -#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF -#define TSDB_DATA_BINARY_NULL 0xFF +#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000LL // an NAN +#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF +#define TSDB_DATA_BINARY_NULL 0xFF +#define TSDB_DATA_GEOMETRY_NULL 0xFF #define TSDB_DATA_UTINYINT_NULL 0xFF #define TSDB_DATA_USMALLINT_NULL 0xFFFF @@ -190,15 +191,16 @@ typedef enum ELogicConditionType { #define TSDB_MAX_COLUMNS 4096 #define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns -#define TSDB_NODE_NAME_LEN 64 -#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string -#define TSDB_CGROUP_LEN 193 // it is a null-terminated string -#define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string -#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string -#define TSDB_DB_NAME_LEN 65 -#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) -#define TSDB_PRIVILEDGE_CONDITION_LEN 200 +#define TSDB_NODE_NAME_LEN 64 +#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string +#define TSDB_CGROUP_LEN 193 // it is a null-terminated string +#define TSDB_OFFSET_LEN 64 // it is a null-terminated string +#define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string +#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string +#define TSDB_DB_NAME_LEN 65 +#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_PRIVILEDGE_CONDITION_LEN 48*1024 #define TSDB_FUNC_NAME_LEN 65 #define TSDB_FUNC_COMMENT_LEN 1024 * 1024 @@ -247,14 +249,15 @@ typedef enum ELogicConditionType { #define TSDB_LABEL_LEN 8 #define TSDB_JOB_STATUS_LEN 32 -#define TSDB_CLUSTER_ID_LEN 40 -#define TSDB_FQDN_LEN 128 -#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) -#define TSDB_IPv4ADDR_LEN 16 -#define TSDB_FILENAME_LEN 128 -#define TSDB_SHOW_SQL_LEN 2048 -#define TSDB_SLOW_QUERY_SQL_LEN 512 -#define TSDB_SHOW_SUBQUERY_LEN 1000 +#define TSDB_CLUSTER_ID_LEN 40 +#define TSDB_FQDN_LEN 128 +#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) +#define TSDB_IPv4ADDR_LEN 16 +#define TSDB_FILENAME_LEN 128 +#define TSDB_SHOW_SQL_LEN 2048 +#define TSDB_SHOW_SCHEMA_JSON_LEN TSDB_MAX_COLUMNS * 256 +#define TSDB_SLOW_QUERY_SQL_LEN 512 +#define TSDB_SHOW_SUBQUERY_LEN 1000 #define TSDB_TRANS_STAGE_LEN 12 #define TSDB_TRANS_TYPE_LEN 16 @@ -267,8 +270,8 @@ typedef enum ELogicConditionType { #define TSDB_DNODE_CONFIG_LEN 128 #define TSDB_DNODE_VALUE_LEN 256 -#define TSDB_ACTIVE_KEY_LEN 109 // history 109:? -#define TSDB_CONN_ACTIVE_KEY_LEN 257 // history 257:? +#define TSDB_ACTIVE_KEY_LEN 109 +#define TSDB_CONN_ACTIVE_KEY_LEN 255 #define TSDB_DEFAULT_PKT_SIZE 65480 // same as RPC_MAX_UDP_SIZE @@ -366,8 +369,13 @@ typedef enum ELogicConditionType { #define TSDB_DB_SCHEMALESS_OFF 0 #define TSDB_DEFAULT_DB_SCHEMALESS TSDB_DB_SCHEMALESS_OFF #define TSDB_MIN_STT_TRIGGER 1 -#define TSDB_MAX_STT_TRIGGER 16 -#define TSDB_DEFAULT_SST_TRIGGER 1 +#ifdef TD_ENTERPRISE +#define TSDB_MAX_STT_TRIGGER 16 +#define TSDB_DEFAULT_SST_TRIGGER 2 +#else +#define TSDB_MAX_STT_TRIGGER 1 +#define TSDB_DEFAULT_SST_TRIGGER 1 +#endif #define TSDB_MIN_HASH_PREFIX (2 - TSDB_TABLE_NAME_LEN) #define TSDB_MAX_HASH_PREFIX (TSDB_TABLE_NAME_LEN - 2) #define TSDB_DEFAULT_HASH_PREFIX 0 @@ -376,8 +384,8 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_HASH_SUFFIX 0 #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1 -#define TSDB_REP_DEF_DB_WAL_RET_PERIOD 0 -#define TSDB_REPS_DEF_DB_WAL_RET_PERIOD 0 +#define TSDB_REP_DEF_DB_WAL_RET_PERIOD 3600 +#define TSDB_REPS_DEF_DB_WAL_RET_PERIOD 3600 #define TSDB_DB_MIN_WAL_RETENTION_SIZE -1 #define TSDB_REP_DEF_DB_WAL_RET_SIZE 0 #define TSDB_REPS_DEF_DB_WAL_RET_SIZE 0 @@ -407,9 +415,11 @@ typedef enum ELogicConditionType { #define TSDB_EXPLAIN_RESULT_ROW_SIZE (16 * 1024) #define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN" -#define TSDB_MAX_FIELD_LEN 65519 // 16384:65519 -#define TSDB_MAX_BINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 -#define TSDB_MAX_NCHAR_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 +#define TSDB_MAX_FIELD_LEN 65519 // 16384:65519 +#define TSDB_MAX_BINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 +#define TSDB_MAX_NCHAR_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 +#define TSDB_MAX_GEOMETRY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 + #define PRIMARYKEY_TIMESTAMP_COL_ID 1 #define COL_REACH_END(colId, maxColId) ((colId) > (maxColId)) @@ -487,6 +497,7 @@ enum { #define TSDB_CONFIG_OPTION_LEN 32 #define TSDB_CONFIG_VALUE_LEN 64 +#define TSDB_CONFIG_SCOPE_LEN 8 #define TSDB_CONFIG_NUMBER 8 #define QUERY_ID_SIZE 20 diff --git a/include/util/tgeosctx.h b/include/util/tgeosctx.h new file mode 100644 index 0000000000000000000000000000000000000000..b5ac5a9333611c074edd89277b7fcf1d86ba21a6 --- /dev/null +++ b/include/util/tgeosctx.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_GEOS_CTX_H_ +#define _TD_UTIL_GEOS_CTX_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +typedef struct SGeosContext { + GEOSContextHandle_t handle; + + GEOSWKTReader *WKTReader; + GEOSWKTWriter *WKTWriter; + + GEOSWKBReader *WKBReader; + GEOSWKBWriter *WKBWriter; + + char errMsg[512]; +} SGeosContext; + +SGeosContext* getThreadLocalGeosCtx(); +void destroyThreadLocalGeosCtx(); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_GEOS_CTX_H_*/ diff --git a/include/util/theap.h b/include/util/theap.h index fb5ff8301a9b08b2cb53c353a363042c2de9cc34..b795db6aea921b64e584eefc9a87018cfa38ff28 100644 --- a/include/util/theap.h +++ b/include/util/theap.h @@ -17,6 +17,7 @@ #define _TD_UTIL_HEAP_H_ #include "os.h" +#include "tarray.h" #ifdef __cplusplus extern "C" { @@ -58,6 +59,54 @@ void heapDequeue(Heap* heap); size_t heapSize(Heap* heap); +typedef bool (*pq_comp_fn)(void* l, void* r, void* param); + +typedef struct PriorityQueueNode { + void* data; +} PriorityQueueNode; + +typedef struct PriorityQueue PriorityQueue; + +PriorityQueue* createPriorityQueue(pq_comp_fn fn, FDelete deleteFn, void* param); + +void taosPQSetFn(PriorityQueue* pq, pq_comp_fn fn); + +void destroyPriorityQueue(PriorityQueue* pq); + +PriorityQueueNode* taosPQTop(PriorityQueue* pq); + +size_t taosPQSize(PriorityQueue* pq); + +PriorityQueueNode* taosPQPush(PriorityQueue* pq, const PriorityQueueNode* node); + +void taosPQPop(PriorityQueue* pq); + +typedef struct BoundedQueue BoundedQueue; + +BoundedQueue* createBoundedQueue(uint32_t maxSize, pq_comp_fn fn, FDelete deleteFn, void* param); + +void taosBQSetFn(BoundedQueue* q, pq_comp_fn fn); + +void destroyBoundedQueue(BoundedQueue* q); + +/* + * Push one node into BQ + * @retval NULL if n is upper than top node in q, and n is not freed + * @retval the pushed Node if pushing succeeded + * @note if maxSize exceeded, the original highest node is popped and freed with deleteFn + * */ +PriorityQueueNode* taosBQPush(BoundedQueue* q, PriorityQueueNode* n); + +PriorityQueueNode* taosBQTop(BoundedQueue* q); + +size_t taosBQSize(BoundedQueue* q); + +size_t taosBQMaxSize(BoundedQueue* q); + +void taosBQBuildHeap(BoundedQueue* q); + +void taosBQPop(BoundedQueue* q); + #ifdef __cplusplus } #endif diff --git a/include/util/tlist.h b/include/util/tlist.h index c684e90a3318444b706720dee0b64ba220437aee..0924c133b9c77b5717ff30386f49fe63c9e506dd 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -241,6 +241,54 @@ void tdListNodeGetData(SList *list, SListNode *node, void *target); void tdListInitIter(SList *list, SListIter *pIter, TD_LIST_DIRECTION_T direction); SListNode *tdListNext(SListIter *pIter); +// macros ==================================================================================== + +// q: for queue +// n: for node +// m: for member + +#define LISTD(TYPE) \ + struct { \ + TYPE *next, *prev; \ + } + +#define LISTD_NEXT(n, m) ((n)->m.next) +#define LISTD_PREV(n, m) ((n)->m.prev) +#define LISTD_INIT(q, m) (LISTD_NEXT(q, m) = LISTD_PREV(q, m) = (q)) +#define LISTD_HEAD(q, m) (LISTD_NEXT(q, m)) +#define LISTD_TAIL(q, m) (LISTD_PREV(q, m)) +#define LISTD_PREV_NEXT(n, m) (LISTD_NEXT(LISTD_PREV(n, m), m)) +#define LISTD_NEXT_PREV(n, m) (LISTD_PREV(LISTD_NEXT(n, m), m)) + +#define LISTD_INSERT_HEAD(q, n, m) \ + do { \ + LISTD_NEXT(n, m) = LISTD_NEXT(q, m); \ + LISTD_PREV(n, m) = (q); \ + LISTD_NEXT_PREV(n, m) = (n); \ + LISTD_NEXT(q, m) = (n); \ + } while (0) + +#define LISTD_INSERT_TAIL(q, n, m) \ + do { \ + LISTD_NEXT(n, m) = (q); \ + LISTD_PREV(n, m) = LISTD_PREV(q, m); \ + LISTD_PREV_NEXT(n, m) = (n); \ + LISTD_PREV(q, m) = (n); \ + } while (0) + +#define LISTD_REMOVE(n, m) \ + do { \ + LISTD_PREV_NEXT(n, m) = LISTD_NEXT(n, m); \ + LISTD_NEXT_PREV(n, m) = LISTD_PREV(n, m); \ + } while (0) + +#define LISTD_FOREACH(q, n, m) for ((n) = LISTD_HEAD(q, m); (n) != (q); (n) = LISTD_NEXT(n, m)) +#define LISTD_FOREACH_REVERSE(q, n, m) for ((n) = LISTD_TAIL(q, m); (n) != (q); (n) = LISTD_PREV(n, m)) +#define LISTD_FOREACH_SAFE(q, n, t, m) \ + for ((n) = LISTD_HEAD(q, m), (t) = LISTD_NEXT(n, m); (n) != (q); (n) = (t), (t) = LISTD_NEXT(n, m)) +#define LISTD_FOREACH_REVERSE_SAFE(q, n, t, m) \ + for ((n) = LISTD_TAIL(q, m), (t) = LISTD_PREV(n, m); (n) != (q); (n) = (t), (t) = LISTD_PREV(n, m)) + #ifdef __cplusplus } #endif diff --git a/include/util/tlosertree.h b/include/util/tlosertree.h index 51906443f5ab874a2e7b16a11304ed6890f90437..b3aa37a537fda3aab2241b263b085624d0b6464a 100644 --- a/include/util/tlosertree.h +++ b/include/util/tlosertree.h @@ -43,7 +43,7 @@ typedef struct SMultiwayMergeTreeInfo { int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo **pTree, uint32_t numOfEntries, void *param, __merge_compare_fn_t compareFn); -void tMergeTreeDestroy(SMultiwayMergeTreeInfo *pTree); +void tMergeTreeDestroy(SMultiwayMergeTreeInfo **pTree); void tMergeTreeAdjust(SMultiwayMergeTreeInfo *pTree, int32_t idx); diff --git a/include/util/tlrucache.h b/include/util/tlrucache.h index c9cf71c2fd99398bd3f5e55bd08127f4278ed1f4..e5e59d0876e1f892dbe25fb03674d5612a04f9d7 100644 --- a/include/util/tlrucache.h +++ b/include/util/tlrucache.h @@ -24,7 +24,8 @@ extern "C" { typedef struct SLRUCache SLRUCache; -typedef void (*_taos_lru_deleter_t)(const void *key, size_t keyLen, void *value); +typedef void (*_taos_lru_deleter_t)(const void *key, size_t keyLen, void *value, void *ud); +typedef int (*_taos_lru_functor_t)(const void *key, size_t keyLen, void *value, void *ud); typedef struct LRUHandle LRUHandle; @@ -41,10 +42,11 @@ SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoo void taosLRUCacheCleanup(SLRUCache *cache); LRUStatus taosLRUCacheInsert(SLRUCache *cache, const void *key, size_t keyLen, void *value, size_t charge, - _taos_lru_deleter_t deleter, LRUHandle **handle, LRUPriority priority); + _taos_lru_deleter_t deleter, LRUHandle **handle, LRUPriority priority, void *ud); LRUHandle *taosLRUCacheLookup(SLRUCache *cache, const void *key, size_t keyLen); void taosLRUCacheErase(SLRUCache *cache, const void *key, size_t keyLen); +void taosLRUCacheApply(SLRUCache *cache, _taos_lru_functor_t functor, void *ud); void taosLRUCacheEraseUnrefEntries(SLRUCache *cache); bool taosLRUCacheRef(SLRUCache *cache, LRUHandle *handle); diff --git a/include/util/trbtree.h b/include/util/trbtree.h index e2264194401bee65a22f0301f0339cb76a8d7356..8353a91f0acc3b5ced557ef594572f7db53c64fd 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -39,7 +39,7 @@ void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z); SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey); SRBTreeNode *tRBTreeDropMin(SRBTree *pTree); SRBTreeNode *tRBTreeDropMax(SRBTree *pTree); -SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode); +SRBTreeNode *tRBTreeGet(const SRBTree *pTree, const SRBTreeNode *pKeyNode); // SRBTreeIter ============================================= #define tRBTreeIterCreate(tree, ascend) \ @@ -67,9 +67,9 @@ struct SRBTree { }; struct SRBTreeIter { - int8_t asc; - SRBTree *pTree; - SRBTreeNode *pNode; + int8_t asc; + const SRBTree *pTree; + SRBTreeNode *pNode; }; #ifdef __cplusplus diff --git a/include/util/tutil.h b/include/util/tutil.h index 7a59aa170aa0f0011ec55cbb5507047c6c5d2a21..a2cfa4cfe56f0c5a6021327f6461bde812ebc46e 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -29,7 +29,7 @@ extern "C" { int32_t strdequote(char *src); size_t strtrim(char *src); char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote); -TdUcs4* wcsnchr(const TdUcs4* haystack, TdUcs4 needle, size_t len); +TdUcs4 *wcsnchr(const TdUcs4 *haystack, TdUcs4 needle, size_t len); char **strsplit(char *src, const char *delim, int32_t *num); char *strtolower(char *dst, const char *src); @@ -37,11 +37,11 @@ char *strntolower(char *dst, const char *src, int32_t n); char *strntolower_s(char *dst, const char *src, int32_t n); int64_t strnatoi(char *num, int32_t len); -size_t tstrncspn(const char *str, size_t ssize, const char *reject, size_t rsize); -size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize); +size_t tstrncspn(const char *str, size_t ssize, const char *reject, size_t rsize); +size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize); -char *strbetween(char *string, char *begin, char *end); -char *paGetToken(char *src, char **token, int32_t *tokenLen); +char *strbetween(char *string, char *begin, char *end); +char *paGetToken(char *src, char **token, int32_t *tokenLen); int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]); int32_t taosHexStrToByteArray(char hexstr[], char bytes[]); @@ -81,12 +81,13 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix, int32_t suffix) { - if ((prefix == 0 && suffix == 0) || (tblen <= (prefix + suffix)) || (tblen <= -1 * (prefix + suffix)) || prefix * suffix < 0) { + if ((prefix == 0 && suffix == 0) || (tblen <= (prefix + suffix)) || (tblen <= -1 * (prefix + suffix)) || + prefix * suffix < 0) { return MurmurHash3_32(tbname, tblen); } else if (prefix > 0 || suffix > 0) { return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix); } else { - char tbName[TSDB_TABLE_FNAME_LEN]; + char tbName[TSDB_TABLE_FNAME_LEN]; int32_t offset = 0; if (prefix < 0) { offset = -1 * prefix; @@ -94,20 +95,33 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } if (suffix < 0) { strncpy(tbName + offset, tbname + tblen + suffix, -1 * suffix); - offset += -1 *suffix; + offset += -1 * suffix; } return MurmurHash3_32(tbName, offset); } } #define TSDB_CHECK_CODE(CODE, LINO, LABEL) \ - if (CODE) { \ - LINO = __LINE__; \ - goto LABEL; \ + do { \ + if ((CODE)) { \ + LINO = __LINE__; \ + goto LABEL; \ + } \ + } while (0) + +#define TSDB_CHECK_NULL(ptr, CODE, LINO, LABEL, ERRNO) \ + if ((ptr) == NULL) { \ + (CODE) = (ERRNO); \ + (LINO) = __LINE__; \ + goto LABEL; \ } +#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) + #define VND_CHECK_CODE(CODE, LINO, LABEL) TSDB_CHECK_CODE(CODE, LINO, LABEL) +#define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr)-offsetof(type, member))) + #ifdef __cplusplus } #endif diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 2159899aa277f550adf76eb7c107bf3da17c9ab1..236cf1f520b903c4d35bef5d51e5108e91cb9d42 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -108,6 +108,9 @@ # time period of keeping log files, in days # logKeepDays 0 +# unit Hour. Latency of data migration +# keepTimeOffset 0 + ############ 3. Debug Flag and levels ############################################# diff --git a/packaging/checkPackageRuning.py b/packaging/checkPackageRuning.py index 2edeeb6dbbb682bb06150e30803a7f05c170a5b1..914ee83f29dfaf118285ea19efdb609168846ff6 100755 --- a/packaging/checkPackageRuning.py +++ b/packaging/checkPackageRuning.py @@ -42,8 +42,8 @@ else: # os.system("rm -rf /var/lib/taos/*") # os.system("systemctl restart taosd ") -# wait a moment ,at least 5 seconds -time.sleep(5) +# wait a moment ,at least 10 seconds +time.sleep(10) # prepare data by taosBenchmark @@ -87,7 +87,7 @@ os.system("rm -rf /tmp/dumpdata/*") # dump data out print("taosdump dump out data") -os.system("taosdump -o /tmp/dumpdata -D test -y -h %s "%serverHost) +os.system("taosdump -o /tmp/dumpdata -D test -h %s "%serverHost) # drop database of test print("drop database test") @@ -95,7 +95,7 @@ os.system(" taos -s ' drop database test ;' -h %s "%serverHost) # dump data in print("taosdump dump data in") -os.system("taosdump -i /tmp/dumpdata -y -h %s "%serverHost) +os.system("taosdump -i /tmp/dumpdata -h %s "%serverHost) result = conn.query("SELECT count(*) from test.meters") diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 9f49cf345a4dd9e36f048f03bd49a28539baec66..07819159c4a531556072be150f91c4bd8a353954 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -124,12 +124,12 @@ if [ -f ${compile_dir}/build/bin/jemalloc-config ]; then cp ${compile_dir}/build/lib/libjemalloc.so.2 ${pkg_dir}${install_user_local_path}/lib/ ln -sf libjemalloc.so.2 ${pkg_dir}${install_user_local_path}/lib/libjemalloc.so fi - if [ -f ${compile_dir}/build/lib/libjemalloc.a ]; then - cp ${compile_dir}/build/lib/libjemalloc.a ${pkg_dir}${install_user_local_path}/lib/ - fi - if [ -f ${compile_dir}/build/lib/libjemalloc_pic.a ]; then - cp ${compile_dir}/build/lib/libjemalloc_pic.a ${pkg_dir}${install_user_local_path}/lib/ - fi + # if [ -f ${compile_dir}/build/lib/libjemalloc.a ]; then + # cp ${compile_dir}/build/lib/libjemalloc.a ${pkg_dir}${install_user_local_path}/lib/ + # fi + # if [ -f ${compile_dir}/build/lib/libjemalloc_pic.a ]; then + # cp ${compile_dir}/build/lib/libjemalloc_pic.a ${pkg_dir}${install_user_local_path}/lib/ + # fi if [ -f ${compile_dir}/build/lib/pkgconfig/jemalloc.pc ]; then cp ${compile_dir}/build/lib/pkgconfig/jemalloc.pc ${pkg_dir}${install_user_local_path}/lib/pkgconfig/ fi diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh index 4b6fc8576b233192a6835d455d73baf14abc1758..ee0985417666333254951f8674cb797577c5035f 100755 --- a/packaging/docker/dockerbuild.sh +++ b/packaging/docker/dockerbuild.sh @@ -123,6 +123,16 @@ else echo "Unknown cpuType: ${cpuType}" exit 1 fi +# check the tdengine cloud base image existed or not +if [ "$cloudBuild" == "y" ]; then + CloudBase=$(docker images | grep tdengine/tdengine-cloud-base ||:) + if [[ "$CloudBase" == "" ]]; then + echo "Rebuild tdengine cloud base image..." + docker build --rm -f "${communityDir}/packaging/docker/DockerfileCloud.base" -t tdengine/tdengine-cloud-base "." --build-arg cpuType=${cpuTypeAlias} + else + echo "Already found tdengine cloud base image" + fi +fi docker build --rm -f "${Dockerfile}" --network=host -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} --build-arg cpuType=${cpuTypeAlias} if [ "$cloudBuild" != "y" ]; then diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 52d5335003097787a9d607ed4f020eb9153ceb31..846d17e7f66bbe41a100e23abe35fd885d2d84dd 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -123,12 +123,12 @@ if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{homepath}/jemalloc/lib ln -sf libjemalloc.so.2 %{buildroot}%{homepath}/jemalloc/lib/libjemalloc.so fi - if [ -f %{_compiledir}/build/lib/libjemalloc.a ]; then - cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{homepath}/jemalloc/lib - fi - if [ -f %{_compiledir}/build/lib/libjemalloc_pic.a ]; then - cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{homepath}/jemalloc/lib - fi +# if [ -f %{_compiledir}/build/lib/libjemalloc.a ]; then +# cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{homepath}/jemalloc/lib +# fi +# if [ -f %{_compiledir}/build/lib/libjemalloc_pic.a ]; then +# cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{homepath}/jemalloc/lib +# fi if [ -f %{_compiledir}/build/lib/pkgconfig/jemalloc.pc ]; then cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{homepath}/jemalloc/lib/pkgconfig fi diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index 081383f89b358325f55cc1a7641015efcb0a4eed..0622b01f2b6c03e26e1b5968f208baf96b0e786e 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -152,7 +152,7 @@ function wgetFile { file=$1 versionPath=$2 sourceP=$3 -nasServerIP="192.168.1.131" +nasServerIP="192.168.1.213" packagePath="/nas/TDengine/v${versionPath}/${verMode}" if [ -f ${file} ];then echoColor YD "${file} already exists ,it will delete it and download it again " diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 1b47b10520147664a3c2f3a558fed1208f84a2ca..f9a11f5540110b796217867a8442625c7d93f2c8 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -315,13 +315,13 @@ function install_jemalloc() { ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so ${csudo}/usr/bin/install -c -d /usr/local/lib - if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then - ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib - fi - if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then - ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib - fi - if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + # if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + # ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + # fi + # if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + # ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + # fi + if [ -f ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc ]; then ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig fi @@ -619,7 +619,6 @@ function install_web() { fi } - function clean_service_on_sysvinit() { if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then ${csudo}service ${serverName2} stop || : @@ -905,27 +904,29 @@ function updateProduct() { openresty_work=false echo - echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}" + echo -e "${GREEN_DARK}To configure ${productName2} ${NC}\t: edit ${cfg_install_dir}/${configFile2}" [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml" + echo -e "${GREEN_DARK}To configure ${clientName2}Adapter ${NC}\t: edit ${configDir}/${clientName2}adapter.toml" if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName2}${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t: ${csudo}systemctl start ${serverName2}${NC}" [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}systemctl start ${clientName2}adapter ${NC}" elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName2} start${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t: ${csudo}service ${serverName2} start${NC}" [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}service ${clientName2}adapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ./${serverName2}${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t: ./${serverName2}${NC}" [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${clientName2}adapter &${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${clientName2}adapter ${NC}" fi + + echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}\t: sudo systemctl enable ${clientName2}keeper ${NC}" if [ ${openresty_work} = 'true' ]; then - echo -e "${GREEN_DARK}To access ${productName2} ${NC}: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}" + echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}" else - echo -e "${GREEN_DARK}To access ${productName2} ${NC}: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell${NC}" + echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell${NC}" fi if ((${prompt_force} == 1)); then @@ -934,6 +935,7 @@ function updateProduct() { fi echo echo -e "\033[44;32;1m${productName2} is updated successfully!${NC}" + echo -e "\033[44;32;1mTo manage ${productName2} instance, view documentation and explorer features, you need to install ${clientName2}Explorer ${NC}" else install_bin install_config @@ -971,9 +973,8 @@ function installProduct() { if [ "$verMode" == "cluster" ]; then install_connector fi - install_examples + install_examples install_web - if [ -z $1 ]; then # install service and client # For installing new install_bin @@ -987,23 +988,25 @@ function installProduct() { # Ask if to start the service echo - echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}" + echo -e "${GREEN_DARK}To configure ${productName2} ${NC}\t: edit ${cfg_install_dir}/${configFile2}" [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml" + echo -e "${GREEN_DARK}To configure ${clientName2}Adapter ${NC}\t: edit ${configDir}/${clientName2}adapter.toml" if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName2}${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t: ${csudo}systemctl start ${serverName2}${NC}" [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}systemctl start ${clientName2}adapter ${NC}" elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName2} start${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t: ${csudo}service ${serverName2} start${NC}" [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}service ${clientName2}adapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${serverName2}${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t: ${serverName2}${NC}" [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${clientName2}adapter &${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${clientName2}adapter ${NC}" fi + echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}\t: sudo systemctl enable ${clientName2}keeper ${NC}" + if [ ! -z "$firstEp" ]; then tmpFqdn=${firstEp%%:*} substr=":" @@ -1013,18 +1016,19 @@ function installProduct() { tmpPort="" fi if [[ "$tmpPort" != "" ]]; then - echo -e "${GREEN_DARK}To access ${productName2} ${NC}: ${clientName2} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t: ${clientName2} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" else - echo -e "${GREEN_DARK}To access ${productName2} ${NC}: ${clientName2} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t: ${clientName2} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" fi echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" echo elif [ ! -z "$serverFqdn" ]; then - echo -e "${GREEN_DARK}To access ${productName2} ${NC}: ${clientName2} -h $serverFqdn${GREEN_DARK} to login into ${productName2} server${NC}" + echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t: ${clientName2} -h $serverFqdn${GREEN_DARK} to login into ${productName2} server${NC}" echo fi echo -e "\033[44;32;1m${productName2} is installed successfully!${NC}" + echo -e "\033[44;32;1mTo manage ${productName2} instance, view documentation and explorer features, you need to install ${clientName2}Explorer ${NC}" echo else # Only install client install_bin diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 53b9c80f10082a1aa70dc6aec54ea54b25be66f1..18ebf9dc8fcf051302933788d3f19aa2cc2b5e91 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -214,13 +214,13 @@ function install_jemalloc() { ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so ${csudo}/usr/bin/install -c -d /usr/local/lib - if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then - ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib - fi - if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then - ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib - fi - if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + # if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + # ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + # fi + # if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + # ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + # fi + if [ -f ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc ]; then ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig fi @@ -267,7 +267,9 @@ function install_log() { } function install_connector() { - ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ + if [ -d ${script_dir}/connector ]; then + ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ + fi } function install_examples() { diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat index 41113c9ae45ca73c9623bffff9b921fdf4525f3f..0b2a55b89c231f2bec5ff9499b7b9909b0dc2317 100644 --- a/packaging/tools/make_install.bat +++ b/packaging/tools/make_install.bat @@ -56,8 +56,8 @@ copy %binary_dir%\\build\\bin\\taos.exe %target_dir% > nul if exist %binary_dir%\\build\\bin\\taosBenchmark.exe ( copy %binary_dir%\\build\\bin\\taosBenchmark.exe %target_dir% > nul ) -if exist %binary_dir%\\build\\lib\\taosws.dll.lib ( - copy %binary_dir%\\build\\lib\\taosws.dll.lib %target_dir%\\driver > nul +if exist %binary_dir%\\build\\lib\\taosws.lib ( + copy %binary_dir%\\build\\lib\\taosws.lib %target_dir%\\driver > nul ) if exist %binary_dir%\\build\\lib\\taosws.dll ( copy %binary_dir%\\build\\lib\\taosws.dll %target_dir%\\driver > nul diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 98c5245cd39367203d7a8d8387b56846d1e19083..0a5f9d26683ff7eba2cc3070f4668beacbeaa2b6 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -241,10 +241,10 @@ function install_jemalloc() { ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.so.2 /usr/local/lib ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so > /dev/null 2>&1 ${csudo}/usr/bin/install -c -d /usr/local/lib - [ -f ${binary_dir}/build/lib/libjemalloc.a ] && - ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.a /usr/local/lib - [ -f ${binary_dir}/build/lib/libjemalloc_pic.a ] && - ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc_pic.a /usr/local/lib + # [ -f ${binary_dir}/build/lib/libjemalloc.a ] && + # ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.a /usr/local/lib + # [ -f ${binary_dir}/build/lib/libjemalloc_pic.a ] && + # ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc_pic.a /usr/local/lib if [ -f "${binary_dir}/build/lib/pkgconfig/jemalloc.pc" ]; then ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig ${csudo}/usr/bin/install -c -m 644 ${binary_dir}/build/lib/pkgconfig/jemalloc.pc \ @@ -432,12 +432,6 @@ function install_examples() { ${csudo}cp -rf ${source_dir}/examples/* ${install_main_dir}/examples || : } -function install_web() { - if [ -d "${binary_dir}/build/share" ]; then - ${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_dir}/share || : - fi -} - function clean_service_on_sysvinit() { if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then ${csudo}service ${serverName} stop || : @@ -592,7 +586,6 @@ function update_TDengine() { install_lib # install_connector install_examples - install_web install_bin install_app diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 28dc770755a9af46bff1f429a7107194e41ae75d..cd59294fe7d3ddb875a4b39955548b4838e03661 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -118,12 +118,12 @@ if [ -f ${build_dir}/bin/jemalloc-config ]; then cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so fi - if [ -f ${build_dir}/lib/libjemalloc.a ]; then - cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib - fi - if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then - cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib - fi + # if [ -f ${build_dir}/lib/libjemalloc.a ]; then + # cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib + # fi + # if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then + # cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib + # fi if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig fi @@ -175,7 +175,7 @@ if [ "$pagMode" == "lite" ]; then fi chmod a+x ${install_dir}/install_client.sh -if [[ $productName == "TDengine" ]]; then +if [[ $productName == "TDengine" ]] && [ "$verMode" != "cloud" ]; then # Copy example code mkdir -p ${install_dir}/examples examples_dir="${top_dir}/examples" @@ -191,7 +191,7 @@ if [[ $productName == "TDengine" ]]; then mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json fi - if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then + if [ "$verMode" == "cluster" ]; then # Copy connector connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 3da005c405cd61aa77ebf89240497ffa485b989f..ad64ca431e22f37a83093ea3564ebdc41793aeb3 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -69,25 +69,29 @@ if [ "$pagMode" == "lite" ]; then bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark " taostools_bin_files="" else - - wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ + if [ "$verMode" == "cloud" ]; then + taostools_bin_files=" ${build_dir}/bin/taosBenchmark" + else + wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ && echo "TDinsight.sh downloaded!" \ || echo "failed to download TDinsight.sh" - # download TDinsight caches - orig_pwd=$(pwd) - tdinsight_caches="" - cd ${build_dir}/bin/ && \ - chmod +x TDinsight.sh - ./TDinsight.sh --download-only ||: -# tdinsight_caches=$(./TDinsight.sh --download-only | xargs -I printf "${build_dir}/bin/{} ") - cd $orig_pwd - echo "TDinsight caches: $tdinsight_caches" - - taostools_bin_files=" ${build_dir}/bin/taosdump \ + # download TDinsight caches + orig_pwd=$(pwd) + tdinsight_caches="" + cd ${build_dir}/bin/ && \ + chmod +x TDinsight.sh + ./TDinsight.sh --download-only ||: + # tdinsight_caches=$(./TDinsight.sh --download-only | xargs -I printf "${build_dir}/bin/{} ") + cd $orig_pwd + echo "TDinsight caches: $tdinsight_caches" + + taostools_bin_files=" ${build_dir}/bin/taosdump \ ${build_dir}/bin/taosBenchmark \ ${build_dir}/bin/TDinsight.sh \ ${build_dir}/bin/tdengine-datasource.zip \ ${build_dir}/bin/tdengine-datasource.zip.md5sum" + fi + [ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx" explorer_bin_files=$(find ${build_dir}/bin/ -name '*-explorer') @@ -122,7 +126,6 @@ else fi install_files="${script_dir}/install.sh" -web_dir="${top_dir}/../enterprise/src/plugins/web" init_file_deb=${script_dir}/../deb/taosd init_file_rpm=${script_dir}/../rpm/taosd @@ -213,12 +216,12 @@ if [ -f ${build_dir}/bin/jemalloc-config ]; then cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so fi - if [ -f ${build_dir}/lib/libjemalloc.a ]; then - cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib - fi - if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then - cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib - fi + # if [ -f ${build_dir}/lib/libjemalloc.a ]; then + # cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib + # fi + # if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then + # cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib + # fi if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig fi @@ -316,17 +319,11 @@ if [[ $dbName == "taos" ]]; then mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json fi - # Add web files - if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then - if [ -d "${web_dir}/admin" ] ; then - mkdir -p ${install_dir}/share/ - cp -Rfap ${web_dir}/admin ${install_dir}/share/ - cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png - cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||: - else - echo "directory not found for enterprise release: ${web_dir}/admin" - fi + if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then + mkdir -p ${install_dir}/share/ + cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||: fi + fi # Copy driver @@ -334,14 +331,14 @@ mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo [ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || : # Copy connector -if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then +if [ "$verMode" == "cluster" ]; then connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then tmp_pwd=`pwd` cd ${install_dir}/connector if [ ! -d taos-connector-jdbc ];then - git clone -b 3.1.0 --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: + git clone -b 3.2.1 --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: fi cd taos-connector-jdbc mvn clean package -Dmaven.test.skip=true @@ -424,7 +421,7 @@ if [ "$exitcode" != "0" ]; then exit $exitcode fi -if [ -n "${taostools_bin_files}" ]; then +if [ -n "${taostools_bin_files}" ] && [ "$verMode" != "cloud" ]; then wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${taostools_install_dir}/bin/TDinsight.sh && echo "TDinsight.sh downloaded!"|| echo "failed to download TDinsight.sh" if [ "$osType" != "Darwin" ]; then tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" --remove-files || : diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index fc392c9684c04ed6ce587d977ec4628df59cbe6f..e79a10c9e9a24014f84cf0c8d29be15cc143bb79 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -169,13 +169,13 @@ function install_jemalloc() { ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so ${csudo}/usr/bin/install -c -d /usr/local/lib - if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then - ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib - fi - if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then - ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib - fi - if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + # if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + # ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + # fi + # if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + # ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + # fi + if [ -f ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc ]; then ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig fi diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 6c671473bfb5046ac18e7b813088ec6d6d3843d2..eca0c5e973ed0d8e69843e8a7c6603bb133b95ad 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -123,8 +123,8 @@ function clean_bin() { ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : ${csudo}rm -f ${bin_link_dir}/${keeperName2} || : - ${csudo}rm -f ${bin_link_dir}/${xName2} || : - ${csudo}rm -f ${bin_link_dir}/${explorerName2} || : + # ${csudo}rm -f ${bin_link_dir}/${xName2} || : + # ${csudo}rm -f ${bin_link_dir}/${explorerName2} || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then ${csudo}rm -f ${bin_link_dir}/${clientName2} || : @@ -147,6 +147,7 @@ function clean_lib() { ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : [ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : #${csudo}rm -rf ${v15_java_app_dir} || : + } function clean_header() { @@ -193,26 +194,26 @@ function clean_service_on_systemd() { fi ${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null - x_service_config="${service_config_dir}/${xName2}.service" - if [ -e "$x_service_config" ]; then - if systemctl is-active --quiet ${xName2}; then - echo "${productName2} ${xName2} is running, stopping it..." - ${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null - fi - ${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${x_service_config} - fi - - explorer_service_config="${service_config_dir}/${explorerName2}.service" - if [ -e "$explorer_service_config" ]; then - if systemctl is-active --quiet ${explorerName2}; then - echo "${productName2} ${explorerName2} is running, stopping it..." - ${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null - fi - ${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${explorer_service_config} - ${csudo}rm -f /etc/${clientName2}/explorer.toml - fi + # x_service_config="${service_config_dir}/${xName2}.service" + # if [ -e "$x_service_config" ]; then + # if systemctl is-active --quiet ${xName2}; then + # echo "${productName2} ${xName2} is running, stopping it..." + # ${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null + # fi + # ${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null + # ${csudo}rm -f ${x_service_config} + # fi + + # explorer_service_config="${service_config_dir}/${explorerName2}.service" + # if [ -e "$explorer_service_config" ]; then + # if systemctl is-active --quiet ${explorerName2}; then + # echo "${productName2} ${explorerName2} is running, stopping it..." + # ${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null + # fi + # ${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null + # ${csudo}rm -f ${explorer_service_config} + # ${csudo}rm -f /etc/${clientName2}/explorer.toml + # fi } function clean_service_on_sysvinit() { diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 18891bb9326619c84039b6888f5ff7867db5d1cb..aa7caaaba3c7453dea3adb604b0e64e5a60ec231 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -46,9 +46,10 @@ enum { RES_TYPE__TMQ_METADATA, }; -#define SHOW_VARIABLES_RESULT_COLS 2 +#define SHOW_VARIABLES_RESULT_COLS 3 #define SHOW_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) +#define SHOW_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE) #define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) #define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) @@ -63,7 +64,7 @@ typedef struct { // statistics int32_t reportCnt; int32_t connKeyCnt; - int32_t passKeyCnt; // with passVer call back + int8_t connHbFlag; // 0 init, 1 send req, 2 get resp int64_t reportBytes; // not implemented int64_t startTime; // ctl @@ -83,8 +84,9 @@ typedef struct { int8_t threadStop; int8_t quitByKill; TdThread thread; - TdThreadMutex lock; // used when app init and cleanup + TdThreadMutex lock; // used when app init and cleanup SHashObj* appSummary; + SHashObj* appHbHash; // key: clusterId SArray* appHbMgrs; // SArray one for each cluster FHbReqHandle reqHandle[CONN_TYPE__MAX]; FHbRspHandle rspHandle[CONN_TYPE__MAX]; @@ -146,6 +148,7 @@ typedef struct STscObj { int64_t id; // ref ID returned by taosAddRef TdThreadMutex mutex; // used to protect the operation on db int32_t numOfReqs; // number of sqlObj bound to this connection + int32_t authVer; SAppInstInfo* pAppInfo; SHashObj* pRequests; SPassInfo passInfo; @@ -227,6 +230,12 @@ typedef struct { STaosxRsp rsp; } SMqTaosxRspObj; +typedef struct SReqRelInfo { + uint64_t userRefId; + uint64_t prevRefId; + uint64_t nextRefId; +} SReqRelInfo; + typedef struct SRequestObj { int8_t resType; // query or tmq uint64_t requestId; @@ -250,10 +259,14 @@ typedef struct SRequestObj { bool validateOnly; // todo refactor bool killed; bool inRetry; + bool isSubReq; uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog uint32_t retry; int64_t allocatorRefId; SQuery* pQuery; + void* pPostPlan; + SReqRelInfo relation; + void* pWrapper; } SRequestObj; typedef struct SSyncQueryParam { @@ -279,6 +292,7 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly); void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int64_t reqid); +void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param); int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols); @@ -368,6 +382,7 @@ typedef struct SSqlCallbackWrapper { SParseContext* pParseCtx; SCatalogReq* pCatalogReq; SRequestObj* pRequest; + void* pPlanInfo; } SSqlCallbackWrapper; SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res); @@ -382,6 +397,12 @@ int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog); bool qnodeRequired(SRequestObj* pRequest); void continueInsertFromCsv(SSqlCallbackWrapper* pWrapper, SRequestObj* pRequest); void destorySqlCallbackWrapper(SSqlCallbackWrapper* pWrapper); +void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, int32_t code); +void restartAsyncQuery(SRequestObj *pRequest, int32_t code); +int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj** pNewRequest); +int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce); +void returnToUser(SRequestObj* pRequest); +void stopAllQueries(SRequestObj *pRequest); #ifdef __cplusplus } diff --git a/source/client/inc/clientLog.h b/source/client/inc/clientLog.h index 908e47083058b25bc0b6439e3e8888e1d2585328..1a6d47d4f991d85ba48f319993712b7b8d057f07 100644 --- a/source/client/inc/clientLog.h +++ b/source/client/inc/clientLog.h @@ -25,6 +25,7 @@ extern "C" { // clang-format off #define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", DEBUG_FATAL, cDebugFlag, __VA_ARGS__); }} while(0) #define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", DEBUG_ERROR, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscErrorL(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLongString("TSC ERROR ", DEBUG_ERROR, cDebugFlag, __VA_ARGS__); }} while(0) #define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0) #define tscWarnL(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLongString("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0) #define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0) @@ -32,6 +33,8 @@ extern "C" { #define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) #define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", 0, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscLog(...) do { taosPrintLog("TSC ", 0, DEBUG_FILE, __VA_ARGS__); } while(0) +#define tscLogL(...) do { taosPrintLongString("TSC ", 0, DEBUG_FILE, __VA_ARGS__); } while(0) // clang-format on #ifdef __cplusplus diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index b20fc6f57a4256f07f55150711d82a6bf05175f7..040064560c20ea267d8d005543a54465d916ee6a 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -64,8 +64,8 @@ extern "C" { #define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) #define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) -#define TS "_ts" -#define TS_LEN 3 +//#define TS "_ts" +//#define TS_LEN 3 #define VALUE "_value" #define VALUE_LEN 6 @@ -232,9 +232,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t * int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); //SArray *smlJsonParseTags(char *start, char *end); bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); -void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn); -int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn); -int nodeListSize(NodeList* list); +//void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn); +//int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn); +//int nodeListSize(NodeList* list); bool smlDoubleToInt64OverFlow(double num); int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); @@ -251,13 +251,14 @@ int32_t smlClearForRerun(SSmlHandle *info); int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); uint8_t smlGetTimestampLen(int64_t num); void clearColValArray(SArray* pCols); -void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag); +void smlDestroyTableInfo(void *para); void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseJSON(SSmlHandle *info, char *payload); +void smlStrReplace(char* src, int32_t len); #ifdef __cplusplus } #endif diff --git a/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h b/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h index c035b6598c8b6290997e65a30f40cd28a11279b1..422bcd57ac1f3a5fa2cfe47eeb1b2c88427b2c20 100644 --- a/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h +++ b/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h @@ -158,6 +158,13 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetVgroupId(JN */ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTableName(JNIEnv *, jobject, jlong); +/* + * Class: com_taosdata_jdbc_tmq_TMQConnector + * Method: tmqGetOffset + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetOffset(JNIEnv *, jobject, jlong); + /* * Class: com_taosdata_jdbc_tmq_TMQConnector * Method: fetchBlockImp @@ -166,6 +173,12 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTableNam JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(JNIEnv *, jobject, jlong, jlong, jobject, jobject); +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqSeekImp(JNIEnv *, jobject, jlong, jstring, jint, + jlong); + +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTopicAssignmentImp(JNIEnv *, jobject, jlong, + jstring, jobject); + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index cae5c8715daafff8f58f82579af2577b9a6a323b..40c27bf164b448a43ce624d3b5a4f21033ad2259 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -29,6 +29,7 @@ #include "trpc.h" #include "tsched.h" #include "ttime.h" +#include "tversion.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -110,8 +111,9 @@ static void deregisterRequest(SRequestObj *pRequest) { if (duration >= (tsSlowLogThreshold * 1000000UL)) { atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); if (tsSlowLogScope & reqType) { - taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s", - taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, pRequest->sqlstr); + taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s", + taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, + pRequest->sqlstr); } } @@ -167,7 +169,7 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.retryMinInterval = tsRedirectPeriod; rpcInit.retryStepFactor = tsRedirectFactor; rpcInit.retryMaxInterval = tsRedirectMaxPeriod; - rpcInit.retryMaxTimouet = tsMaxRetryWaitTime; + rpcInit.retryMaxTimeout = tsMaxRetryWaitTime; int32_t connLimitNum = tsNumOfRpcSessions / (tsNumOfRpcThreads * 3); connLimitNum = TMAX(connLimitNum, 10); @@ -175,6 +177,8 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.connLimitNum = connLimitNum; rpcInit.timeToGetConn = tsTimeToGetAvailableConn; + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); + void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { tscError("failed to init connection to server"); @@ -358,6 +362,47 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); } +void destroySubRequests(SRequestObj *pRequest) { + int32_t reqIdx = -1; + SRequestObj *pReqList[16] = {NULL}; + uint64_t tmpRefId = 0; + + if (pRequest->relation.userRefId && pRequest->relation.userRefId != pRequest->self) { + return; + } + + SRequestObj *pTmp = pRequest; + while (pTmp->relation.prevRefId) { + tmpRefId = pTmp->relation.prevRefId; + pTmp = acquireRequest(tmpRefId); + if (pTmp) { + pReqList[++reqIdx] = pTmp; + releaseRequest(tmpRefId); + } else { + tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tmpRefId, + pTmp->requestId); + break; + } + } + + for (int32_t i = reqIdx; i >= 0; i--) { + removeRequest(pReqList[i]->self); + } + + tmpRefId = pRequest->relation.nextRefId; + while (tmpRefId) { + pTmp = acquireRequest(tmpRefId); + if (pTmp) { + tmpRefId = pTmp->relation.nextRefId; + removeRequest(pTmp->self); + releaseRequest(pTmp->self); + } else { + tscError("0x%" PRIx64 " is not there", tmpRefId); + break; + } + } +} + void doDestroyRequest(void *p) { if (NULL == p) { return; @@ -368,10 +413,14 @@ void doDestroyRequest(void *p) { uint64_t reqId = pRequest->requestId; tscTrace("begin to destroy request %" PRIx64 " p:%p", reqId, pRequest); + destroySubRequests(pRequest); + taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); schedulerFreeJob(&pRequest->body.queryJob, 0); + destorySqlCallbackWrapper(pRequest->pWrapper); + taosMemoryFreeClear(pRequest->msgBuf); taosMemoryFreeClear(pRequest->pDb); @@ -412,6 +461,62 @@ void destroyRequest(SRequestObj *pRequest) { removeRequest(pRequest->self); } +void taosStopQueryImpl(SRequestObj *pRequest) { + pRequest->killed = true; + + // It is not a query, no need to stop. + if (NULL == pRequest->pQuery || QUERY_EXEC_MODE_SCHEDULE != pRequest->pQuery->execMode) { + tscDebug("request 0x%" PRIx64 " no need to be killed since not query", pRequest->requestId); + return; + } + + schedulerFreeJob(&pRequest->body.queryJob, TSDB_CODE_TSC_QUERY_KILLED); + tscDebug("request %" PRIx64 " killed", pRequest->requestId); +} + +void stopAllQueries(SRequestObj *pRequest) { + int32_t reqIdx = -1; + SRequestObj *pReqList[16] = {NULL}; + uint64_t tmpRefId = 0; + + if (pRequest->relation.userRefId && pRequest->relation.userRefId != pRequest->self) { + return; + } + + SRequestObj *pTmp = pRequest; + while (pTmp->relation.prevRefId) { + tmpRefId = pTmp->relation.prevRefId; + pTmp = acquireRequest(tmpRefId); + if (pTmp) { + pReqList[++reqIdx] = pTmp; + releaseRequest(tmpRefId); + } else { + tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tmpRefId, + pTmp->requestId); + break; + } + } + + for (int32_t i = reqIdx; i >= 0; i--) { + taosStopQueryImpl(pReqList[i]); + } + + taosStopQueryImpl(pRequest); + + tmpRefId = pRequest->relation.nextRefId; + while (tmpRefId) { + pTmp = acquireRequest(tmpRefId); + if (pTmp) { + tmpRefId = pTmp->relation.nextRefId; + taosStopQueryImpl(pTmp); + releaseRequest(pTmp->self); + } else { + tscError("0x%" PRIx64 " is not there", tmpRefId); + break; + } + } +} + void crashReportThreadFuncUnexpectedStopped(void) { atomic_store_32(&clientStop, -1); } static void *tscCrashReportThreadFp(void *param) { @@ -656,7 +761,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { } else { tscInfo("set cfg:%s to %s", pItem->name, str); if (TSDB_OPTION_SHELL_ACTIVITY_TIMER == option || TSDB_OPTION_USE_ADAPTER == option) { - code = taosSetCfg(pCfg, pItem->name); + code = taosApplyLocalCfg(pCfg, pItem->name); } } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 0c60592fc50c087454f1a924ef778cc9d09117fa..54e3a6ee48ea7cbd94eb358c07255e8352b94470 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -22,10 +22,10 @@ typedef struct { union { struct { - int64_t clusterId; - int32_t passKeyCnt; - int32_t passVer; - int32_t reqCnt; + SAppHbMgr *pAppHbMgr; + int64_t clusterId; + int32_t reqCnt; + int8_t connHbFlag; }; }; } SHbParam; @@ -34,12 +34,14 @@ static SClientHbMgr clientHbMgr = {0}; static int32_t hbCreateThread(); static void hbStopThread(); +static int32_t hbUpdateUserAuthInfo(SAppHbMgr *pAppHbMgr, SUserAuthBatchRsp *batchRsp); static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; } static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } -static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { +static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog, + SAppHbMgr *pAppHbMgr) { int32_t code = 0; SUserAuthBatchRsp batchRsp = {0}; @@ -56,54 +58,68 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC catalogUpdateUserAuthInfo(pCatalog, rsp); } + if (numOfBatchs > 0) hbUpdateUserAuthInfo(pAppHbMgr, &batchRsp); + + atomic_val_compare_exchange_8(&pAppHbMgr->connHbFlag, 1, 2); + taosArrayDestroy(batchRsp.pArray); return TSDB_CODE_SUCCESS; } -static int32_t hbProcessUserPassInfoRsp(void *value, int32_t valueLen, SClientHbKey *connKey, SAppHbMgr *pAppHbMgr) { - int32_t code = 0; - int32_t numOfBatchs = 0; - SUserPassBatchRsp batchRsp = {0}; - if (tDeserializeSUserPassBatchRsp(value, valueLen, &batchRsp) != 0) { - code = TSDB_CODE_INVALID_MSG; - return code; - } - - numOfBatchs = taosArrayGetSize(batchRsp.pArray); - - SClientHbReq *pReq = NULL; - while ((pReq = taosHashIterate(pAppHbMgr->activeInfo, pReq))) { - STscObj *pTscObj = (STscObj *)acquireTscObj(pReq->connKey.tscRid); - if (!pTscObj) { - continue; - } - SPassInfo *passInfo = &pTscObj->passInfo; - if (!passInfo->fp) { - releaseTscObj(pReq->connKey.tscRid); +static int32_t hbUpdateUserAuthInfo(SAppHbMgr *pAppHbMgr, SUserAuthBatchRsp *batchRsp) { + uint64_t clusterId = pAppHbMgr->pAppInstInfo->clusterId; + for (int i = 0; i < TARRAY_SIZE(clientHbMgr.appHbMgrs); ++i) { + SAppHbMgr *hbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); + if (!hbMgr || hbMgr->pAppInstInfo->clusterId != clusterId) { continue; } - for (int32_t i = 0; i < numOfBatchs; ++i) { - SGetUserPassRsp *rsp = taosArrayGet(batchRsp.pArray, i); - if (0 == strncmp(rsp->user, pTscObj->user, TSDB_USER_LEN)) { - int32_t oldVer = atomic_load_32(&passInfo->ver); - if (oldVer < rsp->version) { - atomic_store_32(&passInfo->ver, rsp->version); + SClientHbReq *pReq = NULL; + SGetUserAuthRsp *pRsp = NULL; + while ((pReq = taosHashIterate(hbMgr->activeInfo, pReq))) { + STscObj *pTscObj = (STscObj *)acquireTscObj(pReq->connKey.tscRid); + if (!pTscObj) { + continue; + } + + if (!pRsp) { + for (int32_t j = 0; j < TARRAY_SIZE(batchRsp->pArray); ++j) { + SGetUserAuthRsp *rsp = TARRAY_GET_ELEM(batchRsp->pArray, j); + if (0 == strncmp(rsp->user, pTscObj->user, TSDB_USER_LEN)) { + pRsp = rsp; + break; + } + } + if (!pRsp) { + releaseTscObj(pReq->connKey.tscRid); + break; + } + } + + pTscObj->authVer = pRsp->version; + + if (pTscObj->sysInfo != pRsp->sysInfo) { + tscDebug("update sysInfo of user %s from %" PRIi8 " to %" PRIi8 ", tscRid:%" PRIi64, pRsp->user, + pTscObj->sysInfo, pRsp->sysInfo, pTscObj->id); + pTscObj->sysInfo = pRsp->sysInfo; + } + + if (pTscObj->passInfo.fp) { + SPassInfo *passInfo = &pTscObj->passInfo; + int32_t oldVer = atomic_load_32(&passInfo->ver); + if (oldVer < pRsp->passVer) { + atomic_store_32(&passInfo->ver, pRsp->passVer); if (passInfo->fp) { - (*passInfo->fp)(passInfo->param, &passInfo->ver, TAOS_NOTIFY_PASSVER); + (*passInfo->fp)(passInfo->param, &pRsp->passVer, TAOS_NOTIFY_PASSVER); } - tscDebug("update passVer of user %s from %d to %d, tscRid:%" PRIi64, rsp->user, oldVer, + tscDebug("update passVer of user %s from %d to %d, tscRid:%" PRIi64, pRsp->user, oldVer, atomic_load_32(&passInfo->ver), pTscObj->id); } - break; } + releaseTscObj(pReq->connKey.tscRid); } - releaseTscObj(pReq->connKey.tscRid); } - - taosArrayDestroy(batchRsp.pArray); - - return code; + return 0; } static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { @@ -121,7 +137,6 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { vgInfo->hashSuffix = rsp->hashSuffix; vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == vgInfo->vgHash) { - taosMemoryFree(vgInfo); tscError("hash init[%d] failed", rsp->vgNum); code = TSDB_CODE_OUT_OF_MEMORY; goto _return; @@ -131,8 +146,6 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j); if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { tscError("hash push failed, errno:%d", errno); - taosHashCleanup(vgInfo->vgHash); - taosMemoryFree(vgInfo); code = TSDB_CODE_OUT_OF_MEMORY; goto _return; } @@ -151,48 +164,52 @@ _return: static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; - SUseDbBatchRsp batchUseRsp = {0}; - if (tDeserializeSUseDbBatchRsp(value, valueLen, &batchUseRsp) != 0) { + SDbHbBatchRsp batchRsp = {0}; + if (tDeserializeSDbHbBatchRsp(value, valueLen, &batchRsp) != 0) { terrno = TSDB_CODE_INVALID_MSG; - return -1; + code = terrno; + goto _return; } - int32_t numOfBatchs = taosArrayGetSize(batchUseRsp.pArray); + int32_t numOfBatchs = taosArrayGetSize(batchRsp.pArray); for (int32_t i = 0; i < numOfBatchs; ++i) { - SUseDbRsp *rsp = taosArrayGet(batchUseRsp.pArray, i); - tscDebug("hb db rsp, db:%s, vgVersion:%d, stateTs:%" PRId64 ", uid:%" PRIx64, rsp->db, rsp->vgVersion, rsp->stateTs, - rsp->uid); - - if (rsp->vgVersion < 0) { - code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); - } else { - SDBVgInfo *vgInfo = NULL; - code = hbGenerateVgInfoFromRsp(&vgInfo, rsp); - if (TSDB_CODE_SUCCESS != code) { - goto _return; - } - - catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, vgInfo); - - if (IS_SYS_DBNAME(rsp->db)) { - code = hbGenerateVgInfoFromRsp(&vgInfo, rsp); + SDbHbRsp *rsp = taosArrayGet(batchRsp.pArray, i); + if (rsp->useDbRsp) { + tscDebug("hb use db rsp, db:%s, vgVersion:%d, stateTs:%" PRId64 ", uid:%" PRIx64, + rsp->useDbRsp->db, rsp->useDbRsp->vgVersion, rsp->useDbRsp->stateTs, rsp->useDbRsp->uid); + + if (rsp->useDbRsp->vgVersion < 0) { + code = catalogRemoveDB(pCatalog, rsp->useDbRsp->db, rsp->useDbRsp->uid); + } else { + SDBVgInfo *vgInfo = NULL; + code = hbGenerateVgInfoFromRsp(&vgInfo, rsp->useDbRsp); if (TSDB_CODE_SUCCESS != code) { goto _return; } - catalogUpdateDBVgInfo(pCatalog, (rsp->db[0] == 'i') ? TSDB_PERFORMANCE_SCHEMA_DB : TSDB_INFORMATION_SCHEMA_DB, - rsp->uid, vgInfo); + catalogUpdateDBVgInfo(pCatalog, rsp->useDbRsp->db, rsp->useDbRsp->uid, vgInfo); + + if (IS_SYS_DBNAME(rsp->useDbRsp->db)) { + code = hbGenerateVgInfoFromRsp(&vgInfo, rsp->useDbRsp); + if (TSDB_CODE_SUCCESS != code) { + goto _return; + } + + catalogUpdateDBVgInfo(pCatalog, (rsp->useDbRsp->db[0] == 'i') ? TSDB_PERFORMANCE_SCHEMA_DB : TSDB_INFORMATION_SCHEMA_DB, rsp->useDbRsp->uid, vgInfo); + } } } - if (code) { - goto _return; + if (rsp->cfgRsp) { + tscDebug("hb db cfg rsp, db:%s, cfgVersion:%d", rsp->cfgRsp->db, rsp->cfgRsp->cfgVersion); + catalogUpdateDbCfg(pCatalog, rsp->cfgRsp->db, rsp->cfgRsp->dbId, rsp->cfgRsp); + rsp->cfgRsp = NULL; } } _return: - tFreeSUseDbBatchRsp(&batchUseRsp); + tFreeSDbHbBatchRsp(&batchRsp); return code; } @@ -220,7 +237,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo return TSDB_CODE_TSC_INVALID_VALUE; } - catalogUpdateTableMeta(pCatalog, rsp); + catalogAsyncUpdateTableMeta(pCatalog, rsp); } } @@ -312,7 +329,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { break; } - hbProcessUserAuthInfoRsp(kv->value, kv->valueLen, pCatalog); + hbProcessUserAuthInfoRsp(kv->value, kv->valueLen, pCatalog, pAppHbMgr); break; } case HEARTBEAT_KEY_DBINFO: { @@ -349,15 +366,6 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { hbProcessStbInfoRsp(kv->value, kv->valueLen, pCatalog); break; } - case HEARTBEAT_KEY_USER_PASSINFO: { - if (kv->valueLen <= 0 || NULL == kv->value) { - tscError("invalid hb user pass info, len:%d, value:%p", kv->valueLen, kv->value); - break; - } - - hbProcessUserPassInfoRsp(kv->value, kv->valueLen, &pRsp->connKey, pAppHbMgr); - break; - } default: tscError("invalid hb key type:%d", kv->key); break; @@ -460,6 +468,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { desc.useconds = now - pRequest->metric.start; desc.reqRid = pRequest->self; desc.stableQuery = pRequest->stableQuery; + desc.isSubQuery = pRequest->isSubReq; taosGetFqdn(desc.fqdn); desc.subPlanNum = pRequest->body.subplanNum; @@ -474,7 +483,6 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { if (code) { taosArrayDestroy(desc.subDesc); desc.subDesc = NULL; - desc.subPlanNum = 0; } desc.subPlanNum = taosArrayGetSize(desc.subDesc); } else { @@ -538,7 +546,7 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_SUCCESS; } -static int32_t hbGetUserBasicInfo(SClientHbKey *connKey, SHbParam *param, SClientHbReq *req) { +static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClientHbReq *req) { STscObj *pTscObj = (STscObj *)acquireTscObj(connKey->tscRid); if (!pTscObj) { tscWarn("tscObj rid %" PRIx64 " not exist", connKey->tscRid); @@ -547,46 +555,61 @@ static int32_t hbGetUserBasicInfo(SClientHbKey *connKey, SHbParam *param, SClien int32_t code = 0; - if (param && (param->passVer != INT32_MIN) && (param->passVer <= pTscObj->passInfo.ver)) { - tscDebug("hb got user basic info, no need since passVer %d <= %d", param->passVer, pTscObj->passInfo.ver); + SKv kv = {.key = HEARTBEAT_KEY_USER_AUTHINFO}; + SKv *pKv = NULL; + if ((pKv = taosHashGet(req->info, &kv.key, sizeof(kv.key)))) { + int32_t userNum = pKv->valueLen / sizeof(SUserAuthVersion); + SUserAuthVersion *userAuths = (SUserAuthVersion *)pKv->value; + for (int32_t i = 0; i < userNum; ++i) { + SUserAuthVersion *pUserAuth = userAuths + i; + // both key and user exist, update version + if (strncmp(pUserAuth->user, pTscObj->user, TSDB_USER_LEN) == 0) { + pUserAuth->version = htonl(-1); // force get userAuthInfo + goto _return; + } + } + // key exists, user not exist, append user + SUserAuthVersion *qUserAuth = + (SUserAuthVersion *)taosMemoryRealloc(pKv->value, (userNum + 1) * sizeof(SUserAuthVersion)); + if (qUserAuth) { + strncpy((qUserAuth + userNum)->user, pTscObj->user, TSDB_USER_LEN); + (qUserAuth + userNum)->version = htonl(-1); // force get userAuthInfo + pKv->value = qUserAuth; + pKv->valueLen += sizeof(SUserAuthVersion); + } else { + code = TSDB_CODE_OUT_OF_MEMORY; + } goto _return; } - SUserPassVersion *user = taosMemoryMalloc(sizeof(SUserPassVersion)); + // key/user not exist, add user + SUserAuthVersion *user = taosMemoryMalloc(sizeof(SUserAuthVersion)); if (!user) { code = TSDB_CODE_OUT_OF_MEMORY; goto _return; } - strncpy(user->user, pTscObj->user, TSDB_USER_LEN); - user->version = htonl(pTscObj->passInfo.ver); - - SKv kv = { - .key = HEARTBEAT_KEY_USER_PASSINFO, - .valueLen = sizeof(SUserPassVersion), - .value = user, - }; + tstrncpy(user->user, pTscObj->user, TSDB_USER_LEN); + user->version = htonl(-1); // force get userAuthInfo + kv.valueLen = sizeof(SUserAuthVersion); + kv.value = user; - tscDebug("hb got user basic info, valueLen:%d, user:%s, passVer:%d, tscRid:%" PRIi64, kv.valueLen, user->user, - pTscObj->passInfo.ver, connKey->tscRid); + tscDebug("hb got user auth info, valueLen:%d, user:%s, authVer:%d, tscRid:%" PRIi64, kv.valueLen, user->user, + pTscObj->authVer, connKey->tscRid); if (!req->info) { req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); } if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) < 0) { + taosMemoryFree(user); code = terrno ? terrno : TSDB_CODE_APP_ERROR; goto _return; } - // assign the passVer - if (param) { - param->passVer = pTscObj->passInfo.ver; - } - _return: releaseTscObj(connKey->tscRid); if (code) { - tscError("hb got user basic info failed since %s", terrstr(code)); + tscError("hb got user auth info failed since %s", terrstr(code)); } return code; @@ -630,7 +653,7 @@ int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S } int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { - SDbVgVersion *dbs = NULL; + SDbCacheInfo *dbs = NULL; uint32_t dbNum = 0; int32_t code = 0; @@ -645,19 +668,20 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl } for (int32_t i = 0; i < dbNum; ++i) { - SDbVgVersion *db = &dbs[i]; - tscDebug("the %dth expired dbFName:%s, dbId:%" PRId64 ", vgVersion:%d, numOfTable:%d, startTs:%" PRId64, i, - db->dbFName, db->dbId, db->vgVersion, db->numOfTable, db->stateTs); + SDbCacheInfo *db = &dbs[i]; + tscDebug("the %dth expired dbFName:%s, dbId:%" PRId64 ", vgVersion:%d, cfgVersion:%d, numOfTable:%d, startTs:%" PRId64, + i, db->dbFName, db->dbId, db->vgVersion, db->cfgVersion, db->numOfTable, db->stateTs); db->dbId = htobe64(db->dbId); db->vgVersion = htonl(db->vgVersion); + db->cfgVersion = htonl(db->cfgVersion); db->numOfTable = htonl(db->numOfTable); db->stateTs = htobe64(db->stateTs); } SKv kv = { .key = HEARTBEAT_KEY_DBINFO, - .valueLen = sizeof(SDbVgVersion) * dbNum, + .valueLen = sizeof(SDbCacheInfo) * dbNum, .value = dbs, }; @@ -743,14 +767,21 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req hbGetQueryBasicInfo(connKey, req); - if (hbParam->passKeyCnt > 0) { - hbGetUserBasicInfo(connKey, hbParam, req); - } - if (hbParam->reqCnt == 0) { - code = hbGetExpiredUserInfo(connKey, pCatalog, req); - if (TSDB_CODE_SUCCESS != code) { - return code; + if (!taosHashGet(clientHbMgr.appHbHash, &hbParam->clusterId, sizeof(hbParam->clusterId))) { + code = hbGetExpiredUserInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + + // invoke after hbGetExpiredUserInfo + if (2 != atomic_load_8(&hbParam->pAppHbMgr->connHbFlag)) { + code = hbGetUserAuthInfo(connKey, hbParam, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + atomic_store_8(&hbParam->pAppHbMgr->connHbFlag, 1); } code = hbGetExpiredDBInfo(connKey, pCatalog, req); @@ -764,7 +795,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req } } - ++hbParam->reqCnt; // success to get catalog info + ++hbParam->reqCnt; // success to get catalog info return TSDB_CODE_SUCCESS; } @@ -791,57 +822,42 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { return NULL; } - int64_t rid = -1; - int32_t code = 0; - - void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); - - SClientHbReq *pOneReq = pIter; - SClientHbKey *connKey = pOneReq ? &pOneReq->connKey : NULL; - if (connKey != NULL) rid = connKey->tscRid; - - STscObj *pTscObj = (STscObj *)acquireTscObj(rid); - if (pTscObj == NULL) { - tFreeClientHbBatchReq(pBatchReq); - return NULL; - } - + void *pIter = NULL; SHbParam param = {0}; + while ((pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter))) { + SClientHbReq *pOneReq = pIter; + SClientHbKey *connKey = &pOneReq->connKey; + STscObj *pTscObj = (STscObj *)acquireTscObj(connKey->tscRid); + + if (!pTscObj) { + continue; + } - while (pIter != NULL) { pOneReq = taosArrayPush(pBatchReq->reqs, pOneReq); - switch (pOneReq->connKey.connType) { + switch (connKey->connType) { case CONN_TYPE__QUERY: { if (param.clusterId == 0) { // init param.clusterId = pOneReq->clusterId; - param.passVer = INT32_MIN; + param.pAppHbMgr = pAppHbMgr; + param.connHbFlag = atomic_load_8(&pAppHbMgr->connHbFlag); } - param.passKeyCnt = atomic_load_32(&pAppHbMgr->passKeyCnt); break; } default: break; } - if (clientHbMgr.reqHandle[pOneReq->connKey.connType]) { - code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, ¶m, pOneReq); + if (clientHbMgr.reqHandle[connKey->connType]) { + int32_t code = (*clientHbMgr.reqHandle[connKey->connType])(connKey, ¶m, pOneReq); if (code) { tscWarn("hbGatherAllInfo failed since %s, tscRid:%" PRIi64 ", connType:%" PRIi8, tstrerror(code), - pOneReq->connKey.tscRid, pOneReq->connKey.connType); + connKey->tscRid, connKey->connType); } } - if (code) { - pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); - pOneReq = pIter; - continue; - } - - pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); - pOneReq = pIter; + releaseTscObj(connKey->tscRid); } - releaseTscObj(rid); return pBatchReq; } @@ -910,6 +926,10 @@ static void *hbThreadFunc(void *param) { int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); if (sz > 0) { hbGatherAppInfo(); + if (sz > 1 && !clientHbMgr.appHbHash) { + clientHbMgr.appHbHash = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); + } + taosHashClear(clientHbMgr.appHbHash); } for (int i = 0; i < sz; i++) { @@ -962,7 +982,7 @@ static void *hbThreadFunc(void *param) { asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); - + taosHashPut(clientHbMgr.appHbHash, &pAppHbMgr->pAppInstInfo->clusterId, sizeof(uint64_t), NULL, 0); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } @@ -970,6 +990,7 @@ static void *hbThreadFunc(void *param) { taosMsleep(HEARTBEAT_INTERVAL); } + taosHashCleanup(clientHbMgr.appHbHash); return NULL; } @@ -1018,7 +1039,7 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { // init stat pAppHbMgr->startTime = taosGetTimestampMs(); pAppHbMgr->connKeyCnt = 0; - pAppHbMgr->passKeyCnt = 0; + pAppHbMgr->connHbFlag = 0; pAppHbMgr->reportCnt = 0; pAppHbMgr->reportBytes = 0; pAppHbMgr->key = taosStrdup(key); @@ -1136,7 +1157,6 @@ void hbMgrCleanUp() { appHbMgrCleanup(); taosArrayDestroy(clientHbMgr.appHbMgrs); taosThreadMutexUnlock(&clientHbMgr.lock); - clientHbMgr.appHbMgrs = NULL; } @@ -1189,12 +1209,6 @@ void hbDeregisterConn(STscObj *pTscObj, SClientHbKey connKey) { } atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); - - taosThreadMutexLock(&pTscObj->mutex); - if (pTscObj->passInfo.fp) { - atomic_sub_fetch_32(&pAppHbMgr->passKeyCnt, 1); - } - taosThreadMutexUnlock(&pTscObj->mutex); } // set heart beat thread quit mode , if quicByKill 1 then kill thread else quit from inner diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index f8eade1d7c62f3da22d765aea5b98622b34a0ba7..d448dd1edf7e49c34f0a736bece4b8febad6c96c 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -26,7 +26,7 @@ #include "tpagedbuf.h" #include "tref.h" #include "tsched.h" - +#include "tversion.h" static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); @@ -237,6 +237,18 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, return TSDB_CODE_SUCCESS; } +int32_t buildPreviousRequest(SRequestObj* pRequest, const char* sql, SRequestObj** pNewRequest) { + int32_t code = + buildRequest(pRequest->pTscObj->id, sql, strlen(sql), pRequest, pRequest->validateOnly, pNewRequest, 0); + if (TSDB_CODE_SUCCESS == code) { + pRequest->relation.prevRefId = (*pNewRequest)->self; + (*pNewRequest)->relation.nextRefId = pRequest->self; + (*pNewRequest)->relation.userRefId = pRequest->self; + (*pNewRequest)->isSubReq = true; + } + return code; +} + int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb) { STscObj* pTscObj = pRequest->pTscObj; @@ -491,7 +503,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t pResInfo->userFields[i].bytes = pSchema[i].bytes; pResInfo->userFields[i].type = pSchema[i].type; - if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR) { + if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR || pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) { pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE; } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) { pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; @@ -815,7 +827,7 @@ int32_t handleAlterTbExecRes(void* res, SCatalog* pCatalog) { } int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog) { - return catalogUpdateTableMeta(pCatalog, (STableMetaRsp*)res); + return catalogAsyncUpdateTableMeta(pCatalog, (STableMetaRsp*)res); } int32_t handleQueryExecRsp(SRequestObj* pRequest) { @@ -877,6 +889,81 @@ static bool incompletaFileParsing(SNode* pStmt) { return QUERY_NODE_VNODE_MODIFY_STMT != nodeType(pStmt) ? false : ((SVnodeModifyOpStmt*)pStmt)->fileProcessing; } +void continuePostSubQuery(SRequestObj* pRequest, TAOS_ROW row) { + SSqlCallbackWrapper* pWrapper = pRequest->pWrapper; + int32_t code = nodesAcquireAllocator(pWrapper->pParseCtx->allocatorId); + if (TSDB_CODE_SUCCESS == code) { + int64_t analyseStart = taosGetTimestampUs(); + code = qContinueParsePostQuery(pWrapper->pParseCtx, pRequest->pQuery, (void**)row); + pRequest->metric.analyseCostUs += taosGetTimestampUs() - analyseStart; + } + if (TSDB_CODE_SUCCESS == code) { + code = qContinuePlanPostQuery(pRequest->pPostPlan); + } + nodesReleaseAllocator(pWrapper->pParseCtx->allocatorId); + + handleQueryAnslyseRes(pWrapper, NULL, code); +} + +void returnToUser(SRequestObj* pRequest) { + if (pRequest->relation.userRefId == pRequest->self || 0 == pRequest->relation.userRefId) { + // return to client + pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code); + return; + } + + SRequestObj* pUserReq = acquireRequest(pRequest->relation.userRefId); + if (pUserReq) { + pUserReq->code = pRequest->code; + // return to client + pUserReq->body.queryFp(pUserReq->body.param, pUserReq, pUserReq->code); + releaseRequest(pRequest->relation.userRefId); + return; + } else { + tscError("0x%" PRIx64 ", user ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self, + pRequest->relation.userRefId, pRequest->requestId); + } +} + +void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) { + SRequestObj* pRequest = (SRequestObj*)res; + if (pRequest->code) { + returnToUser(pRequest); + return; + } + + TAOS_ROW row = NULL; + if (rowNum > 0) { + row = taos_fetch_row(res); // for single row only now + } + + SRequestObj* pNextReq = acquireRequest(pRequest->relation.nextRefId); + if (pNextReq) { + continuePostSubQuery(pNextReq, row); + releaseRequest(pRequest->relation.nextRefId); + } else { + tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self, + pRequest->relation.nextRefId, pRequest->requestId); + } +} + +void handlePostSubQuery(SSqlCallbackWrapper* pWrapper) { + SRequestObj* pRequest = pWrapper->pRequest; + if (TD_RES_QUERY(pRequest)) { + taosAsyncFetchImpl(pRequest, postSubQueryFetchCb, pWrapper); + return; + } + + SRequestObj* pNextReq = acquireRequest(pRequest->relation.nextRefId); + if (pNextReq) { + continuePostSubQuery(pNextReq, NULL); + releaseRequest(pRequest->relation.nextRefId); + } else { + tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self, + pRequest->relation.nextRefId, pRequest->requestId); + } +} + // todo refacto the error code mgmt void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { SSqlCallbackWrapper* pWrapper = param; @@ -911,12 +998,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->retry, pRequest->requestId); - pRequest->prevCode = code; - schedulerFreeJob(&pRequest->body.queryJob, 0); - qDestroyQuery(pRequest->pQuery); - pRequest->pQuery = NULL; - destorySqlCallbackWrapper(pWrapper); - doAsyncQuery(pRequest, true); + restartAsyncQuery(pRequest, code); return; } @@ -937,10 +1019,15 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { return; } - destorySqlCallbackWrapper(pWrapper); + if (pRequest->relation.nextRefId) { + handlePostSubQuery(pWrapper); + } else { + destorySqlCallbackWrapper(pWrapper); + pRequest->pWrapper = NULL; - // return to client - pRequest->body.queryFp(pRequest->body.param, pRequest, code); + // return to client + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + } } SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res) { @@ -1048,6 +1135,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat pRequest->requestId); } else { pRequest->body.subplanNum = pDag->numOfSubplans; + TSWAP(pRequest->pPostPlan, pDag->pPostPlan); } pRequest->metric.execStart = taosGetTimestampUs(); @@ -1083,6 +1171,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); destorySqlCallbackWrapper(pWrapper); + pRequest->pWrapper = NULL; if (TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; } @@ -1102,6 +1191,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM pRequest->body.execMode = pQuery->execMode; if (QUERY_EXEC_MODE_SCHEDULE != pRequest->body.execMode) { destorySqlCallbackWrapper(pWrapper); + pRequest->pWrapper = NULL; } if (pQuery->pRoot && !pRequest->inRetry) { @@ -1207,13 +1297,19 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe return -1; } - int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); + int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TSC_INVALID_FQDN; return terrno; } - - mgmtEpSet->numOfEps++; + uint32_t addr = taosGetIpv4FromFqdn(mgmtEpSet->eps[mgmtEpSet->numOfEps].fqdn); + if (addr == 0xffffffff) { + tscError("failed to resolve firstEp fqdn: %s, code:%s", mgmtEpSet->eps[mgmtEpSet->numOfEps].fqdn, + tstrerror(TSDB_CODE_TSC_INVALID_FQDN)); + memset(&(mgmtEpSet->eps[mgmtEpSet->numOfEps]), 0, sizeof(mgmtEpSet->eps[mgmtEpSet->numOfEps])); + } else { + mgmtEpSet->numOfEps++; + } } if (secondEp && secondEp[0] != 0) { @@ -1223,12 +1319,19 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe } taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); - mgmtEpSet->numOfEps++; + uint32_t addr = taosGetIpv4FromFqdn(mgmtEpSet->eps[mgmtEpSet->numOfEps].fqdn); + if (addr == 0xffffffff) { + tscError("failed to resolve secondEp fqdn: %s, code:%s", mgmtEpSet->eps[mgmtEpSet->numOfEps].fqdn, + tstrerror(TSDB_CODE_TSC_INVALID_FQDN)); + memset(&(mgmtEpSet->eps[mgmtEpSet->numOfEps]), 0, sizeof(mgmtEpSet->eps[mgmtEpSet->numOfEps])); + } else { + mgmtEpSet->numOfEps++; + } } if (mgmtEpSet->numOfEps == 0) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; + terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + return TSDB_CODE_RPC_NETWORK_UNAVAIL; } return 0; @@ -1698,6 +1801,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i len += lenTmp; pStart += lenTmp; + int32_t estimateColLen = 0; for (int32_t j = 0; j < numOfRows; ++j) { if (offset[j] == -1) { continue; @@ -1707,20 +1811,21 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i int32_t jsonInnerType = *data; char* jsonInnerData = data + CHAR_BYTES; if (jsonInnerType == TSDB_DATA_TYPE_NULL) { - len += (VARSTR_HEADER_SIZE + strlen(TSDB_DATA_NULL_STR_L)); + estimateColLen += (VARSTR_HEADER_SIZE + strlen(TSDB_DATA_NULL_STR_L)); } else if (tTagIsJson(data)) { - len += (VARSTR_HEADER_SIZE + ((const STag*)(data))->len); + estimateColLen += (VARSTR_HEADER_SIZE + ((const STag*)(data))->len); } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" - len += varDataTLen(jsonInnerData) + CHAR_BYTES * 2; + estimateColLen += varDataTLen(jsonInnerData) + CHAR_BYTES * 2; } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) { - len += (VARSTR_HEADER_SIZE + 32); + estimateColLen += (VARSTR_HEADER_SIZE + 32); } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { - len += (VARSTR_HEADER_SIZE + 5); + estimateColLen += (VARSTR_HEADER_SIZE + 5); } else { tscError("estimateJsonLen error: invalid type:%d", jsonInnerType); return -1; } } + len += TMAX(colLen, estimateColLen); } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { int32_t lenTmp = numOfRows * sizeof(int32_t); len += (lenTmp + colLen); @@ -1756,6 +1861,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int return TSDB_CODE_TSC_INTERNAL_ERROR; } + taosMemoryFreeClear(pResultInfo->convertJson); pResultInfo->convertJson = taosMemoryCalloc(1, dataLen); if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; char* p1 = pResultInfo->convertJson; @@ -2042,6 +2148,7 @@ TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* de connLimitNum = TMIN(connLimitNum, 500); rpcInit.connLimitNum = connLimitNum; rpcInit.timeToGetConn = tsTimeToGetAvailableConn; + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); clientRpc = rpcOpen(&rpcInit); if (clientRpc == NULL) { @@ -2400,3 +2507,87 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, return pRequest; } + +static void fetchCallback(void* pResult, void* param, int32_t code) { + SRequestObj* pRequest = (SRequestObj*)param; + + SReqResultInfo* pResultInfo = &pRequest->body.resInfo; + + tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, + tstrerror(code), pRequest->requestId); + + pResultInfo->pData = pResult; + pResultInfo->numOfRows = 0; + + if (code != TSDB_CODE_SUCCESS) { + pRequest->code = code; + taosMemoryFreeClear(pResultInfo->pData); + pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); + return; + } + + if (pRequest->code != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pResultInfo->pData); + pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); + return; + } + + pRequest->code = + setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4, true); + if (pRequest->code != TSDB_CODE_SUCCESS) { + pResultInfo->numOfRows = 0; + pRequest->code = code; + tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); + } else { + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, + pRequest->requestId); + + STscObj* pTscObj = pRequest->pTscObj; + SAppClusterSummary* pActivity = &pTscObj->pAppInfo->summary; + atomic_add_fetch_64((int64_t*)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen); + } + + pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); +} + +void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param) { + pRequest->body.fetchFp = fp; + pRequest->body.param = param; + + SReqResultInfo* pResultInfo = &pRequest->body.resInfo; + + // this query has no results or error exists, return directly + if (taos_num_fields(pRequest) == 0 || pRequest->code != TSDB_CODE_SUCCESS) { + pResultInfo->numOfRows = 0; + pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows); + return; + } + + // all data has returned to App already, no need to try again + if (pResultInfo->completed) { + // it is a local executed query, no need to do async fetch + if (QUERY_EXEC_MODE_SCHEDULE != pRequest->body.execMode) { + if (pResultInfo->localResultFetched) { + pResultInfo->numOfRows = 0; + pResultInfo->current = 0; + } else { + pResultInfo->localResultFetched = true; + } + } else { + pResultInfo->numOfRows = 0; + } + + pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows); + return; + } + + SSchedulerReq req = { + .syncReq = false, + .fetchFp = fetchCallback, + .cbParam = pRequest, + }; + + schedulerFetchRows(pRequest->body.queryJob, &req); +} diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c index b61335475125955b801b5b697992e79c98010431..4ad2d4d51f387467fe623cfdb80511458e3b52c9 100644 --- a/source/client/src/clientJniConnector.c +++ b/source/client/src/clientJniConnector.c @@ -579,7 +579,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn dv = GET_DOUBLE_VAL(row[i]); (*env)->CallVoidMethod(env, rowobj, g_rowdataSetDoubleFp, i, (jdouble)dv); } break; - case TSDB_DATA_TYPE_BINARY: { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_GEOMETRY: { memcpy(tmp, row[i], length[i]); // handle the case that terminated does not exist (*env)->CallVoidMethod(env, rowobj, g_rowdataSetStringFp, i, (*env)->NewStringUTF(env, tmp)); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 55337d4a0bd93f6e140e44600fad0203018284da..e262ee04b99168cd3da6259433ee430257f6c5d3 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -135,11 +135,6 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type) switch (type) { case TAOS_NOTIFY_PASSVER: { taosThreadMutexLock(&pObj->mutex); - if (fp && !pObj->passInfo.fp) { - atomic_add_fetch_32(&pObj->pAppInfo->pAppHbMgr->passKeyCnt, 1); - } else if (!fp && pObj->passInfo.fp) { - atomic_sub_fetch_32(&pObj->pAppInfo->pAppHbMgr->passKeyCnt, 1); - } pObj->passInfo.fp = fp; pObj->passInfo.param = param; taosThreadMutexUnlock(&pObj->mutex); @@ -393,9 +388,10 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) } break; case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: { int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); - if (fields[i].type == TSDB_DATA_TYPE_BINARY) { + if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_GEOMETRY) { if (ASSERT(charLen <= fields[i].bytes && charLen >= 0)) { tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); } @@ -475,6 +471,8 @@ const char *taos_data_type(int type) { return "TSDB_DATA_TYPE_NCHAR"; case TSDB_DATA_TYPE_JSON: return "TSDB_DATA_TYPE_JSON"; + case TSDB_DATA_TYPE_GEOMETRY: + return "TSDB_DATA_TYPE_GEOMETRY"; case TSDB_DATA_TYPE_UTINYINT: return "TSDB_DATA_TYPE_UTINYINT"; case TSDB_DATA_TYPE_USMALLINT: @@ -565,17 +563,7 @@ void taos_stop_query(TAOS_RES *res) { return; } - SRequestObj *pRequest = (SRequestObj *)res; - pRequest->killed = true; - - // It is not a query, no need to stop. - if (NULL == pRequest->pQuery || QUERY_EXEC_MODE_SCHEDULE != pRequest->pQuery->execMode) { - tscDebug("request 0x%" PRIx64 " no need to be killed since not query", pRequest->requestId); - return; - } - - schedulerFreeJob(&pRequest->body.queryJob, TSDB_CODE_TSC_QUERY_KILLED); - tscDebug("request %" PRIx64 " killed", pRequest->requestId); + stopAllQueries((SRequestObj *)res); } bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { @@ -771,8 +759,13 @@ static void destoryCatalogReq(SCatalogReq *pCatalogReq) { taosArrayDestroy(pCatalogReq->pDbVgroup); taosArrayDestroy(pCatalogReq->pDbCfg); taosArrayDestroy(pCatalogReq->pDbInfo); - taosArrayDestroyEx(pCatalogReq->pTableMeta, destoryTablesReq); - taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq); + if (pCatalogReq->cloned) { + taosArrayDestroy(pCatalogReq->pTableMeta); + taosArrayDestroy(pCatalogReq->pTableHash); + } else { + taosArrayDestroyEx(pCatalogReq->pTableMeta, destoryTablesReq); + taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq); + } taosArrayDestroy(pCatalogReq->pUdf); taosArrayDestroy(pCatalogReq->pIndex); taosArrayDestroy(pCatalogReq->pUser); @@ -791,26 +784,106 @@ void destorySqlCallbackWrapper(SSqlCallbackWrapper *pWrapper) { taosMemoryFree(pWrapper); } +void destroyCtxInRequest(SRequestObj *pRequest) { + schedulerFreeJob(&pRequest->body.queryJob, 0); + qDestroyQuery(pRequest->pQuery); + pRequest->pQuery = NULL; + destorySqlCallbackWrapper(pRequest->pWrapper); + pRequest->pWrapper = NULL; +} + static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t code) { SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)param; SRequestObj *pRequest = pWrapper->pRequest; SQuery *pQuery = pRequest->pQuery; + qDebug("0x%" PRIx64 " start to semantic analysis, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); + int64_t analyseStart = taosGetTimestampUs(); pRequest->metric.ctgCostUs = analyseStart - pRequest->metric.ctgStart; - qDebug("0x%" PRIx64 " start to semantic analysis, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); - if (code == TSDB_CODE_SUCCESS) { + if (TSDB_CODE_SUCCESS == code) { code = qAnalyseSqlSemantic(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery); + } + + pRequest->metric.analyseCostUs += taosGetTimestampUs() - analyseStart; + + handleQueryAnslyseRes(pWrapper, pResultMeta, code); +} + +int32_t cloneCatalogReq(SCatalogReq **ppTarget, SCatalogReq *pSrc) { + int32_t code = TSDB_CODE_SUCCESS; + SCatalogReq *pTarget = taosMemoryCalloc(1, sizeof(SCatalogReq)); + if (pTarget == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pTarget->pDbVgroup = taosArrayDup(pSrc->pDbVgroup, NULL); + pTarget->pDbCfg = taosArrayDup(pSrc->pDbCfg, NULL); + pTarget->pDbInfo = taosArrayDup(pSrc->pDbInfo, NULL); + pTarget->pTableMeta = taosArrayDup(pSrc->pTableMeta, NULL); + pTarget->pTableHash = taosArrayDup(pSrc->pTableHash, NULL); + pTarget->pUdf = taosArrayDup(pSrc->pUdf, NULL); + pTarget->pIndex = taosArrayDup(pSrc->pIndex, NULL); + pTarget->pUser = taosArrayDup(pSrc->pUser, NULL); + pTarget->pTableIndex = taosArrayDup(pSrc->pTableIndex, NULL); + pTarget->pTableCfg = taosArrayDup(pSrc->pTableCfg, NULL); + pTarget->pTableTag = taosArrayDup(pSrc->pTableTag, NULL); + pTarget->qNodeRequired = pSrc->qNodeRequired; + pTarget->dNodeRequired = pSrc->dNodeRequired; + pTarget->svrVerRequired = pSrc->svrVerRequired; + pTarget->forceUpdate = pSrc->forceUpdate; + pTarget->cloned = true; + + *ppTarget = pTarget; + } + + return code; +} + +void handleSubQueryFromAnalyse(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, SNode *pRoot) { + SRequestObj *pNewRequest = NULL; + SSqlCallbackWrapper *pNewWrapper = NULL; + int32_t code = buildPreviousRequest(pWrapper->pRequest, pWrapper->pRequest->sqlstr, &pNewRequest); + if (code) { + handleQueryAnslyseRes(pWrapper, pResultMeta, code); + return; + } + + pNewRequest->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pNewRequest->pQuery) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pNewRequest->pQuery->pRoot = pRoot; + pRoot = NULL; + pNewRequest->pQuery->execStage = QUERY_EXEC_STAGE_ANALYSE; + } + if (TSDB_CODE_SUCCESS == code) { + code = prepareAndParseSqlSyntax(&pNewWrapper, pNewRequest, false); + } + if (TSDB_CODE_SUCCESS == code) { + code = cloneCatalogReq(&pNewWrapper->pCatalogReq, pWrapper->pCatalogReq); + } + doAsyncQueryFromAnalyse(pResultMeta, pNewWrapper, code); + nodesDestroyNode(pRoot); +} + +void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta, int32_t code) { + SRequestObj *pRequest = pWrapper->pRequest; + SQuery *pQuery = pRequest->pQuery; + + if (code == TSDB_CODE_SUCCESS && pQuery->pPrevRoot) { + SNode *prevRoot = pQuery->pPrevRoot; + pQuery->pPrevRoot = NULL; + handleSubQueryFromAnalyse(pWrapper, pResultMeta, prevRoot); + return; + } + + if (code == TSDB_CODE_SUCCESS) { pRequest->stableQuery = pQuery->stableQuery; if (pQuery->pRoot) { pRequest->stmtType = pQuery->pRoot->type; } - } - - pRequest->metric.analyseCostUs = taosGetTimestampUs() - analyseStart; - if (code == TSDB_CODE_SUCCESS) { if (pQuery->haveResultSet) { setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols); setResPrecision(&pRequest->body.resInfo, pQuery->precision); @@ -823,14 +896,14 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t launchAsyncQuery(pRequest, pQuery, pResultMeta, pWrapper); } else { destorySqlCallbackWrapper(pWrapper); + pRequest->pWrapper = NULL; qDestroyQuery(pRequest->pQuery); pRequest->pQuery = NULL; if (NEED_CLIENT_HANDLE_ERROR(code)) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); - pRequest->prevCode = code; - doAsyncQuery(pRequest, true); + restartAsyncQuery(pRequest, code); return; } @@ -838,7 +911,7 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); pRequest->code = code; - pRequest->body.queryFp(pRequest->body.param, pRequest, code); + returnToUser(pRequest); } } @@ -901,6 +974,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code, tstrerror(code), pWrapper->pRequest->requestId); destorySqlCallbackWrapper(pWrapper); + pRequest->pWrapper = NULL; terrno = code; pRequest->code = code; pRequest->body.queryFp(pRequest->body.param, pRequest, code); @@ -917,6 +991,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code, tstrerror(code), pWrapper->pRequest->requestId); destorySqlCallbackWrapper(pWrapper); + pRequest->pWrapper = NULL; terrno = code; pRequest->code = code; pRequest->body.queryFp(pRequest->body.param, pRequest, code); @@ -964,27 +1039,16 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { return TSDB_CODE_SUCCESS; } -void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { - STscObj *pTscObj = pRequest->pTscObj; - SSqlCallbackWrapper *pWrapper = NULL; +int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce) { int32_t code = TSDB_CODE_SUCCESS; - - if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) { - code = pRequest->prevCode; - terrno = code; - pRequest->code = code; - tscDebug("call sync query cb with code: %s", tstrerror(code)); - pRequest->body.queryFp(pRequest->body.param, pRequest, code); - return; - } - - if (TSDB_CODE_SUCCESS == code) { - pWrapper = taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); - if (pWrapper == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - pWrapper->pRequest = pRequest; - } + STscObj *pTscObj = pRequest->pTscObj; + SSqlCallbackWrapper *pWrapper = taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); + if (pWrapper == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pWrapper->pRequest = pRequest; + pRequest->pWrapper = pWrapper; + *ppWrapper = pWrapper; } if (TSDB_CODE_SUCCESS == code) { @@ -996,7 +1060,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pWrapper->pParseCtx->pCatalog); } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && NULL == pRequest->pQuery) { int64_t syntaxStart = taosGetTimestampUs(); pWrapper->pCatalogReq = taosMemoryCalloc(1, sizeof(SCatalogReq)); @@ -1011,6 +1075,26 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { pRequest->metric.parseCostUs += taosGetTimestampUs() - syntaxStart; } + return code; +} + +void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { + SSqlCallbackWrapper *pWrapper = NULL; + int32_t code = TSDB_CODE_SUCCESS; + + if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) { + code = pRequest->prevCode; + terrno = code; + pRequest->code = code; + tscDebug("call sync query cb with code: %s", tstrerror(code)); + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + return; + } + + if (TSDB_CODE_SUCCESS == code) { + code = prepareAndParseSqlSyntax(&pWrapper, pRequest, updateMetaForce); + } + if (TSDB_CODE_SUCCESS == code) { pRequest->stmtType = pRequest->pQuery->pRoot->type; code = phaseAsyncQuery(pWrapper); @@ -1020,12 +1104,14 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); destorySqlCallbackWrapper(pWrapper); + pRequest->pWrapper = NULL; qDestroyQuery(pRequest->pQuery); pRequest->pQuery = NULL; if (NEED_CLIENT_HANDLE_ERROR(code)) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); + refreshMeta(pRequest->pTscObj, pRequest); pRequest->prevCode = code; doAsyncQuery(pRequest, true); return; @@ -1037,48 +1123,57 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { } } -static void fetchCallback(void *pResult, void *param, int32_t code) { - SRequestObj *pRequest = (SRequestObj *)param; - - SReqResultInfo *pResultInfo = &pRequest->body.resInfo; - - tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, - tstrerror(code), pRequest->requestId); - - pResultInfo->pData = pResult; - pResultInfo->numOfRows = 0; +void restartAsyncQuery(SRequestObj *pRequest, int32_t code) { + int32_t reqIdx = 0; + SRequestObj *pReqList[16] = {NULL}; + SRequestObj *pUserReq = NULL; + pReqList[0] = pRequest; + uint64_t tmpRefId = 0; + SRequestObj *pTmp = pRequest; + while (pTmp->relation.prevRefId) { + tmpRefId = pTmp->relation.prevRefId; + pTmp = acquireRequest(tmpRefId); + if (pTmp) { + pReqList[++reqIdx] = pTmp; + releaseRequest(tmpRefId); + } else { + tscError("0x%" PRIx64 ", prev req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pTmp->self, tmpRefId, + pTmp->requestId); + break; + } + } - if (code != TSDB_CODE_SUCCESS) { - pRequest->code = code; - taosMemoryFreeClear(pResultInfo->pData); - pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); - return; + tmpRefId = pRequest->relation.nextRefId; + while (tmpRefId) { + pTmp = acquireRequest(tmpRefId); + if (pTmp) { + tmpRefId = pTmp->relation.nextRefId; + removeRequest(pTmp->self); + releaseRequest(pTmp->self); + } else { + tscError("0x%" PRIx64 " is not there", tmpRefId); + break; + } } - if (pRequest->code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pResultInfo->pData); - pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); - return; + for (int32_t i = reqIdx; i >= 0; i--) { + destroyCtxInRequest(pReqList[i]); + if (pReqList[i]->relation.userRefId == pReqList[i]->self || 0 == pReqList[i]->relation.userRefId) { + pUserReq = pReqList[i]; + } else { + removeRequest(pReqList[i]->self); + } } - pRequest->code = - setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp *)pResultInfo->pData, pResultInfo->convertUcs4, true); - if (pRequest->code != TSDB_CODE_SUCCESS) { - pResultInfo->numOfRows = 0; - pRequest->code = code; - tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), - pRequest->requestId); + if (pUserReq) { + pUserReq->prevCode = code; + memset(&pUserReq->relation, 0, sizeof(pUserReq->relation)); } else { - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, - pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, - pRequest->requestId); - - STscObj *pTscObj = pRequest->pTscObj; - SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; - atomic_add_fetch_64((int64_t *)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen); + tscError("user req is missing"); + return; } - pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); + doAsyncQuery(pUserReq, true); } void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { @@ -1092,43 +1187,8 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } SRequestObj *pRequest = res; - pRequest->body.fetchFp = fp; - pRequest->body.param = param; - - SReqResultInfo *pResultInfo = &pRequest->body.resInfo; - - // this query has no results or error exists, return directly - if (taos_num_fields(pRequest) == 0 || pRequest->code != TSDB_CODE_SUCCESS) { - pResultInfo->numOfRows = 0; - pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows); - return; - } - - // all data has returned to App already, no need to try again - if (pResultInfo->completed) { - // it is a local executed query, no need to do async fetch - if (QUERY_EXEC_MODE_SCHEDULE != pRequest->body.execMode) { - if (pResultInfo->localResultFetched) { - pResultInfo->numOfRows = 0; - pResultInfo->current = 0; - } else { - pResultInfo->localResultFetched = true; - } - } else { - pResultInfo->numOfRows = 0; - } - - pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows); - return; - } - - SSchedulerReq req = { - .syncReq = false, - .fetchFp = fetchCallback, - .cbParam = pRequest, - }; - schedulerFetchRows(pRequest->body.queryJob, &req); + taosAsyncFetchImpl(pRequest, fp, param); } void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 6d53f2b4c51f1845de00150d7bfa54eacc2e591b..9f9809b2274592fcb9bd75ba32907a040a4a7a5d 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -77,6 +77,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { } if ((code = taosCheckVersionCompatibleFromStr(version, connectRsp.sVer, 3)) != 0) { + tscError("version not compatible. client version: %s, server version: %s", version, connectRsp.sVer); setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); goto End; @@ -98,13 +99,20 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { goto End; } + int updateEpSet = 1; if (connectRsp.dnodeNum == 1) { SEpSet srcEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); SEpSet dstEpSet = connectRsp.epSet; - rpcSetDefaultAddr(pTscObj->pAppInfo->pTransporter, srcEpSet.eps[srcEpSet.inUse].fqdn, - dstEpSet.eps[dstEpSet.inUse].fqdn); - } else if (connectRsp.dnodeNum > 1 && !isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) { - SEpSet* pOrig = &pTscObj->pAppInfo->mgmtEp.epSet; + if (srcEpSet.numOfEps == 1) { + rpcSetDefaultAddr(pTscObj->pAppInfo->pTransporter, srcEpSet.eps[srcEpSet.inUse].fqdn, + dstEpSet.eps[dstEpSet.inUse].fqdn); + updateEpSet = 0; + } + } + if (updateEpSet == 1 && !isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) { + SEpSet corEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + + SEpSet* pOrig = &corEpSet; SEp* pOrigEp = &pOrig->eps[pOrig->inUse]; SEp* pNewEp = &connectRsp.epSet.eps[connectRsp.epSet.inUse]; tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in connRsp", pOrig->inUse, pOrig->numOfEps, @@ -130,6 +138,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { pTscObj->connType = connectRsp.connType; pTscObj->passInfo.ver = connectRsp.passVer; + pTscObj->authVer = connectRsp.authVer; hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType); @@ -426,13 +435,16 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { SColumnInfoData infoData = {0}; infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD1_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD2_LEN; taosArrayPush(pBlock->pDataBlock, &infoData); + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD3_LEN; + taosArrayPush(pBlock->pDataBlock, &infoData); + int32_t numOfCfg = taosArrayGetSize(pVars); blockDataEnsureCapacity(pBlock, numOfCfg); @@ -448,6 +460,11 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { STR_WITH_MAXSIZE_TO_VARSTR(value, pInfo->value, TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); colDataSetVal(pColInfo, i, value, false); + + char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(scope, pInfo->scope, TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE); + pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + colDataSetVal(pColInfo, i, scope, false); } pBlock->info.rows = numOfCfg; diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 32f28e456319cd67a4808fa9f0ffefaedc1d3dbf..dd311db1267967a6ca657346bdf95095aeec15fb 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -56,7 +56,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch cJSON_AddItemToObject(column, "name", cname); cJSON* ctype = cJSON_CreateNumber(s->type); cJSON_AddItemToObject(column, "type", ctype); - if (s->type == TSDB_DATA_TYPE_BINARY) { + if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(column, "length", cbytes); @@ -77,7 +77,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch cJSON_AddItemToObject(tag, "name", tname); cJSON* ttype = cJSON_CreateNumber(s->type); cJSON_AddItemToObject(tag, "type", ttype); - if (s->type == TSDB_DATA_TYPE_BINARY) { + if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(tag, "length", cbytes); @@ -130,7 +130,7 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { cJSON* colType = cJSON_CreateNumber(field->type); cJSON_AddItemToObject(json, "colType", colType); - if (field->type == TSDB_DATA_TYPE_BINARY) { + if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); @@ -155,7 +155,7 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { cJSON_AddItemToObject(json, "colName", colName); cJSON* colType = cJSON_CreateNumber(field->type); cJSON_AddItemToObject(json, "colType", colType); - if (field->type == TSDB_DATA_TYPE_BINARY) { + if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); @@ -457,7 +457,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type); cJSON_AddItemToObject(json, "colType", colType); - if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY) { + if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); @@ -478,7 +478,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { cJSON_AddItemToObject(json, "colName", colName); cJSON* colType = cJSON_CreateNumber(vAlterTbReq.colModType); cJSON_AddItemToObject(json, "colType", colType); - if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY) { + if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); @@ -1286,6 +1286,10 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { taosArrayPush(pArray, &pVgData); pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pQuery) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->msgType = TDMT_VND_ALTER_TABLE; pQuery->stableQuery = false; @@ -1323,6 +1327,9 @@ end: int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD* fields, int numFields) { + if (!taos || !pData || !tbname) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; @@ -1409,6 +1416,9 @@ end: } int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) { + if (!taos || !pData || !tbname) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; @@ -1808,6 +1818,7 @@ end: } char* tmq_get_json_meta(TAOS_RES* res) { + if (res == NULL) return NULL; uDebug("tmq_get_json_meta called"); if (!TD_RES_TMQ_META(res) && !TD_RES_TMQ_METADATA(res)) { return NULL; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6170b0a056ba03432aaf7c8ddca20c18bd377295..ffff3df5d02797c0dd6d289e868a12a046d2d9f4 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -24,77 +24,105 @@ int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL}; int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL}; -void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) { - NodeList *tmp = list; - while (tmp) { - if (fn == NULL) { - if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { - return tmp->data.value; - } - } else { - if (tmp->data.used && fn(tmp->data.key, key) == 0) { - return tmp->data.value; - } - } - - tmp = tmp->next; +//void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) { +// NodeList *tmp = list; +// while (tmp) { +// if (fn == NULL) { +// if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { +// return tmp->data.value; +// } +// } else { +// if (tmp->data.used && fn(tmp->data.key, key) == 0) { +// return tmp->data.value; +// } +// } +// +// tmp = tmp->next; +// } +// return NULL; +//} +// +//int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) { +// NodeList *tmp = *list; +// while (tmp) { +// if (!tmp->data.used) break; +// if (fn == NULL) { +// if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { +// return -1; +// } +// } else { +// if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) { +// return -1; +// } +// } +// +// tmp = tmp->next; +// } +// if (tmp) { +// tmp->data.key = key; +// tmp->data.keyLen = len; +// tmp->data.value = value; +// tmp->data.used = true; +// } else { +// NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList)); +// if (newNode == NULL) { +// return -1; +// } +// newNode->data.key = key; +// newNode->data.keyLen = len; +// newNode->data.value = value; +// newNode->data.used = true; +// newNode->next = *list; +// *list = newNode; +// } +// return 0; +//} +// +//int nodeListSize(NodeList *list) { +// int cnt = 0; +// while (list) { +// if (list->data.used) +// cnt++; +// else +// break; +// list = list->next; +// } +// return cnt; +//} + +static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo* conn, const char* pTabName, AUTH_TYPE type){ + SUserAuthInfo pAuth = {0}; + snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user); + if (NULL == pTabName) { + tNameSetDbName(&pAuth.tbName, info->taos->acctId, info->pRequest->pDb, strlen(info->pRequest->pDb)); + } else { + toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName); } - return NULL; -} + pAuth.type = type; -int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) { - NodeList *tmp = *list; - while (tmp) { - if (!tmp->data.used) break; - if (fn == NULL) { - if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { - return -1; - } - } else { - if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) { - return -1; - } - } + int32_t code = TSDB_CODE_SUCCESS; + SUserAuthRes authRes = {0}; - tmp = tmp->next; - } - if (tmp) { - tmp->data.key = key; - tmp->data.keyLen = len; - tmp->data.value = value; - tmp->data.used = true; - } else { - NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList)); - if (newNode == NULL) { - return -1; - } - newNode->data.key = key; - newNode->data.keyLen = len; - newNode->data.value = value; - newNode->data.used = true; - newNode->next = *list; - *list = newNode; - } - return 0; -} + code = catalogChkAuth(info->pCatalog, conn, &pAuth, &authRes); + nodesDestroyNode(authRes.pCond); -int nodeListSize(NodeList *list) { - int cnt = 0; - while (list) { - if (list->data.used) - cnt++; - else - break; - list = list->next; - } - return cnt; -} + return (code == TSDB_CODE_SUCCESS) ? (authRes.pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code; +} inline bool smlDoubleToInt64OverFlow(double num) { if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; return false; } +void smlStrReplace(char* src, int32_t len){ + if (!tsSmlDot2Underline) return; + for(int i = 0; i < len; i++){ + if(src[i] == '.'){ + src[i] = '_'; + } + } +} + int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { if (pBuf->buf) { memset(pBuf->buf, 0, pBuf->len); @@ -174,6 +202,9 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { memset(childTableName, 0, TSDB_TABLE_NAME_LEN); strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + if(tsSmlDot2Underline){ + smlStrReplace(childTableName, strlen(childTableName)); + } taosArrayRemove(tags, i); break; } @@ -211,6 +242,16 @@ void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tin } } +static void smlDestroySTableMeta(void *para) { + SSmlSTableMeta *meta = *(SSmlSTableMeta**)para; + taosHashCleanup(meta->tagHash); + taosHashCleanup(meta->colHash); + taosArrayDestroy(meta->tags); + taosArrayDestroy(meta->cols); + taosMemoryFreeClear(meta->tableMeta); + taosMemoryFree(meta); +} + SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); if (!meta) { @@ -245,7 +286,7 @@ SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { return meta; cleanup: - taosMemoryFree(meta); + smlDestroySTableMeta(meta); return NULL; } @@ -555,7 +596,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm return TSDB_CODE_SML_INVALID_DATA; } - if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR && + if (((colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_GEOMETRY) && (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) || (colField[*index].type == TSDB_DATA_TYPE_NCHAR && ((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) { @@ -586,7 +627,7 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { } } - if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + if ((type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_GEOMETRY) && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { result = (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; @@ -594,7 +635,7 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { if (type == TSDB_DATA_TYPE_NCHAR) { result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - } else if (type == TSDB_DATA_TYPE_BINARY) { + } else if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_GEOMETRY) { result = result + VARSTR_HEADER_SIZE; } return result; @@ -638,7 +679,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool } static int32_t getBytes(uint8_t type, int32_t length) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) { return smlFindNearestPowerOf2(length, type); } else { return tDataTypes[type].bytes; @@ -720,6 +761,9 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pReq.suid = pTableMeta->uid; pReq.source = TD_REQ_FROM_TAOX; pSql = (action == SCHEMA_ACTION_ADD_COLUMN) ? "sml_add_column" : "sml_modify_column_size"; + } else{ + uError("SML:0x%" PRIx64 " invalid action:%d", info->id, action); + goto end; } code = buildRequest(info->taos->id, pSql, strlen(pSql), NULL, false, &pRequest, 0); @@ -806,6 +850,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { char *measure = taosMemoryMalloc(superTableLen); memcpy(measure, superTable, superTableLen); PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); + smlStrReplace(measure, superTableLen); memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); memcpy(pName.tname, measure, superTableLen); taosMemoryFree(measure); @@ -813,6 +858,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { + code = smlCheckAuth(info, &conn, NULL, AUTH_TYPE_WRITE); + if(code != TSDB_CODE_SUCCESS){ + goto end; + } uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); @@ -857,6 +906,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { goto end; } if (action != SCHEMA_ACTION_NULL) { + code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); + if(code != TSDB_CODE_SUCCESS){ + goto end; + } uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname, action); SArray *pColumns = @@ -927,6 +980,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { goto end; } if (action != SCHEMA_ACTION_NULL) { + code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); + if(code != TSDB_CODE_SUCCESS){ + goto end; + } uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname, action); SArray *pColumns = @@ -1004,9 +1061,11 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } } + taosMemoryFreeClear(sTableData->tableMeta); sTableData->tableMeta = pTableMeta; uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid, - pTableMeta->sversion, pTableMeta->tversion) tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp); + pTableMeta->sversion, pTableMeta->tversion); + tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp); } uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat, info->needModifySchema); @@ -1062,15 +1121,6 @@ static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { } } -static void smlDestroySTableMeta(SSmlSTableMeta *meta) { - taosHashCleanup(meta->tagHash); - taosHashCleanup(meta->colHash); - taosArrayDestroy(meta->tags); - taosArrayDestroy(meta->cols); - taosMemoryFree(meta->tableMeta); - taosMemoryFree(meta); -} - static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); @@ -1110,7 +1160,8 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols return TSDB_CODE_SUCCESS; } -void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { +void smlDestroyTableInfo(void *para) { + SSmlTableInfo *tag = *(SSmlTableInfo**)para; for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); taosHashCleanup(kvHash); @@ -1147,18 +1198,18 @@ void smlDestroyInfo(SSmlHandle *info) { qDestroyQuery(info->pQuery); // destroy info->childTables - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); - while (oneTable) { - smlDestroyTableInfo(info, *oneTable); - oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); - } +// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); +// while (oneTable) { +// smlDestroyTableInfo(oneTable); +// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); +// } // destroy info->superTables - SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); - while (oneSTable) { - smlDestroySTableMeta(*oneSTable); - oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); - } +// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); +// while (oneSTable) { +// smlDestroySTableMeta(*oneSTable); +// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); +// } // destroy info->pVgHash taosHashCleanup(info->pVgHash); @@ -1217,6 +1268,8 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->tableUids = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); + taosHashSetFreeFp(info->childTables, smlDestroyTableInfo); info->id = smlGenId(); info->pQuery = smlInitHandle(); @@ -1323,6 +1376,9 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + if(meta == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); terrno = 0; smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); @@ -1352,7 +1408,14 @@ static int32_t smlInsertData(SSmlHandle *info) { SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); while (oneTable) { SSmlTableInfo *tableData = *oneTable; - tstrncpy(pName.tname, tableData->sTableName, tableData->sTableNameLen + 1); + + int measureLen = tableData->sTableNameLen; + char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); + memcpy(measure, tableData->sTableName, tableData->sTableNameLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + smlStrReplace(measure, measureLen); + memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + memcpy(pName.tname, measure, measureLen); if (info->pRequest->tableList == NULL) { info->pRequest->tableList = taosArrayInit(1, sizeof(SName)); @@ -1367,10 +1430,17 @@ static int32_t smlInsertData(SSmlHandle *info) { conn.requestObjRefId = info->pRequest->self; conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); + if(code != TSDB_CODE_SUCCESS){ + taosMemoryFree(measure); + return code; + } + SVgroupInfo vg; code = catalogGetTableHashVgroup(info->pCatalog, &conn, &pName, &vg); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " catalogGetTableHashVgroup failed. table name: %s", info->id, tableData->childTableName); + taosMemoryFree(measure); return code; } taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); @@ -1379,6 +1449,7 @@ static int32_t smlInsertData(SSmlHandle *info) { (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); if (unlikely(NULL == pMeta || NULL == (*pMeta)->tableMeta)) { uError("SML:0x%" PRIx64 " NULL == pMeta. table name: %s", info->id, tableData->childTableName); + taosMemoryFree(measure); return TSDB_CODE_SML_INTERNAL_ERROR; } @@ -1388,11 +1459,6 @@ static int32_t smlInsertData(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname, tableData->uid, info->dataFormat); - int measureLen = tableData->sTableNameLen; - char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); - memcpy(measure, tableData->sTableName, tableData->sTableNameLen); - PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); - code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, (*pMeta)->tableMeta, tableData->childTableName, measure, measureLen, info->ttl, info->msgBuf.buf, info->msgBuf.len); @@ -1437,18 +1503,18 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { int32_t smlClearForRerun(SSmlHandle *info) { info->reRun = false; // clear info->childTables - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); - while (oneTable) { - smlDestroyTableInfo(info, *oneTable); - oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); - } +// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); +// while (oneTable) { +// smlDestroyTableInfo(info, *oneTable); +// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); +// } // clear info->superTables - SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); - while (oneSTable) { - smlDestroySTableMeta(*oneSTable); - oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); - } +// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); +// while (oneSTable) { +// smlDestroySTableMeta(*oneSTable); +// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); +// } taosHashClear(info->childTables); taosHashClear(info->superTables); @@ -1509,17 +1575,8 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } } - char cTmp = 0; // for print tmp if is raw - if (info->isRawLine) { - cTmp = tmp[len]; - tmp[len] = '\0'; - } - uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id, - info->isRawLine, numLines, info->protocol, len, tmp); - if (info->isRawLine) { - tmp[len] = cTmp; - } + info->isRawLine, numLines, info->protocol, len, info->isRawLine ? "rawdata" : tmp); if (info->protocol == TSDB_SML_LINE_PROTOCOL) { if (info->dataFormat) { @@ -1540,8 +1597,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; } if (code != TSDB_CODE_SUCCESS) { - tmp[len] = '\0'; - uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); + uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, info->isRawLine ? "rawdata" : tmp); return code; } if (info->reRun) { @@ -1586,9 +1642,7 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL do { code = smlModifyDBSchemas(info); - if (code == 0 || code == TSDB_CODE_SML_INVALID_DATA || code == TSDB_CODE_PAR_TOO_MANY_COLUMNS - || code == TSDB_CODE_PAR_INVALID_TAGS_NUM || code == TSDB_CODE_PAR_INVALID_TAGS_LENGTH - || code == TSDB_CODE_PAR_INVALID_ROW_LENGTH || code == TSDB_CODE_MND_FIELD_VALUE_OVERFLOW) { + if (code != TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER && code != TSDB_CODE_SDB_OBJ_CREATING && code != TSDB_CODE_MND_TRANS_CONFLICT) { break; } taosMsleep(100); @@ -1714,9 +1768,8 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, request->code = code; info->cost.endTime = taosGetTimestampUs(); info->cost.code = code; - if (code == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || code == TSDB_CODE_SDB_OBJ_CREATING || - code == TSDB_CODE_PAR_VALUE_TOO_LONG || code == TSDB_CODE_MND_TRANS_CONFLICT || - code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + if (NEED_CLIENT_HANDLE_ERROR(code) || code == TSDB_CODE_SDB_OBJ_CREATING || + code == TSDB_CODE_PAR_VALUE_TOO_LONG || code == TSDB_CODE_MND_TRANS_CONFLICT) { if (cnt++ >= 10) { uInfo("SML:%" PRIx64 " retry:%d/10 end code:%d, msg:%s", info->id, cnt, code, tstrerror(code)); break; diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 7ccf9309648583cd9610a6375fc717297fc7352d..76794fd187f6c06d2297b5be0b57547d516e2402 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -456,7 +456,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { elements->measureLen = strlen(metric->valuestring); if (IS_INVALID_TABLE_LEN(elements->measureLen)) { - uError("OTD:0x%" PRIx64 " Metric lenght is 0 or large than 192", info->id); + uError("OTD:0x%" PRIx64 " Metric length is 0 or large than 192", info->id); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } @@ -695,6 +695,10 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo return TSDB_CODE_SUCCESS; } sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); + return TSDB_CODE_OUT_OF_MEMORY; + } sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ @@ -784,7 +788,7 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); if (tinfo->tableDataCtx == NULL) { smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(info, tinfo); + smlDestroyTableInfo(&tinfo); return TSDB_CODE_SML_INVALID_DATA; } } @@ -992,8 +996,8 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); return TSDB_CODE_INVALID_TIMESTAMP; } - SSmlKv kvTs = {.key = TS, - .keyLen = TS_LEN, + SSmlKv kvTs = {.key = tsSmlTsDefaultName, + .keyLen = strlen(tsSmlTsDefaultName), .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; @@ -1048,12 +1052,18 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { return TSDB_CODE_TSC_INVALID_JSON; } - info->lineNum = payloadNum; - info->dataFormat = true; + if (unlikely(info->lines != NULL)) { + for (int i = 0; i < info->lineNum; i++) { + taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); + if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag); + } taosMemoryFree(info->lines); info->lines = NULL; } + info->lineNum = payloadNum; + info->dataFormat = true; + ret = smlClearForRerun(info); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -1190,8 +1200,8 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * return TSDB_CODE_INVALID_TIMESTAMP; } } - SSmlKv kvTs = {.key = TS, - .keyLen = TS_LEN, + SSmlKv kvTs = {.key = tsSmlTsDefaultName, + .keyLen = strlen(tsSmlTsDefaultName), .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index e79093398e87275a2eabc9bb382c8987a41f1703..1ee2cfbedf38c17fb557296505323569f590eddc 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -157,6 +157,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin measure = (char *)taosMemoryMalloc(currElement->measureLen); memcpy(measure, currElement->measure, currElement->measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + smlStrReplace(measure, measureLen); } STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); if (currElement->measureEscaped) { @@ -168,6 +169,10 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SUCCESS; } sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); + return TSDB_CODE_OUT_OF_MEMORY; + } sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); for (int i = pTableMeta->tableInfo.numOfColumns; @@ -198,7 +203,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin bool keyEscaped = false; size_t keyLenEscaped = 0; while (*sql < sqlEnd) { - if (unlikely(IS_COMMA(*sql))) { + if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); return TSDB_CODE_SML_INVALID_DATA; } @@ -326,7 +331,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin info->currSTableMeta->uid = tinfo->uid; tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); if (tinfo->tableDataCtx == NULL) { - smlDestroyTableInfo(info, tinfo); + smlDestroyTableInfo(&tinfo); smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); return TSDB_CODE_SML_INVALID_DATA; } @@ -361,6 +366,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin measure = (char *)taosMemoryMalloc(currElement->measureLen); memcpy(measure, currElement->measure, currElement->measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + smlStrReplace(measure, measureLen); } STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); if (currElement->measureEscaped) { @@ -372,6 +378,10 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SUCCESS; } *tmp = smlBuildSTableMeta(info->dataFormat); + if(*tmp == NULL){ + taosMemoryFreeClear(pTableMeta); + return TSDB_CODE_OUT_OF_MEMORY; + } (*tmp)->tableMeta = pTableMeta; taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES); @@ -402,7 +412,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin bool keyEscaped = false; size_t keyLenEscaped = 0; while (*sql < sqlEnd) { - if (unlikely(IS_COMMA(*sql))) { + if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); return TSDB_CODE_SML_INVALID_DATA; } @@ -428,19 +438,20 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin size_t valueLen = 0; bool valueEscaped = false; size_t valueLenEscaped = 0; - bool isInQuote = false; + int quoteNum = 0; const char *escapeChar = NULL; while (*sql < sqlEnd) { // parse value if (unlikely(*(*sql) == QUOTE && (*(*sql - 1) != SLASH || (*sql - 1) == escapeChar))) { - isInQuote = !isInQuote; + quoteNum++; (*sql)++; - continue; - } - if (!isInQuote) { - if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + if(quoteNum > 2){ break; } + continue; + } + if (quoteNum % 2 == 0 && (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql)))) { + break; } if (IS_SLASH_LETTER_IN_FIELD_VALUE(*sql) && (*sql - 1) != escapeChar) { escapeChar = *sql; @@ -452,8 +463,8 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } valueLen = *sql - value; - if (unlikely(isInQuote)) { - smlBuildInvalidDataMsg(&info->msgBuf, "only one quote", value); + if (unlikely(quoteNum != 0 && quoteNum != 2)) { + smlBuildInvalidDataMsg(&info->msgBuf, "unbalanced quotes", value); return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(valueLen == 0)) { @@ -642,8 +653,8 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine return TSDB_CODE_INVALID_TIMESTAMP; } // add ts to - SSmlKv kv = {.key = TS, - .keyLen = TS_LEN, + SSmlKv kv = {.key = tsSmlTsDefaultName, + .keyLen = strlen(tsSmlTsDefaultName), .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index 42b8001e5981a781fdf341754dbf07618d342855..c378f9b1c3a7198b49afb760a8fe8782d3765e32 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -91,6 +91,10 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS return TSDB_CODE_SUCCESS; } sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); + return TSDB_CODE_OUT_OF_MEMORY; + } sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ @@ -212,7 +216,7 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); if (tinfo->tableDataCtx == NULL) { smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(info, tinfo); + smlDestroyTableInfo(&tinfo); return TSDB_CODE_SML_INVALID_DATA; } } @@ -256,8 +260,8 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); return TSDB_CODE_INVALID_TIMESTAMP; } - SSmlKv kvTs = {.key = TS, - .keyLen = TS_LEN, + SSmlKv kvTs = {.key = tsSmlTsDefaultName, + .keyLen = strlen(tsSmlTsDefaultName), .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 975b304bf4574bb84e16b1287b3dfec73c8ce869..8ac9550aca4e6705a848b3fef4e95a972008c819 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -939,8 +939,6 @@ int stmtClose(TAOS_STMT* stmt) { stmtCleanSQLInfo(pStmt); taosMemoryFree(stmt); - STMT_DLOG_E("stmt freed"); - return TSDB_CODE_SUCCESS; } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 63e8b3097c3cd7a8199e5001723515dafdb2bb24..b4168046f48143c0f0635db92fdd587f1efeac26 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -27,6 +27,8 @@ #define EMPTY_BLOCK_POLL_IDLE_DURATION 10 #define DEFAULT_AUTO_COMMIT_INTERVAL 5000 +#define OFFSET_IS_RESET_OFFSET(_of) ((_of) < 0) + typedef void (*__tmq_askep_fn_t)(tmq_t* pTmq, int32_t code, SDataBuf* pBuf, void* pParam); struct SMqMgmt { @@ -80,13 +82,14 @@ struct tmq_t { int8_t useSnapshot; int8_t autoCommit; int32_t autoCommitInterval; - int32_t resetOffsetCfg; + int8_t resetOffsetCfg; uint64_t consumerId; bool hbBgEnable; tmq_commit_cb* commitCb; void* commitCbUserParam; // status + SRWLatch lock; int8_t status; int32_t epoch; #if 0 @@ -96,6 +99,7 @@ struct tmq_t { // poll info int64_t pollCnt; int64_t totalRows; +// bool needReportOffsetRows; // timer tmr_h hbLiveTimer; @@ -135,7 +139,8 @@ enum { typedef struct SVgOffsetInfo { STqOffsetVal committedOffset; - STqOffsetVal currentOffset; + STqOffsetVal endOffset; // the last version in TAOS_RES + 1 + STqOffsetVal beginOffset; // the first version in TAOS_RES int64_t walVerBegin; int64_t walVerEnd; } SVgOffsetInfo; @@ -146,9 +151,10 @@ typedef struct { SVgOffsetInfo offsetInfo; int32_t vgId; int32_t vgStatus; - int32_t vgSkipCnt; // here used to mark the slow vgroups - bool receiveInfo; - int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data + int32_t vgSkipCnt; // here used to mark the slow vgroups +// bool receivedInfoFromVnode; // has already received info from vnode + int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data + bool seekUpdated; // offset is updated by seek operator, therefore, not update by vnode rsp. SEpSet epSet; } SMqClientVg; @@ -163,6 +169,7 @@ typedef struct { int8_t tmqRspType; int32_t epoch; // epoch can be used to guard the vgHandle int32_t vgId; + char topicName[TSDB_TOPIC_FNAME_LEN]; SMqClientVg* vgHandle; SMqClientTopic* topicHandle; uint64_t reqId; @@ -175,8 +182,8 @@ typedef struct { } SMqPollRspWrapper; typedef struct { - int64_t refId; - int32_t epoch; +// int64_t refId; +// int32_t epoch; tsem_t rspSem; int32_t rspErr; } SMqSubscribeCbParam; @@ -191,8 +198,9 @@ typedef struct { typedef struct { int64_t refId; int32_t epoch; - SMqClientVg* pVg; - SMqClientTopic* pTopic; + char topicName[TSDB_TOPIC_FNAME_LEN]; +// SMqClientVg* pVg; +// SMqClientTopic* pTopic; int32_t vgId; uint64_t requestId; // request id for debug purpose } SMqPollCbParam; @@ -207,6 +215,17 @@ typedef struct SMqVgCommon { int32_t code; } SMqVgCommon; +typedef struct SMqSeekParam { + tsem_t sem; + int32_t code; +} SMqSeekParam; + +typedef struct SMqCommittedParam { + tsem_t sem; + int32_t code; + SMqVgOffset vgOffset; +} SMqCommittedParam; + typedef struct SMqVgWalInfoParam { int32_t vgId; int32_t epoch; @@ -228,7 +247,7 @@ typedef struct { typedef struct { SMqCommitCbParamSet* params; - SMqVgOffset* pOffset; +// SMqVgOffset* pOffset; char topicName[TSDB_TOPIC_FNAME_LEN]; int32_t vgId; tmq_t* pTmq; @@ -242,8 +261,7 @@ typedef struct SSyncCommitInfo { static int32_t doAskEp(tmq_t* tmq); static int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg); static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet); -static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicName, SMqCommitCbParamSet* pParamSet, - int32_t index, int32_t totalVgroups, int32_t type); +static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, SMqCommitCbParamSet* pParamSet); static void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId); static void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param); static void addToQueueCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param); @@ -258,7 +276,7 @@ tmq_conf_t* tmq_conf_new() { conf->withTbName = false; conf->autoCommit = true; conf->autoCommitInterval = DEFAULT_AUTO_COMMIT_INTERVAL; - conf->resetOffset = TMQ_OFFSET__RESET_EARLIEAST; + conf->resetOffset = TMQ_OFFSET__RESET_EARLIEST; conf->hbBgEnable = true; return conf; @@ -280,6 +298,9 @@ void tmq_conf_destroy(tmq_conf_t* conf) { } tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { + if (conf == NULL || key == NULL || value == NULL){ + return TMQ_CONF_INVALID; + } if (strcasecmp(key, "group.id") == 0) { tstrncpy(conf->groupId, value, TSDB_CGROUP_LEN); return TMQ_CONF_OK; @@ -312,7 +333,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value conf->resetOffset = TMQ_OFFSET__RESET_NONE; return TMQ_CONF_OK; } else if (strcasecmp(value, "earliest") == 0) { - conf->resetOffset = TMQ_OFFSET__RESET_EARLIEAST; + conf->resetOffset = TMQ_OFFSET__RESET_EARLIEST; return TMQ_CONF_OK; } else if (strcasecmp(value, "latest") == 0) { conf->resetOffset = TMQ_OFFSET__RESET_LATEST; @@ -351,7 +372,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_OK; } - if (strcasecmp(key, "enable.heartbeat.background") == 0) { +// if (strcasecmp(key, "enable.heartbeat.background") == 0) { // if (strcasecmp(value, "true") == 0) { // conf->hbBgEnable = true; // return TMQ_CONF_OK; @@ -359,10 +380,10 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value // conf->hbBgEnable = false; // return TMQ_CONF_OK; // } else { - tscError("the default value of enable.heartbeat.background is true, can not be seted"); - return TMQ_CONF_INVALID; +// tscError("the default value of enable.heartbeat.background is true, can not be seted"); +// return TMQ_CONF_INVALID; // } - } +// } if (strcasecmp(key, "td.connect.ip") == 0) { conf->ip = taosStrdup(value); @@ -394,6 +415,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value tmq_list_t* tmq_list_new() { return (tmq_list_t*)taosArrayInit(0, sizeof(void*)); } int32_t tmq_list_append(tmq_list_t* list, const char* src) { + if(list == NULL) return -1; SArray* container = &list->container; if (src == NULL || src[0] == 0) return -1; char* topic = taosStrdup(src); @@ -402,84 +424,28 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { } void tmq_list_destroy(tmq_list_t* list) { + if(list == NULL) return; SArray* container = &list->container; taosArrayDestroyP(container, taosMemoryFree); } int32_t tmq_list_get_size(const tmq_list_t* list) { + if(list == NULL) return -1; const SArray* container = &list->container; return taosArrayGetSize(container); } char** tmq_list_to_c_array(const tmq_list_t* list) { + if(list == NULL) return NULL; const SArray* container = &list->container; return container->pData; } -static SMqClientVg* foundClientVg(SArray* pTopicList, const char* pName, int32_t vgId, int32_t* index, - int32_t* numOfVgroups) { - int32_t numOfTopics = taosArrayGetSize(pTopicList); - *index = -1; - *numOfVgroups = 0; - - for (int32_t i = 0; i < numOfTopics; ++i) { - SMqClientTopic* pTopic = taosArrayGet(pTopicList, i); - if (strcmp(pTopic->topicName, pName) != 0) { - continue; - } - - *numOfVgroups = taosArrayGetSize(pTopic->vgs); - for (int32_t j = 0; j < (*numOfVgroups); ++j) { - SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); - if (pClientVg->vgId == vgId) { - *index = j; - return pClientVg; - } - } - } - - return NULL; -} - -// Two problems do not need to be addressed here -// 1. update to of epset. the response of poll request will automatically handle this problem -// 2. commit failure. This one needs to be resolved. static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; - // if (code != TSDB_CODE_SUCCESS) { // if commit offset failed, let's try again - // taosThreadMutexLock(&pParam->pTmq->lock); - // int32_t numOfVgroups, index; - // SMqClientVg* pVg = foundClientVg(pParam->pTmq->clientTopics, pParam->topicName, pParam->vgId, &index, - // &numOfVgroups); if (pVg == NULL) { - // tscDebug("consumer:0x%" PRIx64 - // " subKey:%s vgId:%d commit failed, code:%s has been transferred to other consumer, no need retry - // ordinal:%d/%d", pParam->pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, tstrerror(code), - // index + 1, numOfVgroups); - // } else { // let's retry the commit - // int32_t code1 = doSendCommitMsg(pParam->pTmq, pVg, pParam->topicName, pParamSet, index, numOfVgroups); - // if (code1 != TSDB_CODE_SUCCESS) { // retry failed. - // tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64 - // " retry failed, ignore this commit. code:%s ordinal:%d/%d", - // pParam->pTmq->consumerId, pParam->topicName, pVg->vgId, pVg->offsetInfo.committedOffset.version, - // tstrerror(terrno), index + 1, numOfVgroups); - // } - // } - // - // taosThreadMutexUnlock(&pParam->pTmq->lock); - // - // taosMemoryFree(pParam->pOffset); - // taosMemoryFree(pBuf->pData); - // taosMemoryFree(pBuf->pEpSet); - // - // commitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId); - // return 0; - // } - // - // // todo replace the pTmq with refId - - taosMemoryFree(pParam->pOffset); +// taosMemoryFree(pParam->pOffset); taosMemoryFree(pBuf->pData); taosMemoryFree(pBuf->pEpSet); @@ -487,54 +453,48 @@ static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { return 0; } -static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicName, SMqCommitCbParamSet* pParamSet, - int32_t index, int32_t totalVgroups, int32_t type) { - SMqVgOffset* pOffset = taosMemoryCalloc(1, sizeof(SMqVgOffset)); - if (pOffset == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } +static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, SMqCommitCbParamSet* pParamSet) { + SMqVgOffset pOffset = {0}; - pOffset->consumerId = tmq->consumerId; - pOffset->offset.val = pVg->offsetInfo.currentOffset; + pOffset.consumerId = tmq->consumerId; + pOffset.offset.val = *offset; int32_t groupLen = strlen(tmq->groupId); - memcpy(pOffset->offset.subKey, tmq->groupId, groupLen); - pOffset->offset.subKey[groupLen] = TMQ_SEPARATOR; - strcpy(pOffset->offset.subKey + groupLen + 1, pTopicName); + memcpy(pOffset.offset.subKey, tmq->groupId, groupLen); + pOffset.offset.subKey[groupLen] = TMQ_SEPARATOR; + strcpy(pOffset.offset.subKey + groupLen + 1, pTopicName); int32_t len = 0; int32_t code = 0; - tEncodeSize(tEncodeMqVgOffset, pOffset, len, code); + tEncodeSize(tEncodeMqVgOffset, &pOffset, len, code); if (code < 0) { return TSDB_CODE_INVALID_PARA; } void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); if (buf == NULL) { - taosMemoryFree(pOffset); return TSDB_CODE_OUT_OF_MEMORY; } - ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); + ((SMsgHead*)buf)->vgId = htonl(vgId); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); SEncoder encoder; tEncoderInit(&encoder, abuf, len); - tEncodeMqVgOffset(&encoder, pOffset); + tEncodeMqVgOffset(&encoder, &pOffset); tEncoderClear(&encoder); // build param SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam)); if (pParam == NULL) { - taosMemoryFree(pOffset); taosMemoryFree(buf); return TSDB_CODE_OUT_OF_MEMORY; } pParam->params = pParamSet; - pParam->pOffset = pOffset; - pParam->vgId = pVg->vgId; +// pParam->pOffset = pOffset; + pParam->vgId = vgId; pParam->pTmq = tmq; tstrncpy(pParam->topicName, pTopicName, tListLen(pParam->topicName)); @@ -542,7 +502,6 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN // build send info SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pMsgSendInfo == NULL) { - taosMemoryFree(pOffset); taosMemoryFree(buf); taosMemoryFree(pParam); return TSDB_CODE_OUT_OF_MEMORY; @@ -555,25 +514,16 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN pMsgSendInfo->param = pParam; pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->fp = tmqCommitCb; - pMsgSendInfo->msgType = type; + pMsgSendInfo->msgType = TDMT_VND_TMQ_COMMIT_OFFSET; atomic_add_fetch_32(&pParamSet->waitingRspNum, 1); atomic_add_fetch_32(&pParamSet->totalRspNum, 1); - SEp* pEp = GET_ACTIVE_EP(&pVg->epSet); - char offsetBuf[80] = {0}; - tFormatOffset(offsetBuf, tListLen(offsetBuf), &pOffset->offset.val); + SEp* pEp = GET_ACTIVE_EP(epSet); - char commitBuf[80] = {0}; - tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); - tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d send offset:%s prev:%s, ep:%s:%d, ordinal:%d/%d, req:0x%" PRIx64, - tmq->consumerId, pOffset->offset.subKey, pVg->vgId, offsetBuf, commitBuf, pEp->fqdn, pEp->port, index + 1, - totalVgroups, pMsgSendInfo->requestId); int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo); - - return TSDB_CODE_SUCCESS; + return asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, &transporterId, pMsgSendInfo); } static SMqClientTopic* getTopicByName(tmq_t* tmq, const char* pTopicName) { @@ -591,156 +541,200 @@ static SMqClientTopic* getTopicByName(tmq_t* tmq, const char* pTopicName) { return NULL; } -static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, int32_t type, tmq_commit_cb* pCommitFp, void* userParam) { - char* pTopicName = NULL; - int32_t vgId = 0; - int32_t code = 0; - - if (pRes == NULL || tmq == NULL) { - pCommitFp(tmq, TSDB_CODE_INVALID_PARA, userParam); - return; - } - - if (TD_RES_TMQ(pRes)) { - SMqRspObj* pRspObj = (SMqRspObj*)pRes; - pTopicName = pRspObj->topic; - vgId = pRspObj->vgId; - } else if (TD_RES_TMQ_META(pRes)) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)pRes; - pTopicName = pMetaRspObj->topic; - vgId = pMetaRspObj->vgId; - } else if (TD_RES_TMQ_METADATA(pRes)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)pRes; - pTopicName = pRspObj->topic; - vgId = pRspObj->vgId; - } else { - pCommitFp(tmq, TSDB_CODE_TMQ_INVALID_MSG, userParam); - return; - } - +static SMqCommitCbParamSet* prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam, int32_t rspNum){ SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { - pCommitFp(tmq, TSDB_CODE_OUT_OF_MEMORY, userParam); - return; + return NULL; } pParamSet->refId = tmq->refId; pParamSet->epoch = tmq->epoch; pParamSet->callbackFn = pCommitFp; pParamSet->userParam = userParam; + pParamSet->waitingRspNum = rspNum; + + return pParamSet; +} - int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); - tscDebug("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); +static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClientVg** pVg){ SMqClientTopic* pTopic = getTopicByName(tmq, pTopicName); if (pTopic == NULL) { - tscWarn("consumer:0x%" PRIx64 " failed to find the specified topic:%s, total topics:%d", tmq->consumerId, - pTopicName, numOfTopics); - taosMemoryFree(pParamSet); - pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); - return; + tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); + return TSDB_CODE_TMQ_INVALID_TOPIC; } - int32_t j = 0; - int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); - for (j = 0; j < numOfVgroups; j++) { - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg->vgId == vgId) { + int32_t numOfVgs = taosArrayGetSize(pTopic->vgs); + for (int32_t i = 0; i < numOfVgs; ++i) { + SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); + if (pClientVg->vgId == vgId) { + *pVg = pClientVg; break; } } - if (j == numOfVgroups) { - tscWarn("consumer:0x%" PRIx64 " failed to find the specified vgId:%d, total Vgs:%d, topic:%s", tmq->consumerId, - vgId, numOfVgroups, pTopicName); - taosMemoryFree(pParamSet); - pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); - return; + return *pVg == NULL ? TSDB_CODE_TMQ_INVALID_VGID : TSDB_CODE_SUCCESS; +} + +static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STqOffsetVal* offsetVal, tmq_commit_cb* pCommitFp, void* userParam) { + int32_t code = 0; + tscInfo("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); + taosRLockLatch(&tmq->lock); + SMqClientVg* pVg = NULL; + code = getClientVg(tmq, pTopicName, vgId, &pVg); + if(code != 0){ + goto end; + } + if (offsetVal->type <= 0) { + code = TSDB_CODE_TMQ_INVALID_MSG; + goto end; } + if (tOffsetEqual(offsetVal, &pVg->offsetInfo.committedOffset)){ + code = TSDB_CODE_TMQ_SAME_COMMITTED_VALUE; + goto end; + } + char offsetBuf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(offsetBuf, tListLen(offsetBuf), offsetVal); - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg->offsetInfo.currentOffset.type > 0 && !tOffsetEqual(&pVg->offsetInfo.currentOffset, &pVg->offsetInfo.committedOffset)) { - code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups, type); + char commitBuf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); - // failed to commit, callback user function directly. - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFree(pParamSet); - pCommitFp(tmq, code, userParam); - } - } else { // do not perform commit, callback user function directly. + SMqCommitCbParamSet* pParamSet = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 0); + if (pParamSet == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, offsetVal, pTopicName, pParamSet); + if (code != TSDB_CODE_SUCCESS) { + tscError("consumer:0x%" PRIx64 " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s", + tmq->consumerId, pTopicName, pVg->vgId, offsetBuf, commitBuf, tstrerror(terrno)); taosMemoryFree(pParamSet); - pCommitFp(tmq, code, userParam); + goto end; } + + tscInfo("consumer:0x%" PRIx64 " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s", + tmq->consumerId, pTopicName, pVg->vgId, offsetBuf, commitBuf); + pVg->offsetInfo.committedOffset = *offsetVal; + +end: + taosRUnLockLatch(&tmq->lock); + return code; } -static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) { - SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); - if (pParamSet == NULL) { - pCommitFp(tmq, TSDB_CODE_OUT_OF_MEMORY, userParam); - return; +static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* pCommitFp, void* userParam){ + char* pTopicName = NULL; + int32_t vgId = 0; + STqOffsetVal offsetVal = {0}; + int32_t code = 0; + + if (pRes == NULL || tmq == NULL) { + code = TSDB_CODE_INVALID_PARA; + goto end; } - pParamSet->refId = tmq->refId; - pParamSet->epoch = tmq->epoch; - pParamSet->callbackFn = pCommitFp; - pParamSet->userParam = userParam; + if (TD_RES_TMQ(pRes)) { + SMqRspObj* pRspObj = (SMqRspObj*)pRes; + pTopicName = pRspObj->topic; + vgId = pRspObj->vgId; + offsetVal = pRspObj->rsp.rspOffset; + } else if (TD_RES_TMQ_META(pRes)) { + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)pRes; + pTopicName = pMetaRspObj->topic; + vgId = pMetaRspObj->vgId; + offsetVal = pMetaRspObj->metaRsp.rspOffset; + } else if (TD_RES_TMQ_METADATA(pRes)) { + SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)pRes; + pTopicName = pRspObj->topic; + vgId = pRspObj->vgId; + offsetVal = pRspObj->rsp.rspOffset; + } else { + code = TSDB_CODE_TMQ_INVALID_MSG; + goto end; + } + code = asyncCommitOffset(tmq, pTopicName, vgId, &offsetVal, pCommitFp, userParam); + +end: + if(code != TSDB_CODE_SUCCESS && pCommitFp != NULL){ + if(code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS; + pCommitFp(tmq, code, userParam); + } +} + +static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) { + int32_t code = 0; // init as 1 to prevent concurrency issue - pParamSet->waitingRspNum = 1; + SMqCommitCbParamSet* pParamSet = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 1); + if (pParamSet == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + taosRLockLatch(&tmq->lock); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); - tscDebug("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics); + tscInfo("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics); for (int32_t i = 0; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); - tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, - numOfVgroups); + tscInfo("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, numOfVgroups); for (int32_t j = 0; j < numOfVgroups; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg->offsetInfo.currentOffset.type > 0 && !tOffsetEqual(&pVg->offsetInfo.currentOffset, &pVg->offsetInfo.committedOffset)) { - int32_t code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups, TDMT_VND_TMQ_COMMIT_OFFSET); + if (pVg->offsetInfo.endOffset.type > 0 && !tOffsetEqual(&pVg->offsetInfo.endOffset, &pVg->offsetInfo.committedOffset)) { + char offsetBuf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(offsetBuf, tListLen(offsetBuf), &pVg->offsetInfo.endOffset); + + char commitBuf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); + + code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, &pVg->offsetInfo.endOffset, pTopic->topicName, pParamSet); if (code != TSDB_CODE_SUCCESS) { - tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64 " failed, code:%s ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.committedOffset.version, tstrerror(terrno), - j + 1, numOfVgroups); + tscError("consumer:0x%" PRIx64 " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s ordinal:%d/%d", + tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, tstrerror(terrno), j + 1, numOfVgroups); continue; } - // update the offset value. - pVg->offsetInfo.committedOffset = pVg->offsetInfo.currentOffset; + tscInfo("consumer:0x%" PRIx64 " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s, ordinal:%d/%d", + tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, j + 1, numOfVgroups); + pVg->offsetInfo.committedOffset = pVg->offsetInfo.endOffset; } else { - tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, current:%" PRId64 ", ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.currentOffset.version, j + 1, numOfVgroups); + tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, current:%" PRId64 ", ordinal:%d/%d", + tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups); } } } + taosRUnLockLatch(&tmq->lock); - tscDebug("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1, - numOfTopics); + tscInfo("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1, numOfTopics); - // no request is sent - if (pParamSet->totalRspNum == 0) { - taosMemoryFree(pParamSet); - pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); + // request is sent + if (pParamSet->totalRspNum != 0) { + // count down since waiting rsp num init as 1 + commitRspCountDown(pParamSet, tmq->consumerId, "", 0); return; } - // count down since waiting rsp num init as 1 - commitRspCountDown(pParamSet, tmq->consumerId, "", 0); +end: + taosMemoryFree(pParamSet); + if(pCommitFp != NULL) { + pCommitFp(tmq, code, userParam); + } + return; } static void generateTimedTask(int64_t refId, int32_t type) { tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); - if (tmq != NULL) { - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); - *pTaskType = type; - taosWriteQitem(tmq->delayedTask, pTaskType); - tsem_post(&tmq->rspSem); - } + if(tmq == NULL) return; + + int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); + if(pTaskType == NULL) return; + + *pTaskType = type; + taosWriteQitem(tmq->delayedTask, pTaskType); + tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); } @@ -756,19 +750,19 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) { taosMemoryFree(param); } -void tmqAssignDelayedReportTask(void* param, void* tmrId) { - int64_t refId = *(int64_t*)param; - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); - if (tmq != NULL) { - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); - *pTaskType = TMQ_DELAYED_TASK__REPORT; - taosWriteQitem(tmq->delayedTask, pTaskType); - tsem_post(&tmq->rspSem); - } - - taosReleaseRef(tmqMgmt.rsetId, refId); - taosMemoryFree(param); -} +//void tmqAssignDelayedReportTask(void* param, void* tmrId) { +// int64_t refId = *(int64_t*)param; +// tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); +// if (tmq != NULL) { +// int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); +// *pTaskType = TMQ_DELAYED_TASK__REPORT; +// taosWriteQitem(tmq->delayedTask, pTaskType); +// tsem_post(&tmq->rspSem); +// } +// +// taosReleaseRef(tmqMgmt.rsetId, refId); +// taosMemoryFree(param); +//} int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { if (pMsg) { @@ -790,6 +784,29 @@ void tmqSendHbReq(void* param, void* tmrId) { SMqHbReq req = {0}; req.consumerId = tmq->consumerId; req.epoch = tmq->epoch; + taosRLockLatch(&tmq->lock); +// if(tmq->needReportOffsetRows){ + req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows)); + for(int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++){ + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); + TopicOffsetRows* data = taosArrayReserve(req.topics, 1); + strcpy(data->topicName, pTopic->topicName); + data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows)); + for(int j = 0; j < numOfVgroups; j++){ + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1); + offRows->vgId = pVg->vgId; + offRows->rows = pVg->numOfRows; + offRows->offset = pVg->offsetInfo.beginOffset; + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset); + tscInfo("consumer:0x%" PRIx64 ",report offset: vgId:%d, offset:%s, rows:%"PRId64, tmq->consumerId, offRows->vgId, buf, offRows->rows); + } + } +// tmq->needReportOffsetRows = false; +// } + taosRUnLockLatch(&tmq->lock); int32_t tlen = tSerializeSMqHbReq(NULL, 0, &req); if (tlen < 0) { @@ -829,13 +846,14 @@ void tmqSendHbReq(void* param, void* tmrId) { asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); OVER: + tDeatroySMqHbReq(&req); taosTmrReset(tmqSendHbReq, 1000, param, tmqMgmt.timer, &tmq->hbLiveTimer); taosReleaseRef(tmqMgmt.rsetId, refId); } static void defaultCommitCbFn(tmq_t* pTmq, int32_t code, void* param) { if (code != 0) { - tscDebug("consumer:0x%" PRIx64 ", failed to commit offset, code:%s", pTmq->consumerId, tstrerror(code)); + tscError("consumer:0x%" PRIx64 ", failed to commit offset, code:%s", pTmq->consumerId, tstrerror(code)); } } @@ -888,7 +906,7 @@ static void* tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; tDeleteSMqAskEpRsp(&pEpRspWrapper->msg); - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; taosMemoryFreeClear(pRsp->pEpset); @@ -901,7 +919,7 @@ static void* tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { taosMemoryFreeClear(pRsp->pEpset); taosMemoryFree(pRsp->metaRsp.metaRsp); - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) { + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; taosMemoryFreeClear(pRsp->pEpset); @@ -952,17 +970,29 @@ int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) { } int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { + if(tmq == NULL) return TSDB_CODE_INVALID_PARA; if (*topics == NULL) { *topics = tmq_list_new(); } + taosRLockLatch(&tmq->lock); for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i); tmq_list_append(*topics, strchr(topic->topicName, '.') + 1); } + taosRUnLockLatch(&tmq->lock); return 0; } int32_t tmq_unsubscribe(tmq_t* tmq) { + if(tmq == NULL) return TSDB_CODE_INVALID_PARA; + if (tmq->autoCommit) { + int32_t rsp = tmq_commit_sync(tmq, NULL); + if (rsp != 0) { + return rsp; + } + } + taosSsleep(2); // sleep 2s for hb to send offset and rows to server + int32_t rsp; int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); @@ -1025,6 +1055,7 @@ static void tmqMgmtInit(void) { } tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { + if(conf == NULL) return NULL; taosThreadOnce(&tmqInit, tmqMgmtInit); if (tmqInitRes != 0) { terrno = tmqInitRes; @@ -1057,6 +1088,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->status = TMQ_CONSUMER_STATUS__INIT; pTmq->pollCnt = 0; pTmq->epoch = 0; +// pTmq->needReportOffsetRows = true; // set conf strcpy(pTmq->clientId, conf->clientId); @@ -1068,6 +1100,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->commitCb = conf->commitCb; pTmq->commitCbUserParam = conf->commitCbUserParam; pTmq->resetOffsetCfg = conf->resetOffset; + taosInitRWLatch(&pTmq->lock); pTmq->hbBgEnable = conf->hbBgEnable; @@ -1100,7 +1133,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pRefId, tmqMgmt.timer); } - char buf[80] = {0}; + char buf[TSDB_OFFSET_LEN] = {0}; STqOffsetVal offset = {.type = pTmq->resetOffsetCfg}; tFormatOffset(buf, tListLen(buf), &offset); tscInfo("consumer:0x%" PRIx64 " is setup, refId:%" PRId64 @@ -1116,7 +1149,8 @@ _failed: } int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { - const int32_t MAX_RETRY_COUNT = 120 * 4; // let's wait for 4 mins at most + if(tmq == NULL) return TSDB_CODE_INVALID_PARA; + const int32_t MAX_RETRY_COUNT = 120 * 2; // let's wait for 2 mins at most const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); void* buf = NULL; @@ -1124,7 +1158,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SCMSubscribeReq req = {0}; int32_t code = 0; - tscDebug("consumer:0x%" PRIx64 " cgroup:%s, subscribe %d topics", tmq->consumerId, tmq->groupId, sz); + tscInfo("consumer:0x%" PRIx64 " cgroup:%s, subscribe %d topics", tmq->consumerId, tmq->groupId, sz); req.consumerId = tmq->consumerId; tstrncpy(req.clientId, tmq->clientId, 256); @@ -1136,6 +1170,11 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { goto FAIL; } + req.withTbName = tmq->withTbName; + req.autoCommit = tmq->autoCommit; + req.autoCommitInterval = tmq->autoCommitInterval; + req.resetOffsetCfg = tmq->resetOffsetCfg; + for (int32_t i = 0; i < sz; i++) { char* topic = taosArrayGetP(container, i); @@ -1147,7 +1186,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } tNameExtractFullName(&name, topicFName); - tscDebug("consumer:0x%" PRIx64 " subscribe topic:%s", tmq->consumerId, topicFName); + tscInfo("consumer:0x%" PRIx64 " subscribe topic:%s", tmq->consumerId, topicFName); taosArrayPush(req.topicNames, &topicFName); } @@ -1169,7 +1208,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { goto FAIL; } - SMqSubscribeCbParam param = { .rspErr = 0, .refId = tmq->refId, .epoch = tmq->epoch }; + SMqSubscribeCbParam param = { .rspErr = 0}; if (tsem_init(¶m.rspSem, 0, 0) != 0) { code = TSDB_CODE_TSC_INTERNAL_ERROR; goto FAIL; @@ -1203,12 +1242,12 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { int32_t retryCnt = 0; while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) { if (retryCnt++ > MAX_RETRY_COUNT) { - tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, max retry reached:%d", tmq->consumerId, retryCnt); - code = TSDB_CODE_TSC_INTERNAL_ERROR; + tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); + code = TSDB_CODE_MND_CONSUMER_NOT_READY; goto FAIL; } - tscDebug("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); + tscInfo("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); taosMsleep(500); } @@ -1235,16 +1274,45 @@ FAIL: } void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* param) { + if(conf == NULL) return; conf->commitCb = cb; conf->commitCbUserParam = param; } -static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { +static SMqClientVg* getVgInfo(tmq_t* tmq, char* topicName, int32_t vgId){ + int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); + for(int i = 0; i < topicNumCur; i++){ + SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); + if(strcmp(pTopicCur->topicName, topicName) == 0){ + int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); + for (int32_t j = 0; j < vgNumCur; j++) { + SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); + if(pVgCur->vgId == vgId){ + return pVgCur; + } + } + } + } + return NULL; +} + +static SMqClientTopic* getTopicInfo(tmq_t* tmq, char* topicName){ + int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); + for(int i = 0; i < topicNumCur; i++){ + SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); + if(strcmp(pTopicCur->topicName, topicName) == 0){ + return pTopicCur; + } + } + return NULL; +} + +int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SMqPollCbParam* pParam = (SMqPollCbParam*)param; int64_t refId = pParam->refId; - SMqClientVg* pVg = pParam->pVg; - SMqClientTopic* pTopic = pParam->pTopic; +// SMqClientVg* pVg = pParam->pVg; +// SMqClientTopic* pTopic = pParam->pTopic; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { @@ -1259,15 +1327,13 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t vgId = pParam->vgId; uint64_t requestId = pParam->requestId; - taosMemoryFree(pParam); - if (code != 0) { if (pMsg->pData) taosMemoryFree(pMsg->pData); if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet); // in case of consumer mismatch, wait for 500ms and retry if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { - taosMsleep(500); +// taosMsleep(500); atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER); tscDebug("consumer:0x%" PRIx64 " wait for the re-balance, wait for 500ms and set status to be RECOVER", tmq->consumerId); @@ -1281,8 +1347,8 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP; taosWriteQitem(tmq->mqueue, pRspWrapper); - } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert - taosMsleep(500); +// } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert +// taosMsleep(5); } else{ tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId, vgId, epoch, tstrerror(code), requestId); @@ -1304,6 +1370,8 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); + taosMemoryFree(pParam); + return 0; } @@ -1325,22 +1393,23 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } pRspWrapper->tmqRspType = rspType; - pRspWrapper->vgHandle = pVg; - pRspWrapper->topicHandle = pTopic; +// pRspWrapper->vgHandle = pVg; +// pRspWrapper->topicHandle = pTopic; pRspWrapper->reqId = requestId; pRspWrapper->pEpset = pMsg->pEpSet; - pRspWrapper->vgId = pVg->vgId; + pRspWrapper->vgId = vgId; + strcpy(pRspWrapper->topicName, pParam->topicName); pMsg->pEpSet = NULL; - if (rspType == TMQ_MSG_TYPE__POLL_RSP) { + if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp); tDecoderClear(&decoder); memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); - char buf[80]; - tFormatOffset(buf, 80, &pRspWrapper->dataRsp.rspOffset); + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.rspOffset); tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64, tmq->consumerId, vgId, pRspWrapper->dataRsp.reqOffset.version, buf, rspType, requestId); } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { @@ -1349,7 +1418,7 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tDecodeMqMetaRsp(&decoder, &pRspWrapper->metaRsp); tDecoderClear(&decoder); memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead)); - } else if (rspType == TMQ_MSG_TYPE__TAOSX_RSP) { + } else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp); @@ -1368,22 +1437,31 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); + taosMemoryFree(pParam); return 0; CREATE_MSG_FAIL: if (epoch == tmq->epoch) { - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = getVgInfo(tmq, pParam->topicName, vgId); + if(pVg){ + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + } + taosWUnLockLatch(&tmq->lock); } tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); + taosMemoryFree(pParam); return -1; } typedef struct SVgroupSaveInfo { - STqOffsetVal offset; + STqOffsetVal currentOffset; + STqOffsetVal commitOffset; + STqOffsetVal seekOffset; int64_t numOfRows; } SVgroupSaveInfo; @@ -1399,7 +1477,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic tstrncpy(pTopic->topicName, pTopicEp->topic, TSDB_TOPIC_FNAME_LEN); tstrncpy(pTopic->db, pTopicEp->db, TSDB_DB_FNAME_LEN); - tscDebug("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d", tmq->consumerId, pTopic->topicName, vgNumGet); + tscInfo("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d", tmq->consumerId, pTopic->topicName, vgNumGet); pTopic->vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); for (int32_t j = 0; j < vgNumGet; j++) { @@ -1408,12 +1486,8 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic makeTopicVgroupKey(vgKey, pTopic->topicName, pVgEp->vgId); SVgroupSaveInfo* pInfo = taosHashGet(pVgOffsetHashMap, vgKey, strlen(vgKey)); - int64_t numOfRows = 0; - STqOffsetVal offsetNew = {.type = tmq->resetOffsetCfg}; - if (pInfo != NULL) { - offsetNew = pInfo->offset; - numOfRows = pInfo->numOfRows; - } + STqOffsetVal offsetNew = {0}; + offsetNew.type = tmq->resetOffsetCfg; SMqClientVg clientVg = { .pollCnt = 0, @@ -1422,13 +1496,17 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic .vgStatus = TMQ_VG_STATUS__IDLE, .vgSkipCnt = 0, .emptyBlockReceiveTs = 0, - .numOfRows = numOfRows, + .numOfRows = pInfo ? pInfo->numOfRows : 0, }; - clientVg.offsetInfo.currentOffset = offsetNew; - clientVg.offsetInfo.committedOffset = offsetNew; + clientVg.offsetInfo.endOffset = pInfo ? pInfo->currentOffset : offsetNew; + clientVg.offsetInfo.committedOffset = pInfo ? pInfo->commitOffset : offsetNew; + clientVg.offsetInfo.beginOffset = pInfo ? pInfo->seekOffset : offsetNew; clientVg.offsetInfo.walVerBegin = -1; clientVg.offsetInfo.walVerEnd = -1; + clientVg.seekUpdated = false; +// clientVg.receivedInfoFromVnode = false; + taosArrayPush(pTopic->vgs, &clientVg); } } @@ -1445,12 +1523,7 @@ static void freeClientVgInfo(void* param) { static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { bool set = false; - int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); int32_t topicNumGet = taosArrayGetSize(pRsp->topics); - - char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; - tscDebug("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d", - tmq->consumerId, tmq->epoch, epoch, topicNumGet, topicNumCur); if (epoch <= tmq->epoch) { return false; } @@ -1466,23 +1539,29 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) return false; } + taosWLockLatch(&tmq->lock); + int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); + + char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; + tscInfo("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d", + tmq->consumerId, tmq->epoch, epoch, topicNumGet, topicNumCur); // todo extract method for (int32_t i = 0; i < topicNumCur; i++) { // find old topic SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); if (pTopicCur->vgs) { int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); - tscDebug("consumer:0x%" PRIx64 ", current vg num: %d", tmq->consumerId, vgNumCur); + tscInfo("consumer:0x%" PRIx64 ", current vg num: %d", tmq->consumerId, vgNumCur); for (int32_t j = 0; j < vgNumCur; j++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); makeTopicVgroupKey(vgKey, pTopicCur->topicName, pVgCur->vgId); - char buf[80]; - tFormatOffset(buf, 80, &pVgCur->offsetInfo.currentOffset); - tscDebug("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.endOffset); + tscInfo("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, buf); - SVgroupSaveInfo info = {.offset = pVgCur->offsetInfo.currentOffset, .numOfRows = pVgCur->numOfRows}; + SVgroupSaveInfo info = {.currentOffset = pVgCur->offsetInfo.endOffset, .seekOffset = pVgCur->offsetInfo.beginOffset, .commitOffset = pVgCur->offsetInfo.committedOffset, .numOfRows = pVgCur->numOfRows}; taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)); } } @@ -1502,12 +1581,13 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) taosArrayDestroyEx(tmq->clientTopics, freeClientVgInfo); } tmq->clientTopics = newTopics; + taosWUnLockLatch(&tmq->lock); int8_t flag = (topicNumGet == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY; atomic_store_8(&tmq->status, flag); atomic_store_32(&tmq->epoch, epoch); - tscDebug("consumer:0x%" PRIx64 " update topic info completed", tmq->consumerId); + tscInfo("consumer:0x%" PRIx64 " update topic info completed", tmq->consumerId); return set; } @@ -1517,7 +1597,7 @@ int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { if (tmq == NULL) { terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; - pParam->pUserFn(tmq, terrno, NULL, pParam->pParam); +// pParam->pUserFn(tmq, terrno, NULL, pParam->pParam); taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -1543,7 +1623,7 @@ int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { SMqRspHead* head = pMsg->pData; int32_t epoch = atomic_load_32(&tmq->epoch); if (head->epoch <= epoch) { - tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, no need to update local ep", + tscInfo("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, no need to update local ep", tmq->consumerId, head->epoch, epoch); if (tmq->status == TMQ_CONSUMER_STATUS__RECOVER) { @@ -1555,7 +1635,7 @@ int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { } } else { - tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, update local ep", tmq->consumerId, + tscInfo("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, update local ep", tmq->consumerId, head->epoch, epoch); } @@ -1578,7 +1658,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl pReq->consumerId = tmq->consumerId; pReq->timeout = timeout; pReq->epoch = tmq->epoch; - pReq->reqOffset = pVg->offsetInfo.currentOffset; + pReq->reqOffset = pVg->offsetInfo.endOffset; pReq->head.vgId = pVg->vgId; pReq->useSnapshot = tmq->useSnapshot; pReq->reqId = generateRequestId(); @@ -1625,7 +1705,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, return pRspObj; } -SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper) { +SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); pRspObj->resType = RES_TYPE__TMQ_METADATA; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); @@ -1640,6 +1720,13 @@ SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper) { setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); } + // extract the rows in this data packet + for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, i); + int64_t rows = htobe64(pRetrieve->numOfRows); + pVg->numOfRows += rows; + (*numOfRows) += rows; + } return pRspObj; } @@ -1676,8 +1763,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p pParam->refId = pTmq->refId; pParam->epoch = pTmq->epoch; - pParam->pVg = pVg; - pParam->pTopic = pTopic; +// pParam->pVg = pVg; // pVg may be released,fix it +// pParam->pTopic = pTopic; + strcpy(pParam->topicName, pTopic->topicName); pParam->vgId = pVg->vgId; pParam->requestId = req.reqId; @@ -1696,14 +1784,15 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p sendInfo->msgType = TDMT_VND_TMQ_CONSUME; int64_t transporterId = 0; - char offsetFormatBuf[80]; - tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.currentOffset); + char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.endOffset); tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, pTmq->consumerId, pTopic->topicName, pVg->vgId, pTmq->epoch, offsetFormatBuf, req.reqId); asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; + pVg->seekUpdated = false; // reset this flag. pTmq->pollCnt++; return TSDB_CODE_SUCCESS; @@ -1711,6 +1800,12 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p // broadcast the poll request to all related vnodes static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { + if(atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER){ + return 0; + } + int32_t code = 0; + + taosWLockLatch(&tmq->lock); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); tscDebug("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics); @@ -1720,7 +1815,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { for (int j = 0; j < numOfVg; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 100ms + if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 10ms tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, tmq->epoch, pVg->vgId); continue; @@ -1735,15 +1830,17 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { } atomic_store_32(&pVg->vgSkipCnt, 0); - int32_t code = doTmqPollImpl(tmq, pTopic, pVg, timeout); + code = doTmqPollImpl(tmq, pTopic, pVg, timeout); if (code != TSDB_CODE_SUCCESS) { - return code; + goto end; } } } - tscDebug("consumer:0x%" PRIx64 " end to poll data", tmq->consumerId); - return 0; +end: + taosWUnLockLatch(&tmq->lock); + tscDebug("consumer:0x%" PRIx64 " end to poll data, code:%d", tmq->consumerId, code); + return code; } static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) { @@ -1766,6 +1863,24 @@ static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* p return 0; } +static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal* rspOffset, int64_t sver, int64_t ever, int64_t consumerId, bool hasData){ + if (!pVg->seekUpdated) { + tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", consumerId); + if(hasData) pVg->offsetInfo.beginOffset = *reqOffset; + pVg->offsetInfo.endOffset = *rspOffset; + } else { + tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", consumerId); + } + + // update the status + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + + // update the valid wal version range + pVg->offsetInfo.walVerBegin = sver; + pVg->offsetInfo.walVerEnd = ever + 1; +// pVg->receivedInfoFromVnode = true; +} + static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, tmq->qall->numOfItems); @@ -1788,15 +1903,23 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; tscError("consumer:0x%" PRIx64 " unexpected rsp from poll, code:%s", tmq->consumerId, tstrerror(terrno)); return NULL; - } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; int32_t consumerEpoch = atomic_load_32(&tmq->epoch); SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp; if (pDataRsp->head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; - + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + taosWUnLockLatch(&tmq->lock); + return NULL; + } // update the epset if (pollRspWrapper->pEpset != NULL) { SEp* pEp = GET_ACTIVE_EP(pollRspWrapper->pEpset); @@ -1806,22 +1929,13 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { pVg->epSet = *pollRspWrapper->pEpset; } - // update the local offset value only for the returned values. - pVg->offsetInfo.currentOffset = pDataRsp->rspOffset; - - // update the status - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, pDataRsp->head.walsver, pDataRsp->head.walever, tmq->consumerId, pDataRsp->blockNum != 0); - // update the valid wal version range - pVg->offsetInfo.walVerBegin = pDataRsp->head.walsver; - pVg->offsetInfo.walVerEnd = pDataRsp->head.walever; - pVg->receiveInfo = true; - - char buf[80]; - tFormatOffset(buf, 80, &pDataRsp->rspOffset); + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset); if (pDataRsp->blockNum == 0) { tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64 - " total:%" PRId64 " reqId:0x%" PRIx64, + ", total:%" PRId64 ", reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); pRspWrapper = tmqFreeRspWrapper(pRspWrapper); pVg->emptyBlockReceiveTs = taosGetTimestampMs(); @@ -1832,14 +1946,16 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { tmq->totalRows += numOfRows; pVg->emptyBlockReceiveTs = 0; tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 - " vg total:%" PRId64 " total:%" PRId64 ", reqId:0x%" PRIx64, + ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); taosFreeQitem(pollRspWrapper); + taosWUnLockLatch(&tmq->lock); return pRsp; } + taosWUnLockLatch(&tmq->lock); } else { - tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", + tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch); pRspWrapper = tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pollRspWrapper); @@ -1852,38 +1968,54 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId); if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; - if(pollRspWrapper->metaRsp.rspOffset.type != 0){ // if offset is validate - pVg->offsetInfo.currentOffset = pollRspWrapper->metaRsp.rspOffset; + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + taosWUnLockLatch(&tmq->lock); + return NULL; } - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + + updateVgInfo(pVg, &pollRspWrapper->metaRsp.rspOffset, &pollRspWrapper->metaRsp.rspOffset, pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId, true); // build rsp SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); taosFreeQitem(pollRspWrapper); + taosWUnLockLatch(&tmq->lock); return pRsp; } else { - tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", + tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->metaRsp.head.epoch, consumerEpoch); pRspWrapper = tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pollRspWrapper); } - } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) { + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; int32_t consumerEpoch = atomic_load_32(&tmq->epoch); if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; - if(pollRspWrapper->taosxRsp.rspOffset.type != 0){ // if offset is validate - pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset; + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + taosWUnLockLatch(&tmq->lock); + return NULL; } - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + + updateVgInfo(pVg, &pollRspWrapper->taosxRsp.reqOffset, &pollRspWrapper->taosxRsp.rspOffset, pollRspWrapper->taosxRsp.head.walsver, pollRspWrapper->taosxRsp.head.walever, tmq->consumerId, pollRspWrapper->taosxRsp.blockNum != 0); if (pollRspWrapper->taosxRsp.blockNum == 0) { - tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 " reqId:0x%" PRIx64, + tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId); pVg->emptyBlockReceiveTs = taosGetTimestampMs(); pRspWrapper = tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pollRspWrapper); + taosWUnLockLatch(&tmq->lock); continue; } else { pVg->emptyBlockReceiveTs = 0; // reset the ts @@ -1893,25 +2025,25 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { void* pRsp = NULL; int64_t numOfRows = 0; if (pollRspWrapper->taosxRsp.createTableNum == 0) { - pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows); + tscError("consumer:0x%" PRIx64" createTableNum should > 0 if rsp type is data_meta", tmq->consumerId); } else { - pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper); + pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows); } tmq->totalRows += numOfRows; - char buf[80]; - tFormatOffset(buf, 80, &pVg->offsetInfo.currentOffset); - tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 - ", vg total:%" PRId64 " total:%" PRId64 " reqId:0x%" PRIx64, - tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows, - tmq->totalRows, pollRspWrapper->reqId); + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); + tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 + ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, + tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows, + tmq->totalRows, pollRspWrapper->reqId); taosFreeQitem(pollRspWrapper); + taosWUnLockLatch(&tmq->lock); return pRsp; - } else { - tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", + tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch); pRspWrapper = tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pollRspWrapper); @@ -1931,15 +2063,17 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { } TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { + if(tmq == NULL) return NULL; + void* rspObj; int64_t startTime = taosGetTimestampMs(); - tscDebug("consumer:0x%" PRIx64 " start to poll at %" PRId64 ", timeout:%" PRId64, tmq->consumerId, startTime, + tscInfo("consumer:0x%" PRIx64 " start to poll at %" PRId64 ", timeout:%" PRId64, tmq->consumerId, startTime, timeout); // in no topic status, delayed task also need to be processed if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) { - tscDebug("consumer:0x%" PRIx64 " poll return since consumer is init", tmq->consumerId); + tscInfo("consumer:0x%" PRIx64 " poll return since consumer is init", tmq->consumerId); taosMsleep(500); // sleep for a while return NULL; } @@ -1951,7 +2085,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { return NULL; } - tscDebug("consumer:0x%" PRIx64 " not ready, retry:%d/40 in 500ms", tmq->consumerId, retryCnt); + tscInfo("consumer:0x%" PRIx64 " not ready, retry:%d/40 in 500ms", tmq->consumerId, retryCnt); taosMsleep(500); } } @@ -1960,7 +2094,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { tmqHandleAllDelayedTask(tmq); if (tmqPollImpl(tmq, timeout) < 0) { - tscDebug("consumer:0x%" PRIx64 " return due to poll error", tmq->consumerId); + tscError("consumer:0x%" PRIx64 " return due to poll error", tmq->consumerId); } rspObj = tmqHandleAllRsp(tmq, timeout, false); @@ -1968,7 +2102,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { tscDebug("consumer:0x%" PRIx64 " return rsp %p", tmq->consumerId, rspObj); return (TAOS_RES*)rspObj; } else if (terrno == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { - tscDebug("consumer:0x%" PRIx64 " return null since no committed offset", tmq->consumerId); + tscInfo("consumer:0x%" PRIx64 " return null since no committed offset", tmq->consumerId); return NULL; } @@ -1976,7 +2110,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { int64_t currentTime = taosGetTimestampMs(); int64_t elapsedTime = currentTime - startTime; if (elapsedTime > timeout) { - tscDebug("consumer:0x%" PRIx64 " (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64, + tscInfo("consumer:0x%" PRIx64 " (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64, tmq->consumerId, tmq->epoch, startTime, currentTime); return NULL; } @@ -1988,7 +2122,8 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { } } -static void displayConsumeStatistics(const tmq_t* pTmq) { +static void displayConsumeStatistics(tmq_t* pTmq) { + taosRLockLatch(&pTmq->lock); int32_t numOfTopics = taosArrayGetSize(pTmq->clientTopics); tscDebug("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d", pTmq->consumerId, pTmq->pollCnt, pTmq->totalRows, numOfTopics, pTmq->epoch); @@ -2004,12 +2139,14 @@ static void displayConsumeStatistics(const tmq_t* pTmq) { tscDebug("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows); } } - + taosRUnLockLatch(&pTmq->lock); tscDebug("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId); } int32_t tmq_consumer_close(tmq_t* tmq) { - tscDebug("consumer:0x%" PRIx64 " start to close consumer, status:%d", tmq->consumerId, tmq->status); + if(tmq == NULL) return TSDB_CODE_INVALID_PARA; + + tscInfo("consumer:0x%" PRIx64 " start to close consumer, status:%d", tmq->consumerId, tmq->status); displayConsumeStatistics(tmq); if (tmq->status == TMQ_CONSUMER_STATUS__READY) { @@ -2020,6 +2157,7 @@ int32_t tmq_consumer_close(tmq_t* tmq) { return rsp; } } + taosSsleep(2); // sleep 2s for hb to send offset and rows to server int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); @@ -2035,7 +2173,7 @@ int32_t tmq_consumer_close(tmq_t* tmq) { tmq_list_destroy(lst); } else { - tscWarn("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId); + tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId); } taosRemoveRef(tmqMgmt.rsetId, tmq->refId); @@ -2053,6 +2191,9 @@ const char* tmq_err2str(int32_t err) { } tmq_res_t tmq_get_res_type(TAOS_RES* res) { + if (res == NULL){ + return TMQ_RES_INVALID; + } if (TD_RES_TMQ(res)) { return TMQ_RES_DATA; } else if (TD_RES_TMQ_META(res)) { @@ -2065,6 +2206,9 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) { } const char* tmq_get_topic_name(TAOS_RES* res) { + if (res == NULL){ + return NULL; + } if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; return strchr(pRspObj->topic, '.') + 1; @@ -2080,6 +2224,10 @@ const char* tmq_get_topic_name(TAOS_RES* res) { } const char* tmq_get_db_name(TAOS_RES* res) { + if (res == NULL){ + return NULL; + } + if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; return strchr(pRspObj->db, '.') + 1; @@ -2095,6 +2243,9 @@ const char* tmq_get_db_name(TAOS_RES* res) { } int32_t tmq_get_vgroup_id(TAOS_RES* res) { + if (res == NULL){ + return -1; + } if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; return pRspObj->vgId; @@ -2110,11 +2261,16 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { } int64_t tmq_get_vgroup_offset(TAOS_RES* res) { + if (res == NULL){ + return TSDB_CODE_INVALID_PARA; + } if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*) res; - STqOffsetVal* pOffset = &pRspObj->rsp.rspOffset; + STqOffsetVal* pOffset = &pRspObj->rsp.reqOffset; if (pOffset->type == TMQ_OFFSET__LOG) { - return pRspObj->rsp.rspOffset.version; + return pRspObj->rsp.reqOffset.version; + }else{ + tscError("invalid offset type:%d", pOffset->type); } } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pRspObj = (SMqMetaRspObj*)res; @@ -2123,16 +2279,21 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { } } else if (TD_RES_TMQ_METADATA(res)) { SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*) res; - if (pRspObj->rsp.rspOffset.type == TMQ_OFFSET__LOG) { - return pRspObj->rsp.rspOffset.version; + if (pRspObj->rsp.reqOffset.type == TMQ_OFFSET__LOG) { + return pRspObj->rsp.reqOffset.version; } + } else{ + tscError("invalid tmq type:%d", *(int8_t*)res); } // data from tsdb, no valid offset info - return -1; + return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } const char* tmq_get_table_name(TAOS_RES* res) { + if (res == NULL){ + return NULL; + } if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; if (!pRspObj->rsp.withTbName || pRspObj->rsp.blockTbName == NULL || pRspObj->resIter < 0 || @@ -2152,10 +2313,17 @@ const char* tmq_get_table_name(TAOS_RES* res) { } void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* param) { + if (tmq == NULL) { + tscError("invalid tmq handle, null"); + if(cb != NULL) { + cb(tmq, TSDB_CODE_INVALID_PARA, param); + } + return; + } if (pRes == NULL) { // here needs to commit all offsets. asyncCommitAllOffsets(tmq, cb, param); } else { // only commit one offset - asyncCommitOffset(tmq, pRes, TDMT_VND_TMQ_COMMIT_OFFSET, cb, param); + asyncCommitFromResult(tmq, pRes, cb, param); } } @@ -2166,6 +2334,11 @@ static void commitCallBackFn(tmq_t *UNUSED_PARAM(tmq), int32_t code, void* param } int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { + if (tmq == NULL) { + tscError("invalid tmq handle, null"); + return TSDB_CODE_INVALID_PARA; + } + int32_t code = 0; SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); @@ -2175,7 +2348,7 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { if (pRes == NULL) { asyncCommitAllOffsets(tmq, commitCallBackFn, pInfo); } else { - asyncCommitOffset(tmq, pRes, TDMT_VND_TMQ_COMMIT_OFFSET, commitCallBackFn, pInfo); + asyncCommitFromResult(tmq, pRes, commitCallBackFn, pInfo); } tsem_wait(&pInfo->sem); @@ -2184,10 +2357,118 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { tsem_destroy(&pInfo->sem); taosMemoryFree(pInfo); - tscDebug("consumer:0x%" PRIx64 " sync commit done, code:%s", tmq->consumerId, tstrerror(code)); + tscInfo("consumer:0x%" PRIx64 " sync res commit done, code:%s", tmq->consumerId, tstrerror(code)); + return code; +} + +// wal range will be ok after calling tmq_get_topic_assignment or poll interface +static int32_t checkWalRange(SVgOffsetInfo* offset, int64_t value){ + if (offset->walVerBegin == -1 || offset->walVerEnd == -1) { + tscError("Assignment or poll interface need to be called first"); + return TSDB_CODE_TMQ_NEED_INITIALIZED; + } + + if (value != -1 && (value < offset->walVerBegin || value > offset->walVerEnd)) { + tscError("invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", value, offset->walVerBegin, offset->walVerEnd); + return TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE; + } + + return 0; +} + +int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset){ + if (tmq == NULL || pTopicName == NULL) { + tscError("invalid tmq handle, null"); + return TSDB_CODE_INVALID_PARA; + } + + int32_t accId = tmq->pTscObj->acctId; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; + sprintf(tname, "%d.%s", accId, pTopicName); + + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = NULL; + int32_t code = getClientVg(tmq, tname, vgId, &pVg); + if(code != 0){ + taosWUnLockLatch(&tmq->lock); + return code; + } + + SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; + code = checkWalRange(pOffsetInfo, offset); + if (code != 0) { + taosWUnLockLatch(&tmq->lock); + return code; + } + taosWUnLockLatch(&tmq->lock); + + STqOffsetVal offsetVal = {.type = TMQ_OFFSET__LOG, .version = offset}; + + SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); + if (pInfo == NULL) { + tscError("consumer:0x%"PRIx64" failed to prepare seek operation", tmq->consumerId); + return TSDB_CODE_OUT_OF_MEMORY; + } + + tsem_init(&pInfo->sem, 0, 0); + pInfo->code = 0; + + code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, commitCallBackFn, pInfo); + if(code == 0){ + tsem_wait(&pInfo->sem); + code = pInfo->code; + } + + if(code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS; + tsem_destroy(&pInfo->sem); + taosMemoryFree(pInfo); + + tscInfo("consumer:0x%" PRIx64 " sync send commit to vgId:%d, offset:%" PRId64" code:%s", tmq->consumerId, vgId, offset, tstrerror(code)); + return code; } +void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param){ + int32_t code = 0; + if (tmq == NULL || pTopicName == NULL) { + tscError("invalid tmq handle, null"); + code = TSDB_CODE_INVALID_PARA; + goto end; + } + + int32_t accId = tmq->pTscObj->acctId; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; + sprintf(tname, "%d.%s", accId, pTopicName); + + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = NULL; + code = getClientVg(tmq, tname, vgId, &pVg); + if(code != 0){ + taosWUnLockLatch(&tmq->lock); + goto end; + } + + SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; + code = checkWalRange(pOffsetInfo, offset); + if (code != 0) { + taosWUnLockLatch(&tmq->lock); + goto end; + } + taosWUnLockLatch(&tmq->lock); + + STqOffsetVal offsetVal = {.type = TMQ_OFFSET__LOG, .version = offset}; + + code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, cb, param); + + tscInfo("consumer:0x%" PRIx64 " async send commit to vgId:%d, offset:%" PRId64" code:%s", tmq->consumerId, vgId, offset, tstrerror(code)); + +end: + if(code != 0 && cb != NULL){ + if(code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS; + cb(tmq, code, param); + } +} + void updateEpCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param) { SAskEpInfo* pInfo = param; pInfo->code = code; @@ -2298,7 +2579,7 @@ void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param) { sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp); - tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, reqId:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); + tscInfo("consumer:0x%" PRIx64 " ask ep from mnode, reqId:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); int64_t transporterId = 0; asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); @@ -2319,8 +2600,12 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { } // if no more waiting rsp - pParamSet->callbackFn(tmq, pParamSet->code, pParamSet->userParam); + if(pParamSet->callbackFn != NULL){ + pParamSet->callbackFn(tmq, pParamSet->code, pParamSet->userParam); + } + taosMemoryFree(pParamSet); +// tmq->needReportOffsetRows = true; taosReleaseRef(tmqMgmt.rsetId, refId); return 0; @@ -2329,12 +2614,10 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) { int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); if (waitingRspNum == 0) { - tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic, - vgId); + tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic, vgId); tmqCommitDone(pParamSet); } else { - tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d commit-rsp received, remain:%d", consumerId, pTopic, vgId, - waitingRspNum); + tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d commit-rsp received, remain:%d", consumerId, pTopic, vgId, waitingRspNum); } } @@ -2380,7 +2663,7 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { SMqRspHead* pHead = pMsg->pData; tmq_topic_assignment assignment = {.begin = pHead->walsver, - .end = pHead->walever, + .end = pHead->walever + 1, .currentOffset = rsp.rspOffset.version, .vgId = pParam->vgId}; @@ -2393,11 +2676,16 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { tsem_post(&pCommon->rsp); } + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pParam); return 0; } static void destroyCommonInfo(SMqVgCommon* pCommon) { + if(pCommon == NULL){ + return; + } taosArrayDestroy(pCommon->pList); tsem_destroy(&pCommon->rsp); taosThreadMutexDestroy(&pCommon->mutex); @@ -2405,56 +2693,282 @@ static void destroyCommonInfo(SMqVgCommon* pCommon) { taosMemoryFree(pCommon); } +static bool isInSnapshotMode(int8_t type, bool useSnapshot){ + if ((type < TMQ_OFFSET__LOG && useSnapshot) || type > TMQ_OFFSET__LOG) { + return true; + } + return false; +} + +static int32_t tmCommittedCb(void* param, SDataBuf* pMsg, int32_t code) { + SMqCommittedParam* pParam = param; + + if (code != 0){ + goto end; + } + if (pMsg) { + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)pMsg->pData, pMsg->len); + if (tDecodeMqVgOffset(&decoder, &pParam->vgOffset) < 0) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + tDecoderClear(&decoder); + } + + end: + if(pMsg){ + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + } + pParam->code = code; + tsem_post(&pParam->sem); + return 0; +} + +int64_t getCommittedFromServer(tmq_t *tmq, char* tname, int32_t vgId, SEpSet* epSet){ + int32_t code = 0; + SMqVgOffset pOffset = {0}; + + pOffset.consumerId = tmq->consumerId; + + int32_t groupLen = strlen(tmq->groupId); + memcpy(pOffset.offset.subKey, tmq->groupId, groupLen); + pOffset.offset.subKey[groupLen] = TMQ_SEPARATOR; + strcpy(pOffset.offset.subKey + groupLen + 1, tname); + + int32_t len = 0; + tEncodeSize(tEncodeMqVgOffset, &pOffset, len, code); + if (code < 0) { + return TSDB_CODE_INVALID_PARA; + } + + void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); + if (buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + ((SMsgHead*)buf)->vgId = htonl(vgId); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + SEncoder encoder; + tEncoderInit(&encoder, abuf, len); + tEncodeMqVgOffset(&encoder, &pOffset); + tEncoderClear(&encoder); + + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + taosMemoryFree(buf); + return TSDB_CODE_OUT_OF_MEMORY; + } + + SMqCommittedParam* pParam = taosMemoryMalloc(sizeof(SMqCommittedParam)); + if (pParam == NULL) { + taosMemoryFree(buf); + taosMemoryFree(sendInfo); + return TSDB_CODE_OUT_OF_MEMORY; + } + tsem_init(&pParam->sem, 0, 0); + + sendInfo->msgInfo = (SDataBuf){.pData = buf, .len = sizeof(SMsgHead) + len, .handle = NULL}; + sendInfo->requestId = generateRequestId(); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = tmCommittedCb; + sendInfo->msgType = TDMT_VND_TMQ_VG_COMMITTEDINFO; + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, &transporterId, sendInfo); + + tsem_wait(&pParam->sem); + code = pParam->code; + if(code == TSDB_CODE_SUCCESS){ + if(pParam->vgOffset.offset.val.type == TMQ_OFFSET__LOG){ + code = pParam->vgOffset.offset.val.version; + }else{ + code = TSDB_CODE_TMQ_SNAPSHOT_ERROR; + } + } + tsem_destroy(&pParam->sem); + taosMemoryFree(pParam); + + return code; +} + +int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId){ + if (tmq == NULL || pTopicName == NULL) { + tscError("invalid tmq handle, null"); + return TSDB_CODE_INVALID_PARA; + } + + int32_t accId = tmq->pTscObj->acctId; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; + sprintf(tname, "%d.%s", accId, pTopicName); + + taosWLockLatch(&tmq->lock); + + SMqClientVg* pVg = NULL; + int32_t code = getClientVg(tmq, tname, vgId, &pVg); + if(code != 0){ + taosWUnLockLatch(&tmq->lock); + return code; + } + + SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; + int32_t type = pOffsetInfo->endOffset.type; + if (isInSnapshotMode(type, tmq->useSnapshot)) { + tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, position error", tmq->consumerId, type); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_SNAPSHOT_ERROR; + } + + code = checkWalRange(pOffsetInfo, -1); + if (code != 0) { + taosWUnLockLatch(&tmq->lock); + return code; + } + SEpSet epSet = pVg->epSet; + int64_t begin = pVg->offsetInfo.walVerBegin; + int64_t end = pVg->offsetInfo.walVerEnd; + taosWUnLockLatch(&tmq->lock); + + int64_t position = 0; + if(type == TMQ_OFFSET__LOG){ + position = pOffsetInfo->endOffset.version; + }else if(type == TMQ_OFFSET__RESET_EARLIEST || type == TMQ_OFFSET__RESET_LATEST){ + code = getCommittedFromServer(tmq, tname, vgId, &epSet); + if(code == TSDB_CODE_TMQ_NO_COMMITTED){ + if(type == TMQ_OFFSET__RESET_EARLIEST){ + position = begin; + } else if(type == TMQ_OFFSET__RESET_LATEST){ + position = end; + } + }else{ + position = code; + } + }else{ + tscError("consumer:0x%" PRIx64 " offset type:%d can not be reach here", tmq->consumerId, type); + } + + tscInfo("consumer:0x%" PRIx64 " tmq_position vgId:%d position:%" PRId64, tmq->consumerId, vgId, position); + return position; +} + +int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId){ + if (tmq == NULL || pTopicName == NULL) { + tscError("invalid tmq handle, null"); + return TSDB_CODE_INVALID_PARA; + } + + int32_t accId = tmq->pTscObj->acctId; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; + sprintf(tname, "%d.%s", accId, pTopicName); + + taosWLockLatch(&tmq->lock); + + SMqClientVg* pVg = NULL; + int32_t code = getClientVg(tmq, tname, vgId, &pVg); + if(code != 0){ + taosWUnLockLatch(&tmq->lock); + return code; + } + + SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; + if (isInSnapshotMode(pOffsetInfo->endOffset.type, tmq->useSnapshot)) { + tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, pOffsetInfo->endOffset.type); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_SNAPSHOT_ERROR; + } + + if (isInSnapshotMode(pOffsetInfo->committedOffset.type, tmq->useSnapshot)) { + tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, pOffsetInfo->committedOffset.type); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_SNAPSHOT_ERROR; + } + + int64_t committed = 0; + if(pOffsetInfo->committedOffset.type == TMQ_OFFSET__LOG){ + committed = pOffsetInfo->committedOffset.version; + taosWUnLockLatch(&tmq->lock); + goto end; + } + SEpSet epSet = pVg->epSet; + taosWUnLockLatch(&tmq->lock); + + committed = getCommittedFromServer(tmq, tname, vgId, &epSet); + +end: + tscInfo("consumer:0x%" PRIx64 " tmq_committed vgId:%d committed:%" PRId64, tmq->consumerId, vgId, committed); + return committed; +} + int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_assignment** assignment, int32_t* numOfAssignment) { + if(tmq == NULL || pTopicName == NULL || assignment == NULL || numOfAssignment == NULL){ + tscError("invalid tmq handle, null"); + return TSDB_CODE_INVALID_PARA; + } *numOfAssignment = 0; *assignment = NULL; + SMqVgCommon* pCommon = NULL; int32_t accId = tmq->pTscObj->acctId; - char tname[128] = {0}; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; sprintf(tname, "%d.%s", accId, pTopicName); + int32_t code = TSDB_CODE_SUCCESS; + taosWLockLatch(&tmq->lock); SMqClientTopic* pTopic = getTopicByName(tmq, tname); if (pTopic == NULL) { - return TSDB_CODE_INVALID_PARA; + code = TSDB_CODE_TMQ_INVALID_TOPIC; + goto end; } // in case of snapshot is opened, no valid offset will return *numOfAssignment = taosArrayGetSize(pTopic->vgs); + for (int32_t j = 0; j < (*numOfAssignment); ++j) { + SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); + int32_t type = pClientVg->offsetInfo.beginOffset.type; + if (isInSnapshotMode(type, tmq->useSnapshot)) { + tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type); + code = TSDB_CODE_TMQ_SNAPSHOT_ERROR; + goto end; + } + } *assignment = taosMemoryCalloc(*numOfAssignment, sizeof(tmq_topic_assignment)); if (*assignment == NULL) { tscError("consumer:0x%" PRIx64 " failed to malloc buffer, size:%" PRIzu, tmq->consumerId, (*numOfAssignment) * sizeof(tmq_topic_assignment)); - return TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; } bool needFetch = false; for (int32_t j = 0; j < (*numOfAssignment); ++j) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); - if (!pClientVg->receiveInfo) { + if (pClientVg->offsetInfo.beginOffset.type != TMQ_OFFSET__LOG) { needFetch = true; break; } tmq_topic_assignment* pAssignment = &(*assignment)[j]; - if (pClientVg->offsetInfo.currentOffset.type == TMQ_OFFSET__LOG) { - pAssignment->currentOffset = pClientVg->offsetInfo.currentOffset.version; - } else { - pAssignment->currentOffset = 0; - } - + pAssignment->currentOffset = pClientVg->offsetInfo.beginOffset.version; pAssignment->begin = pClientVg->offsetInfo.walVerBegin; pAssignment->end = pClientVg->offsetInfo.walVerEnd; pAssignment->vgId = pClientVg->vgId; + tscInfo("consumer:0x%" PRIx64 " get assignment from local:%d->%" PRId64, tmq->consumerId, + pAssignment->vgId, pAssignment->currentOffset); } if (needFetch) { - SMqVgCommon* pCommon = taosMemoryCalloc(1, sizeof(SMqVgCommon)); + pCommon = taosMemoryCalloc(1, sizeof(SMqVgCommon)); if (pCommon == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return terrno; + code = terrno; + goto end; } pCommon->pList= taosArrayInit(4, sizeof(tmq_topic_assignment)); @@ -2469,8 +2983,8 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a SMqVgWalInfoParam* pParam = taosMemoryMalloc(sizeof(SMqVgWalInfoParam)); if (pParam == NULL) { - destroyCommonInfo(pCommon); - return terrno; + code = terrno; + goto end; } pParam->epoch = tmq->epoch; @@ -2480,34 +2994,35 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a SMqPollReq req = {0}; tmqBuildConsumeReqImpl(&req, tmq, 10, pTopic, pClientVg); + req.reqOffset = pClientVg->offsetInfo.beginOffset; int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req); if (msgSize < 0) { taosMemoryFree(pParam); - destroyCommonInfo(pCommon); - return terrno; + code = terrno; + goto end; } char* msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { taosMemoryFree(pParam); - destroyCommonInfo(pCommon); - return terrno; + code = terrno; + goto end; } if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) { taosMemoryFree(msg); taosMemoryFree(pParam); - destroyCommonInfo(pCommon); - return terrno; + code = terrno; + goto end; } SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { taosMemoryFree(pParam); taosMemoryFree(msg); - destroyCommonInfo(pCommon); - return terrno; + code = terrno; + goto end; } sendInfo->msgInfo = (SDataBuf){.pData = msg, .len = msgSize, .handle = NULL}; @@ -2518,29 +3033,26 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a sendInfo->msgType = TDMT_VND_TMQ_VG_WALINFO; int64_t transporterId = 0; - char offsetFormatBuf[80]; - tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.currentOffset); + char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.beginOffset); - tscDebug("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, - tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); + tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, + tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo); } tsem_wait(&pCommon->rsp); - int32_t code = pCommon->code; + code = pCommon->code; terrno = code; if (code != TSDB_CODE_SUCCESS) { - taosMemoryFree(*assignment); - *assignment = NULL; - *numOfAssignment = 0; - } else { - int32_t num = taosArrayGetSize(pCommon->pList); - for(int32_t i = 0; i < num; ++i) { - (*assignment)[i] = *(tmq_topic_assignment*)taosArrayGet(pCommon->pList, i); - } - *numOfAssignment = num; + goto end; + } + int32_t num = taosArrayGetSize(pCommon->pList); + for(int32_t i = 0; i < num; ++i) { + (*assignment)[i] = *(tmq_topic_assignment*)taosArrayGet(pCommon->pList, i); } + *numOfAssignment = num; for (int32_t j = 0; j < (*numOfAssignment); ++j) { tmq_topic_assignment* p = &(*assignment)[j]; @@ -2552,105 +3064,138 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } SVgOffsetInfo* pOffsetInfo = &pClientVg->offsetInfo; - - pOffsetInfo->currentOffset.type = TMQ_OFFSET__LOG; - - char offsetBuf[80] = {0}; - tFormatOffset(offsetBuf, tListLen(offsetBuf), &pOffsetInfo->currentOffset); - - tscDebug("vgId:%d offset is update to:%s", p->vgId, offsetBuf); + tscInfo("consumer:0x%" PRIx64 " %s vgId:%d offset is update to:%"PRId64, tmq->consumerId, pTopic->topicName, p->vgId, p->currentOffset); pOffsetInfo->walVerBegin = p->begin; pOffsetInfo->walVerEnd = p->end; - pOffsetInfo->currentOffset.version = p->currentOffset; - pOffsetInfo->committedOffset.version = p->currentOffset; } } + } - destroyCommonInfo(pCommon); - return code; - } else { - return TSDB_CODE_SUCCESS; +end: + if(code != TSDB_CODE_SUCCESS){ + taosMemoryFree(*assignment); + *assignment = NULL; + *numOfAssignment = 0; } + destroyCommonInfo(pCommon); + taosWUnLockLatch(&tmq->lock); + return code; } +void tmq_free_assignment(tmq_topic_assignment* pAssignment) { + if (pAssignment == NULL) { + return; + } + + taosMemoryFree(pAssignment); +} + +static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) { + if (pMsg) { + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + } + SMqSeekParam* pParam = param; + pParam->code = code; + tsem_post(&pParam->sem); + return 0; +} + +// seek interface have to send msg to server to cancel push handle if needed, because consumer may be in wait status if there is no data to poll int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset) { - if (tmq == NULL) { + if (tmq == NULL || pTopicName == NULL) { tscError("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } int32_t accId = tmq->pTscObj->acctId; - char tname[128] = {0}; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; sprintf(tname, "%d.%s", accId, pTopicName); - SMqClientTopic* pTopic = getTopicByName(tmq, tname); - if (pTopic == NULL) { - tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); - return TSDB_CODE_INVALID_PARA; - } + taosWLockLatch(&tmq->lock); SMqClientVg* pVg = NULL; - int32_t numOfVgs = taosArrayGetSize(pTopic->vgs); - for (int32_t i = 0; i < numOfVgs; ++i) { - SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); - if (pClientVg->vgId == vgId) { - pVg = pClientVg; - break; - } - } - - if (pVg == NULL) { - tscError("consumer:0x%" PRIx64 " invalid vgroup id:%d", tmq->consumerId, vgId); - return TSDB_CODE_INVALID_PARA; + int32_t code = getClientVg(tmq, tname, vgId, &pVg); + if(code != 0){ + taosWUnLockLatch(&tmq->lock); + return code; } SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; - int32_t type = pOffsetInfo->currentOffset.type; - if (type != TMQ_OFFSET__LOG) { + int32_t type = pOffsetInfo->endOffset.type; + if (isInSnapshotMode(type, tmq->useSnapshot)) { tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, seek not allowed", tmq->consumerId, type); - return TSDB_CODE_INVALID_PARA; + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } - if (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd) { - tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", - tmq->consumerId, offset, pOffsetInfo->walVerBegin, pOffsetInfo->walVerEnd); - return TSDB_CODE_INVALID_PARA; + code = checkWalRange(pOffsetInfo, offset); + if (code != 0) { + taosWUnLockLatch(&tmq->lock); + return code; } + tscInfo("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, vgId); // update the offset, and then commit to vnode - if (pOffsetInfo->currentOffset.type == TMQ_OFFSET__LOG) { - pOffsetInfo->currentOffset.version = offset; - pOffsetInfo->committedOffset.version = INT64_MIN; + pOffsetInfo->endOffset.type = TMQ_OFFSET__LOG; + pOffsetInfo->endOffset.version = offset; + pOffsetInfo->beginOffset = pOffsetInfo->endOffset; + pVg->seekUpdated = true; + SEpSet epSet = pVg->epSet; + taosWUnLockLatch(&tmq->lock); + + SMqSeekReq req = {0}; + snprintf(req.subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s:%s", tmq->groupId, tname); + req.head.vgId = vgId; + req.consumerId = tmq->consumerId; + + int32_t msgSize = tSerializeSMqSeekReq(NULL, 0, &req); + if (msgSize < 0) { + return TSDB_CODE_PAR_INTERNAL_ERROR; } - SMqRspObj rspObj = {.resType = RES_TYPE__TMQ, .vgId = pVg->vgId}; - tstrncpy(rspObj.topic, tname, tListLen(rspObj.topic)); + char* msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + return TSDB_CODE_OUT_OF_MEMORY; + } - tscDebug("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, pVg->vgId); + if (tSerializeSMqSeekReq(msg, msgSize, &req) < 0) { + taosMemoryFree(msg); + return TSDB_CODE_PAR_INTERNAL_ERROR; + } - SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); - if (pInfo == NULL) { - tscError("consumer:0x%"PRIx64" failed to prepare seek operation", tmq->consumerId); + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + taosMemoryFree(msg); return TSDB_CODE_OUT_OF_MEMORY; } - tsem_init(&pInfo->sem, 0, 0); - pInfo->code = 0; + SMqSeekParam* pParam = taosMemoryMalloc(sizeof(SMqSeekParam)); + if (pParam == NULL) { + taosMemoryFree(msg); + taosMemoryFree(sendInfo); + return TSDB_CODE_OUT_OF_MEMORY; + } + tsem_init(&pParam->sem, 0, 0); - asyncCommitOffset(tmq, &rspObj, TDMT_VND_TMQ_SEEK_TO_OFFSET, commitCallBackFn, pInfo); + sendInfo->msgInfo = (SDataBuf){.pData = msg, .len = msgSize, .handle = NULL}; + sendInfo->requestId = generateRequestId(); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = tmqSeekCb; + sendInfo->msgType = TDMT_VND_TMQ_SEEK; - tsem_wait(&pInfo->sem); - int32_t code = pInfo->code; + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - tsem_destroy(&pInfo->sem); - taosMemoryFree(pInfo); + tsem_wait(&pParam->sem); + code = pParam->code; + tsem_destroy(&pParam->sem); + taosMemoryFree(pParam); - if (code != TSDB_CODE_SUCCESS) { - tscError("consumer:0x%" PRIx64 " failed to send seek to vgId:%d, code:%s", tmq->consumerId, pVg->vgId, - tstrerror(code)); - } + tscInfo("consumer:0x%" PRIx64 "send seek to vgId:%d, return code:%s", tmq->consumerId, vgId, tstrerror(code)); return code; -} \ No newline at end of file +} diff --git a/source/client/src/clientTmqConnector.c b/source/client/src/clientTmqConnector.c index 894c51d13c9dd8e4205763893206de7ef86cdf7c..6ec82aa6ef0b5697e098d573935cec6e23e2f08c 100644 --- a/source/client/src/clientTmqConnector.c +++ b/source/client/src/clientTmqConnector.c @@ -17,9 +17,16 @@ #include "jniCommon.h" #include "taos.h" -int __init_tmq = 0; +int __init_tmq = 0; jmethodID g_offsetCallback; +jclass g_assignmentClass; +jmethodID g_assignmentConstructor; +jmethodID g_assignmentSetVgId; +jmethodID g_assignmentSetCurrentOffset; +jmethodID g_assignmentSetBegin; +jmethodID g_assignmentSetEnd; + void tmqGlobalMethod(JNIEnv *env) { // make sure init function executed once switch (atomic_val_compare_exchange_32(&__init_tmq, 0, 1)) { @@ -46,6 +53,38 @@ void tmqGlobalMethod(JNIEnv *env) { jniDebug("tmq method register finished"); } +int __init_assignment = 0; +void tmqAssignmentMethod(JNIEnv *env) { + // make sure init function executed once + switch (atomic_val_compare_exchange_32(&__init_assignment, 0, 1)) { + case 0: + break; + case 1: + do { + taosMsleep(0); + } while (atomic_load_32(&__init_assignment) == 1); + case 2: + return; + } + + if (g_vm == NULL) { + (*env)->GetJavaVM(env, &g_vm); + } + + jclass assignment = (*env)->FindClass(env, "com/taosdata/jdbc/tmq/Assignment"); + g_assignmentClass = (*env)->NewGlobalRef(env, assignment); + g_assignmentConstructor = (*env)->GetMethodID(env, g_assignmentClass, "", "()V"); + g_assignmentSetVgId = (*env)->GetMethodID(env, g_assignmentClass, "setVgId", "(I)V"); // int + g_assignmentSetCurrentOffset = (*env)->GetMethodID(env, g_assignmentClass, "setCurrentOffset", "(J)V"); // long + g_assignmentSetBegin = (*env)->GetMethodID(env, g_assignmentClass, "setBegin", "(J)V"); // long + g_assignmentSetEnd = (*env)->GetMethodID(env, g_assignmentClass, "setEnd", "(J)V"); // long + + (*env)->DeleteLocalRef(env, assignment); + + atomic_store_32(&__init_assignment, 2); + jniDebug("tmq method assignment finished"); +} + // deprecated void commit_cb(tmq_t *tmq, int32_t code, void *param) { JNIEnv *env = NULL; @@ -266,8 +305,9 @@ JNIEXPORT void JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqCommitAsync(JN tmq_commit_async(tmq, res, commit_cb, consumer); } -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_consumerCommitAsync(JNIEnv *env, jobject jobj, jlong jtmq, - jlong jres, jobject offset) { +JNIEXPORT void JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_consumerCommitAsync(JNIEnv *env, jobject jobj, + jlong jtmq, jlong jres, + jobject offset) { tmqGlobalMethod(env); tmq_t *tmq = (tmq_t *)jtmq; if (tmq == NULL) { @@ -335,7 +375,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetVgroupId(JN TAOS_RES *res = (TAOS_RES *)jres; if (res == NULL) { jniDebug("jobj:%p, invalid res handle", jobj); - return -1; + return JNI_RESULT_SET_NULL; } return tmq_get_vgroup_id(res); } @@ -350,6 +390,15 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTableNam return (*env)->NewStringUTF(env, tmq_get_table_name(res)); } +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetOffset(JNIEnv *env, jobject jobj, jlong jres) { + TAOS_RES *res = (TAOS_RES *)jres; + if (res == NULL) { + jniDebug("jobj:%p, invalid res handle", jobj); + return JNI_RESULT_SET_NULL; + } + return tmq_get_vgroup_offset(res); +} + JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(JNIEnv *env, jobject jobj, jlong con, jlong res, jobject rowobj, jobject arrayListObj) { @@ -369,7 +418,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp( jniDebug("jobj:%p, conn:%p, resultset:%p, no data to retrieve", jobj, tscon, (void *)res); return JNI_FETCH_END; } else { - jniError("jobj:%p, conn:%p, query interrupted, tmq fetch block error code:%d, msg:%s", jobj, tscon, error_code, taos_errstr(tres)); + jniError("jobj:%p, conn:%p, query interrupted, tmq fetch block error code:%d, msg:%s", jobj, tscon, error_code, + taos_errstr(tres)); return JNI_RESULT_SET_NULL; } } @@ -399,3 +449,72 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp( (*env)->CallVoidMethod(env, rowobj, g_blockdataSetByteArrayFp, jniFromNCharToByteArray(env, (char *)data, len)); return JNI_SUCCESS; } + +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqSeekImp(JNIEnv *env, jobject jobj, jlong jtmq, + jstring jtopic, jint partition, + jlong offset) { + tmq_t *tmq = (tmq_t *)jtmq; + if (tmq == NULL) { + jniDebug("jobj:%p, tmq is closed", jobj); + return TMQ_CONSUMER_NULL; + } + + if (jtopic == NULL) { + jniDebug("jobj:%p, topic is null", jobj); + return TMQ_TOPIC_NULL; + } + const char *topicName = (*env)->GetStringUTFChars(env, jtopic, NULL); + + int32_t res = tmq_offset_seek(tmq, topicName, partition, offset); + + if (res != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, tmq seek error, code:%d, msg:%s", jobj, res, tmq_err2str(res)); + } + + (*env)->ReleaseStringUTFChars(env, jtopic, topicName); + return (jint)res; +} + +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTopicAssignmentImp(JNIEnv *env, jobject jobj, + jlong jtmq, jstring jtopic, + jobject jarrayList) { + tmqAssignmentMethod(env); + tmq_t *tmq = (tmq_t *)jtmq; + if (tmq == NULL) { + jniDebug("jobj:%p, tmq is closed", jobj); + return TMQ_CONSUMER_NULL; + } + + if (jtopic == NULL) { + jniDebug("jobj:%p, topic is null", jobj); + return TMQ_TOPIC_NULL; + } + + const char *topicName = (*env)->GetStringUTFChars(env, jtopic, NULL); + + tmq_topic_assignment *pAssign = NULL; + int32_t numOfAssignment = 0; + int32_t res = tmq_get_topic_assignment(tmq, topicName, &pAssign, &numOfAssignment); + + if (res != TSDB_CODE_SUCCESS) { + (*env)->ReleaseStringUTFChars(env, jtopic, topicName); + jniError("jobj:%p, tmq get topic assignment error, topic:%s, code:%d, msg:%s", jobj, topicName, res, + tmq_err2str(res)); + tmq_free_assignment(pAssign); + return (jint)res; + } + + (*env)->ReleaseStringUTFChars(env, jtopic, topicName); + + for (int i = 0; i < numOfAssignment; ++i) { + tmq_topic_assignment assignment = pAssign[i]; + jobject jassignment = (*env)->NewObject(env, g_assignmentClass, g_assignmentConstructor); + (*env)->CallVoidMethod(env, jassignment, g_assignmentSetVgId, assignment.vgId); + (*env)->CallVoidMethod(env, jassignment, g_assignmentSetCurrentOffset, assignment.currentOffset); + (*env)->CallVoidMethod(env, jassignment, g_assignmentSetBegin, assignment.begin); + (*env)->CallVoidMethod(env, jassignment, g_assignmentSetEnd, assignment.end); + (*env)->CallBooleanMethod(env, jarrayList, g_arrayListAddFp, jassignment); + } + tmq_free_assignment(pAssign); + return JNI_SUCCESS; +} diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index b04727bfc030ed9d2181462725778a75c6202d99..6f978b0143b8f30de1d83f2db3e0d404c6aec44d 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -34,6 +34,8 @@ namespace { void printSubResults(void* pRes, int32_t* totalRows) { char buf[1024]; + int32_t vgId = tmq_get_vgroup_id(pRes); + int64_t offset = tmq_get_vgroup_offset(pRes); while (1) { TAOS_ROW row = taos_fetch_row(pRes); if (row == NULL) { @@ -45,7 +47,7 @@ void printSubResults(void* pRes, int32_t* totalRows) { int32_t precision = taos_result_precision(pRes); taos_print_row(buf, row, fields, numOfFields); *totalRows += 1; - printf("precision: %d, row content: %s\n", precision, buf); + printf("vgId: %d, offset: %lld, precision: %d, row content: %s\n", vgId, offset, precision, buf); } // taos_free_result(pRes); @@ -1073,6 +1075,258 @@ TEST(clientCase, sub_db_test) { fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); } +TEST(clientCase, tmq_commit) { +// taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); + + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + tmq_conf_t* conf = tmq_conf_new(); + + tmq_conf_set(conf, "enable.auto.commit", "false"); + tmq_conf_set(conf, "auto.commit.interval.ms", "2000"); + tmq_conf_set(conf, "group.id", "group_id_2"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + tmq_conf_destroy(conf); + + char topicName[128] = "tp"; + // 创建订阅 topics 列表 + tmq_list_t* topicList = tmq_list_new(); + tmq_list_append(topicList, topicName); + + // 启动订阅 + tmq_subscribe(tmq, topicList); + tmq_list_destroy(topicList); + + int32_t totalRows = 0; + int32_t msgCnt = 0; + int32_t timeout = 2000; + + tmq_topic_assignment* pAssign = NULL; + int32_t numOfAssign = 0; + + int32_t code = tmq_get_topic_assignment(tmq, topicName, &pAssign, &numOfAssign); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_free_assignment(pAssign); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); + return; + } + + for(int i = 0; i < numOfAssign; i++){ + printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end); + + int64_t committed = tmq_committed(tmq, topicName, pAssign[i].vgId); + printf("committed vgId:%d, committed:%lld\n", pAssign[i].vgId, committed); + + int64_t position = tmq_position(tmq, topicName, pAssign[i].vgId); + printf("position vgId:%d, position:%lld\n", pAssign[i].vgId, position); + tmq_offset_seek(tmq, topicName, pAssign[i].vgId, 1); + position = tmq_position(tmq, topicName, pAssign[i].vgId); + printf("after seek 1, position vgId:%d, position:%lld\n", pAssign[i].vgId, position); + } + + while (1) { + printf("start to poll\n"); + TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout); + if (pRes) { + printSubResults(pRes, &totalRows); + } else { + break; + } + + tmq_commit_sync(tmq, pRes); + for(int i = 0; i < numOfAssign; i++) { + int64_t committed = tmq_committed(tmq, topicName, pAssign[i].vgId); + printf("committed vgId:%d, committed:%lld\n", pAssign[i].vgId, committed); + if(committed > 0){ + int32_t code = tmq_commit_offset_sync(tmq, topicName, pAssign[i].vgId, 4); + printf("tmq_commit_offset_sync vgId:%d, offset:4, code:%d\n", pAssign[i].vgId, code); + int64_t committed = tmq_committed(tmq, topicName, pAssign[i].vgId); + printf("after tmq_commit_offset_sync, committed vgId:%d, committed:%lld\n", pAssign[i].vgId, committed); + } + } + if (pRes != NULL) { + taos_free_result(pRes); + } + +// tmq_offset_seek(tmq, "tp", pAssign[0].vgId, pAssign[0].begin); + } + + tmq_free_assignment(pAssign); + + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); +} + +TEST(clientCase, td_25129) { +// taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); + + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + tmq_conf_t* conf = tmq_conf_new(); + + tmq_conf_set(conf, "enable.auto.commit", "false"); + tmq_conf_set(conf, "auto.commit.interval.ms", "2000"); + tmq_conf_set(conf, "group.id", "group_id_2"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + tmq_conf_destroy(conf); + + char topicName[128] = "tp"; + // 创建订阅 topics 列表 + tmq_list_t* topicList = tmq_list_new(); + tmq_list_append(topicList, topicName); + + // 启动订阅 + tmq_subscribe(tmq, topicList); + tmq_list_destroy(topicList); + + TAOS_FIELD* fields = NULL; + int32_t numOfFields = 0; + int32_t precision = 0; + int32_t totalRows = 0; + int32_t msgCnt = 0; + int32_t timeout = 2000; + + int32_t count = 0; + + tmq_topic_assignment* pAssign = NULL; + int32_t numOfAssign = 0; + + int32_t code = tmq_get_topic_assignment(tmq, topicName, &pAssign, &numOfAssign); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_free_assignment(pAssign); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); + return; + } + + for(int i = 0; i < numOfAssign; i++){ + printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end); + } + +// tmq_offset_seek(tmq, "tp", pAssign[0].vgId, 4); + tmq_free_assignment(pAssign); + + code = tmq_get_topic_assignment(tmq, topicName, &pAssign, &numOfAssign); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_free_assignment(pAssign); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); + return; + } + + for(int i = 0; i < numOfAssign; i++){ + printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end); + } + + tmq_free_assignment(pAssign); + + code = tmq_get_topic_assignment(tmq, topicName, &pAssign, &numOfAssign); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_free_assignment(pAssign); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); + return; + } + + for(int i = 0; i < numOfAssign; i++){ + int64_t committed = tmq_committed(tmq, topicName, pAssign[i].vgId); + printf("assign i:%d, vgId:%d, committed:%lld, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, committed, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end); + } + + while (1) { + printf("start to poll\n"); + TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout); + if (pRes) { + char buf[128]; + + const char* topicName = tmq_get_topic_name(pRes); +// const char* dbName = tmq_get_db_name(pRes); +// int32_t vgroupId = tmq_get_vgroup_id(pRes); +// +// printf("topic: %s\n", topicName); +// printf("db: %s\n", dbName); +// printf("vgroup id: %d\n", vgroupId); + + printSubResults(pRes, &totalRows); + + code = tmq_get_topic_assignment(tmq, topicName, &pAssign, &numOfAssign); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_free_assignment(pAssign); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); + return; + } + + for(int i = 0; i < numOfAssign; i++){ + printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end); + } + } else { + for(int i = 0; i < numOfAssign; i++) { + tmq_offset_seek(tmq, topicName, pAssign[i].vgId, pAssign[i].currentOffset); + } + tmq_commit_sync(tmq, pRes); + break; + } + +// tmq_commit_sync(tmq, pRes); + if (pRes != NULL) { + taos_free_result(pRes); + // if ((++count) > 1) { + // break; + // } + } else { + break; + } + +// tmq_offset_seek(tmq, "tp", pAssign[0].vgId, pAssign[0].begin); + } + + tmq_free_assignment(pAssign); + + code = tmq_get_topic_assignment(tmq, "tp", &pAssign, &numOfAssign); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_free_assignment(pAssign); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); + return; + } + + for(int i = 0; i < numOfAssign; i++){ + printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end); + } + + tmq_free_assignment(pAssign); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); +} + TEST(clientCase, sub_tb_test) { taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); @@ -1100,7 +1354,7 @@ TEST(clientCase, sub_tb_test) { // 创建订阅 topics 列表 tmq_list_t* topicList = tmq_list_new(); - tmq_list_append(topicList, "topic_t1"); + tmq_list_append(topicList, "t1"); // 启动订阅 tmq_subscribe(tmq, topicList); @@ -1118,7 +1372,7 @@ TEST(clientCase, sub_tb_test) { tmq_topic_assignment* pAssign = NULL; int32_t numOfAssign = 0; - int32_t code = tmq_get_topic_assignment(tmq, "topic_t1", &pAssign, &numOfAssign); + int32_t code = tmq_get_topic_assignment(tmq, "t1", &pAssign, &numOfAssign); if (code != 0) { printf("error occurs:%s\n", tmq_err2str(code)); tmq_consumer_close(tmq); @@ -1127,7 +1381,16 @@ TEST(clientCase, sub_tb_test) { return; } - tmq_offset_seek(tmq, "topic_t1", pAssign[0].vgId, 0); + tmq_offset_seek(tmq, "t1", pAssign[0].vgId, 4); + + code = tmq_get_topic_assignment(tmq, "t1", &pAssign, &numOfAssign); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); + return; + } while (1) { TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout); diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 1578b8b607f8ea19b4ee3efb7573c054b6c379fc..054698fff0bdc542ac31847e2bb70beb4b3e573f 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -224,6 +224,8 @@ TEST(testCase, smlParseCols_Error_Test) { "st,tt=aa c 1=2 1626006833639000000,", //field value double quote,slash "st,tt=aa c=\"a\"a\" 1626006833639000000,", + "escape_test,tag1=\"tag1_value\",tag2=\"tag2_value\" co l0=\"col0_value\",col1=\"col1_value\" 1680918783010000000", + "escape_test,tag1=\"tag1_value\",tag2=\"tag2_value\" col0=\"co\"l\"0_value\",col1=\"col1_value\" 1680918783010000000" }; SSmlHandle *info = smlBuildSmlInfo(NULL); diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 9c6d941172b7ec58737a33f1515da20f122f0c01..356ea2be1cb35be9ae0cedb6061abfa60b3307a3 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -1,4 +1,8 @@ aux_source_directory(src COMMON_SRC) +IF (TD_ENTERPRISE) +LIST(APPEND COMMON_SRC ${TD_ENTERPRISE_DIR}/src/plugins/common/src/tglobal.c) +ENDIF() + add_library(common STATIC ${COMMON_SRC}) if (DEFINED GRANT_CFG_INCLUDE_DIR) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 062bae68c83206d7e99e9ec3f8198ab7ec35961f..0940fcef6a749489077d45f931f4c27e8800f4ee 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -33,7 +33,7 @@ static const SSysDbTableSchema dnodesSchema[] = { {.name = "support_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, - {.name = "reboot_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, + {.name = "reboot_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, {.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, #ifdef TD_ENTERPRISE {.name = "active_code", .bytes = TSDB_ACTIVE_KEY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, @@ -47,7 +47,7 @@ static const SSysDbTableSchema mnodesSchema[] = { {.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, - {.name = "reboot_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, + {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, }; static const SSysDbTableSchema modulesSchema[] = { @@ -73,7 +73,7 @@ static const SSysDbTableSchema clusterSchema[] = { {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "uptime", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, - {.name = "version", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "version", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "expire_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, }; @@ -102,8 +102,6 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "wal_fsync_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "wal_retention_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "wal_retention_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true}, - {.name = "wal_roll_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "wal_segment_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true}, {.name = "stt_trigger", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "table_prefix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, @@ -160,9 +158,9 @@ static const SSysDbTableSchema streamSchema[] = { static const SSysDbTableSchema streamTaskSchema[] = { {.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "task_id", .bytes = 8, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "task_id", .bytes = 32, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "node_type", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "node_id", .bytes = 8, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "node_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "level", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; @@ -273,6 +271,7 @@ static const SSysDbTableSchema variablesSchema[] = { {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "value", .bytes = TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "scope", .bytes = TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema topicSchema[] = { @@ -280,7 +279,9 @@ static const SSysDbTableSchema topicSchema[] = { {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, - // TODO config + {.name = "schema", .bytes = TSDB_MAX_BINARY_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, + {.name = "meta", .bytes = 4 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, + {.name = "type", .bytes = 8 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, }; @@ -288,7 +289,9 @@ static const SSysDbTableSchema subscriptionSchema[] = { {.name = "topic_name", .bytes = TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, {.name = "consumer_group", .bytes = TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, - {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, + {.name = "consumer_id", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, + {.name = "offset", .bytes = TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, + {.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, }; static const SSysDbTableSchema vnodesSchema[] = { @@ -313,7 +316,7 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true}, {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema), true}, {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema), true}, - {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, + {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema), true}, {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema), true}, {TSDB_INS_TABLE_DATABASES, userDBSchema, tListLen(userDBSchema), false}, {TSDB_INS_TABLE_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema), false}, @@ -348,7 +351,7 @@ static const SSysDbTableSchema connectionsSchema[] = { static const SSysDbTableSchema consumerSchema[] = { - {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, + {.name = "consumer_id", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, {.name = "consumer_group", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, @@ -357,6 +360,7 @@ static const SSysDbTableSchema consumerSchema[] = { {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "subscribe_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "rebalance_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, + {.name = "parameters", .bytes = 64 + TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, }; static const SSysDbTableSchema offsetSchema[] = { @@ -379,6 +383,7 @@ static const SSysDbTableSchema querySchema[] = { {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "exec_usec", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, {.name = "stable_query", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL, .sysInfo = false}, + {.name = "sub_query", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL, .sysInfo = false}, {.name = "sub_num", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "sub_status", .bytes = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5f7e43668a746a814119e55bfd6bee5736100836..5188b1e27c3b0fcebde8561ac211bcc3c7ba17df 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -23,6 +23,20 @@ int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + if (pColumnInfoData->reassigned) { + int32_t totalSize = 0; + for (int32_t row = 0; row < numOfRows; ++row) { + char* pColData = pColumnInfoData->pData + pColumnInfoData->varmeta.offset[row]; + int32_t colSize = 0; + if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) { + colSize = getJsonValueLen(pColData); + } else { + colSize = varDataTLen(pColData); + } + totalSize += colSize; + } + return totalSize; + } return pColumnInfoData->varmeta.length; } else { if (pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) { @@ -33,6 +47,16 @@ int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRo } } +int32_t colDataGetRowLength(const SColumnInfoData* pColumnInfoData, int32_t rowIdx) { + if (colDataIsNull_s(pColumnInfoData, rowIdx)) return 0; + + if (!IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) return pColumnInfoData->info.bytes; + if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) + return getJsonValueLen(colDataGetData(pColumnInfoData, rowIdx)); + else + return varDataTLen(colDataGetData(pColumnInfoData, rowIdx)); +} + int32_t colDataGetFullLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return pColumnInfoData->varmeta.length + sizeof(int32_t) * numOfRows; @@ -42,10 +66,6 @@ int32_t colDataGetFullLength(const SColumnInfoData* pColumnInfoData, int32_t num } } -void colDataTrim(SColumnInfoData* pColumnInfoData) { - // TODO -} - int32_t getJsonValueLen(const char* data) { int32_t dataLen = 0; if (*data == TSDB_DATA_TYPE_NULL) { @@ -64,10 +84,6 @@ int32_t getJsonValueLen(const char* data) { return dataLen; } -int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) { - return colDataSetVal(pColumnInfoData, rowIndex, pData, isNull); -} - int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) { if (isNull) { // There is a placehold for each NULL value of binary or nchar type. @@ -126,7 +142,30 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const return 0; } -int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) { +int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData) { + int32_t type = pColumnInfoData->info.type; + if (IS_VAR_DATA_TYPE(type)) { + int32_t dataLen = 0; + if (type == TSDB_DATA_TYPE_JSON) { + dataLen = getJsonValueLen(pData); + } else { + dataLen = varDataTLen(pData); + } + + SVarColAttr* pAttr = &pColumnInfoData->varmeta; + + pColumnInfoData->varmeta.offset[dstRowIdx] = pColumnInfoData->varmeta.offset[srcRowIdx]; + pColumnInfoData->reassigned = true; + } else { + memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * dstRowIdx, pData, pColumnInfoData->info.bytes); + colDataClearNull_f(pColumnInfoData->nullbitmap, dstRowIdx); + } + + return 0; +} + + +static int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) { if (!IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return TSDB_CODE_SUCCESS; } @@ -445,12 +484,11 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd size_t headerSize = sizeof(int32_t); size_t colHeaderSize = sizeof(int32_t) * numOfCols; - size_t payloadSize = pageSize - (headerSize + colHeaderSize); // TODO speedup by checking if the whole page can fit in firstly. if (!hasVarCol) { size_t rowSize = blockDataGetRowSize(pBlock); - int32_t capacity = payloadSize / (rowSize + numOfCols * bitmapChar / 8.0); + int32_t capacity = blockDataGetCapacityInRow(pBlock, pageSize, headerSize + colHeaderSize); if (capacity <= 0) { return TSDB_CODE_FAILED; } @@ -511,6 +549,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 pDst->info = pBlock->info; pDst->info.rows = 0; pDst->info.capacity = 0; + pDst->info.rowSize = 0; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {0}; @@ -581,8 +620,25 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { *(int32_t*)pStart = dataSize; pStart += sizeof(int32_t); - memcpy(pStart, pCol->pData, dataSize); - pStart += dataSize; + if (pCol->reassigned && IS_VAR_DATA_TYPE(pCol->info.type)) { + for (int32_t row = 0; row < numOfRows; ++row) { + char* pColData = pCol->pData + pCol->varmeta.offset[row]; + int32_t colSize = 0; + if (pCol->info.type == TSDB_DATA_TYPE_JSON) { + colSize = getJsonValueLen(pColData); + } else { + colSize = varDataTLen(pColData); + } + memcpy(pStart, pColData, colSize); + pStart += colSize; + } + } else { + if (dataSize != 0) { + // ubsan reports error if pCol->pData==NULL && dataSize==0 + memcpy(pStart, pCol->pData, dataSize); + } + pStart += dataSize; + } } return 0; @@ -632,8 +688,10 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { return TSDB_CODE_FAILED; } } - - memcpy(pCol->pData, pStart, colLength); + if (colLength != 0) { + // ubsan reports error if colLength==0 && pCol->pData == 0 + memcpy(pCol->pData, pStart, colLength); + } pStart += colLength; } @@ -741,8 +799,8 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { * @return */ size_t blockDataGetSerialMetaSize(uint32_t numOfCols) { - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column - // length | + // | version | total length | total rows | total columns | flag seg| block group id | column schema + // | each column length | return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int8_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t); } @@ -821,41 +879,8 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { return 0; } -static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock, - int32_t tupleIndex) { - int32_t code = 0; - size_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pDst = &pDstCols[i]; - SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, i); - - if (pSrc->hasNull && colDataIsNull(pSrc, pSrcBlock->info.rows, tupleIndex, pSrcBlock->pBlockAgg[i])) { - code = colDataSetVal(pDst, numOfRows, NULL, true); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else { - char* p = colDataGetData(pSrc, tupleIndex); - code = colDataSetVal(pDst, numOfRows, p, false); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - } - - return TSDB_CODE_SUCCESS; -} - static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, const int32_t* index) { -#if 0 - for (int32_t i = 0; i < pDataBlock->info.rows; ++i) { - int32_t code = doAssignOneTuple(pCols, i, pDataBlock, index[i]); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } -#else + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = &pCols[i]; @@ -880,7 +905,7 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB } } } -#endif + return TSDB_CODE_SUCCESS; } @@ -1040,114 +1065,6 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { return TSDB_CODE_SUCCESS; } -#if 0 -typedef struct SHelper { - int32_t index; - union { - char* pData; - int64_t i64; - double d64; - }; -} SHelper; - -SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* pBlock) { - int32_t sortValLengthPerRow = 0; - int32_t numOfCols = taosArrayGetSize(pOrderInfo); - - for (int32_t i = 0; i < numOfCols; ++i) { - SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i); - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->slotId); - pInfo->pColData = pColInfo; - sortValLengthPerRow += pColInfo->info.bytes; - } - - size_t len = sortValLengthPerRow * pBlock->info.rows; - - char* buf = taosMemoryCalloc(1, len); - SHelper* phelper = taosMemoryCalloc(numOfRows, sizeof(SHelper)); - for (int32_t i = 0; i < numOfRows; ++i) { - phelper[i].index = i; - phelper[i].pData = buf + sortValLengthPerRow * i; - } - - int32_t offset = 0; - for (int32_t i = 0; i < numOfCols; ++i) { - SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i); - for (int32_t j = 0; j < numOfRows; ++j) { - phelper[j].i64 = *(int32_t*)pInfo->pColData->pData + pInfo->pColData->info.bytes * j; - // memcpy(phelper[j].pData + offset, pInfo->pColData->pData + pInfo->pColData->info.bytes * j, - // pInfo->pColData->info.bytes); - } - - offset += pInfo->pColData->info.bytes; - } - - taosMemoryFree(buf); - return phelper; -} - -int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) { - const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*)param; - - SHelper* left = (SHelper*)p1; - SHelper* right = (SHelper*)p2; - - SArray* pInfo = pHelper->orderInfo; - - int32_t offset = 0; - int32_t leftx = *(int32_t*)left->pData; //*(int32_t*)(left->pData + offset); - int32_t rightx = *(int32_t*)right->pData; //*(int32_t*)(right->pData + offset); - - if (leftx == rightx) { - return 0; - } else { - return (leftx < rightx) ? -1 : 1; - } - return 0; -} - -int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) { - // Allocate the additional buffer. - int64_t p0 = taosGetTimestampUs(); - - SSDataBlockSortHelper helper = {.pDataBlock = pDataBlock, .orderInfo = pOrderInfo}; - - uint32_t rows = pDataBlock->info.rows; - SHelper* index = createTupleIndex_rv(rows, helper.orderInfo, pDataBlock); - if (index == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return terrno; - } - - taosqsort(index, rows, sizeof(SHelper), &helper, dataBlockCompar_rv); - - int64_t p1 = taosGetTimestampUs(); - SColumnInfoData* pCols = createHelpColInfoData(pDataBlock); - if (pCols == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return terrno; - } - - int64_t p2 = taosGetTimestampUs(); - - // int32_t code = blockDataAssign(pCols, pDataBlock, index); - // if (code != TSDB_CODE_SUCCESS) { - // terrno = code; - // return code; - // } - - int64_t p3 = taosGetTimestampUs(); - - copyBackToBlock(pDataBlock, pCols); - int64_t p4 = taosGetTimestampUs(); - - printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, - p3 - p2, p4 - p3, rows); - // destroyTupleIndex(index); - return 0; -} -#endif - void blockDataCleanup(SSDataBlock* pDataBlock) { blockDataEmpty(pDataBlock); SDataBlockInfo* pInfo = &pDataBlock->info; @@ -1284,8 +1201,7 @@ void blockDataFreeRes(SSDataBlock* pBlock) { colDataDestroy(pColInfoData); } - taosArrayDestroy(pBlock->pDataBlock); - pBlock->pDataBlock = NULL; + pBlock->pDataBlock = taosArrayDestroy(pBlock->pDataBlock); taosMemoryFreeClear(pBlock->pBlockAgg); memset(&pBlock->info, 0, sizeof(SDataBlockInfo)); } @@ -1300,6 +1216,7 @@ void* blockDataDestroy(SSDataBlock* pBlock) { return NULL; } +// todo remove it int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { dst->info = src->info; dst->info.rows = 0; @@ -1532,26 +1449,43 @@ SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index) { return taosArrayGet(pBlock->pDataBlock, index); } -size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { +size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize, int32_t extraSize) { size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(numOfCols); + int32_t payloadSize = pageSize - extraSize; int32_t rowSize = pBlock->info.rowSize; int32_t nRows = payloadSize / rowSize; ASSERT(nRows >= 1); - // the true value must be less than the value of nRows - int32_t additional = 0; + int32_t numVarCols = 0; + int32_t numFixCols = 0; for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pCol->info.type)) { - additional += nRows * sizeof(int32_t); + ++numVarCols; } else { - additional += BitmapLen(nRows); + ++numFixCols; } } - int32_t newRows = (payloadSize - additional) / rowSize; + // find the data payload whose size is greater than payloadSize + int result = -1; + int start = 1; + int end = nRows; + while (start <= end) { + int mid = start + (end - start) / 2; + //data size + var data type columns offset + fixed data type columns bitmap len + int midSize = rowSize * mid + numVarCols * sizeof(int32_t) * mid + numFixCols * BitmapLen(mid); + if (midSize > payloadSize) { + result = mid; + end = mid - 1; + } else { + start = mid + 1; + } + } + + int32_t newRows = (result != -1) ? result - 1 : nRows; + // the true value must be less than the value of nRows ASSERT(newRows <= nRows && newRows >= 1); return newRows; @@ -1646,7 +1580,8 @@ static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, s static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, n, total); + // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, n, total); + memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[n], (total - n) * sizeof(int32_t)); // clear the offset value of the unused entries. memset(&pColInfoData->varmeta.offset[total - n], 0, n); @@ -1678,18 +1613,8 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) { static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); + // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); - } else { // reset the bitmap value - /*int32_t stopIndex = BitmapLen(n) * 8; - for(int32_t i = n; i < stopIndex; ++i) { - colDataClearNull_f(pColInfoData->nullbitmap, i); - } - - int32_t remain = BitmapLen(total) - BitmapLen(n); - if (remain > 0) { - memset(pColInfoData->nullbitmap+BitmapLen(n), 0, remain); - }*/ } } @@ -1742,7 +1667,20 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { int32_t len = colDataGetLength(pColData, rows); tlen += taosEncodeFixedI32(buf, len); - tlen += taosEncodeBinary(buf, pColData->pData, len); + if (pColData->reassigned && IS_VAR_DATA_TYPE(pColData->info.type)) { + for (int32_t row = 0; row < rows; ++row) { + char* pData = pColData->pData + pColData->varmeta.offset[row]; + int32_t colSize = 0; + if (pColData->info.type == TSDB_DATA_TYPE_JSON) { + colSize = getJsonValueLen(pData); + } else { + colSize = varDataTLen(pData); + } + tlen += taosEncodeBinary(buf, pData, colSize); + } + } else { + tlen += taosEncodeBinary(buf, pColData->pData, len); + } } return tlen; } @@ -1783,32 +1721,6 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { return (void*)buf; } -int32_t tEncodeDataBlocks(void** buf, const SArray* blocks) { - int32_t tlen = 0; - int32_t sz = taosArrayGetSize(blocks); - tlen += taosEncodeFixedI32(buf, sz); - - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pBlock = taosArrayGet(blocks, i); - tlen += tEncodeDataBlock(buf, pBlock); - } - - return tlen; -} - -void* tDecodeDataBlocks(const void* buf, SArray** blocks) { - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - - *blocks = taosArrayInit(sz, sizeof(SSDataBlock)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock pBlock = {0}; - buf = tDecodeDataBlock(buf, &pBlock); - taosArrayPush(*blocks, &pBlock); - } - return (void*)buf; -} - static char* formatTimestamp(char* buf, int64_t val, int precision) { time_t tt; int32_t ms = 0; @@ -1858,100 +1770,6 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { return buf; } -#if 0 -void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag) { - SArray* dataBlocks = taosArrayInit(1, sizeof(SSDataBlock*)); - taosArrayPush(dataBlocks, &pBlock); - blockDebugShowDataBlocks(dataBlocks, flag); - taosArrayDestroy(dataBlocks); -} - -void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { - char pBuf[128] = {0}; - int32_t sz = taosArrayGetSize(dataBlocks); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); - size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); - - int32_t rows = pDataBlock->info.rows; - printf("%s |block ver %" PRIi64 " |block type %d |child id %d|group id %" PRIu64 "\n", flag, - pDataBlock->info.version, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, - pDataBlock->info.id.groupId); - for (int32_t j = 0; j < rows; j++) { - printf("%s |", flag); - for (int32_t k = 0; k < numOfCols; k++) { - SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - if (k == 0) { - printf("cols:%d |", (int32_t)numOfCols); - } - if (colDataIsNull(pColInfoData, rows, j, NULL)) { - printf(" %15s |", "NULL"); - continue; - } - - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); - printf(" %25s |", pBuf); - break; - case TSDB_DATA_TYPE_BOOL: - printf(" %15" PRIi8 " |", *(int8_t*)var); - break; - case TSDB_DATA_TYPE_TINYINT: - printf(" %15" PRIi8 " |", *(int8_t*)var); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf(" %15" PRIi16 " |", *(int16_t*)var); - break; - case TSDB_DATA_TYPE_INT: - printf(" %15d |", *(int32_t*)var); - break; - case TSDB_DATA_TYPE_UTINYINT: - printf(" %15" PRIu8 " |", *(uint8_t*)var); - break; - case TSDB_DATA_TYPE_USMALLINT: - printf(" %15" PRIu16 " |", *(uint16_t*)var); - break; - case TSDB_DATA_TYPE_UINT: - printf(" %15u |", *(uint32_t*)var); - break; - case TSDB_DATA_TYPE_BIGINT: - printf(" %15" PRId64 " |", *(int64_t*)var); - break; - case TSDB_DATA_TYPE_UBIGINT: - printf(" %15" PRIu64 " |", *(uint64_t*)var); - break; - case TSDB_DATA_TYPE_FLOAT: - printf(" %15f |", *(float*)var); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf(" %15lf |", *(double*)var); - break; - case TSDB_DATA_TYPE_VARCHAR: { - char* pData = colDataGetVarData(pColInfoData, j); - int32_t dataSize = TMIN(sizeof(pBuf) - 1, varDataLen(pData)); - memset(pBuf, 0, dataSize + 1); - strncpy(pBuf, varDataVal(pData), dataSize); - printf(" %15s |", pBuf); - } break; - case TSDB_DATA_TYPE_NCHAR: { - char* pData = colDataGetVarData(pColInfoData, j); - int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); - memset(pBuf, 0, dataSize); - (void)taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); - printf(" %15s |", pBuf); - } break; - default: - break; - } - } - printf("\n"); - } - } -} -#endif - // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) { int32_t size = 2048*1024; @@ -1970,7 +1788,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) if (len >= size - 1) return dumpBuf; for (int32_t j = 0; j < rows; j++) { - len += snprintf(dumpBuf + len, size - len, "%s %d|", flag, j); + len += snprintf(dumpBuf + len, size - len, "%s|", flag); if (len >= size - 1) return dumpBuf; for (int32_t k = 0; k < colNum; k++) { @@ -2033,7 +1851,8 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) len += snprintf(dumpBuf + len, size - len, " %15d |", *(bool*)var); if (len >= size - 1) return dumpBuf; break; - case TSDB_DATA_TYPE_VARCHAR: { + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_GEOMETRY: { memset(pBuf, 0, sizeof(pBuf)); char* pData = colDataGetVarData(pColInfoData, j); int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); @@ -2052,188 +1871,13 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) } break; } } - len += snprintf(dumpBuf + len, size - len, "\n"); + len += snprintf(dumpBuf + len, size - len, "%d\n", j); if (len >= size - 1) return dumpBuf; } len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag); return dumpBuf; } -/** - * @brief TODO: Assume that the final generated result it less than 3M - * - * @param pReq - * @param pDataBlocks - * @param vgId - * @param suid - * - */ -#if 0 -int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlock, STSchema* pTSchema, int32_t vgId, - tb_uid_t suid) { - int32_t bufSize = sizeof(SSubmitReq); - int32_t sz = 1; - for (int32_t i = 0; i < sz; ++i) { - const SDataBlockInfo* pBlkInfo = &pDataBlock->info; - - int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); - bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(colNum)); - bufSize += sizeof(SSubmitBlk); - } - - *pReq = taosMemoryCalloc(1, bufSize); - if (!(*pReq)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - void* pDataBuf = *pReq; - - int32_t msgLen = sizeof(SSubmitReq); - int32_t numOfBlks = 0; - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); - - for (int32_t i = 0; i < sz; ++i) { - int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); - int32_t rows = pDataBlock->info.rows; - - if (colNum <= 1) { - // invalid if only with TS col - continue; - } - - if (rb.nCols != colNum) { - tdSRowSetTpInfo(&rb, colNum, pTSchema->flen); - } - - SSubmitBlk* pSubmitBlk = POINTER_SHIFT(pDataBuf, msgLen); - pSubmitBlk->suid = suid; - pSubmitBlk->uid = pDataBlock->info.id.groupId; - pSubmitBlk->numOfRows = rows; - pSubmitBlk->sversion = pTSchema->version; - - msgLen += sizeof(SSubmitBlk); - int32_t dataLen = 0; - for (int32_t j = 0; j < rows; ++j) { // iterate by row - tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen + dataLen)); // set row buf - bool isStartKey = false; - int32_t offset = 0; - for (int32_t k = 0; k < colNum; ++k) { // iterate by column - SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - STColumn* pCol = &pTSchema->columns[k]; - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - if (!isStartKey) { - isStartKey = true; - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, - offset, k); - continue; // offset should keep 0 for next column - - } else if (colDataIsNull_s(pColInfoData, j)) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NULL, NULL, - false, offset, k); - } else { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, - true, offset, k); - } - break; - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY - if (colDataIsNull_s(pColInfoData, j)) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pColInfoData->info.type, TD_VTYPE_NULL, NULL, - false, offset, k); - } else { - void* data = colDataGetData(pColInfoData, j); - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pColInfoData->info.type, TD_VTYPE_NORM, data, - true, offset, k); - } - break; - } - case TSDB_DATA_TYPE_VARBINARY: - case TSDB_DATA_TYPE_DECIMAL: - case TSDB_DATA_TYPE_BLOB: - case TSDB_DATA_TYPE_JSON: - case TSDB_DATA_TYPE_MEDIUMBLOB: - uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); - break; - default: - if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { - if (colDataIsNull_s(pColInfoData, j)) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NULL, NULL, false, - offset, k); - } else if (pCol->type == pColInfoData->info.type) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, var, true, offset, - k); - } else { - char tv[8] = {0}; - if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { - float v = 0; - GET_TYPED_DATA(v, float, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { - double v = 0; - GET_TYPED_DATA(v, double, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else { - uint64_t v = 0; - GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset, - k); - } - } else { - uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); - } - break; - } - offset += TYPE_BYTES[pCol->type]; // sum/avg would convert to int64_t/uint64_t/double during aggregation - } - tdSRowEnd(&rb); - dataLen += TD_ROW_LEN(rb.pBuf); -#ifdef TD_DEBUG_PRINT_ROW - tdSRowPrint(rb.pBuf, pTSchema, __func__); -#endif - } - - ++numOfBlks; - - pSubmitBlk->dataLen = dataLen; - msgLen += pSubmitBlk->dataLen; - } - - if (numOfBlks > 0) { - (*pReq)->length = msgLen; - - (*pReq)->header.vgId = htonl(vgId); - (*pReq)->header.contLen = htonl(msgLen); - (*pReq)->length = (*pReq)->header.contLen; - (*pReq)->numOfBlocks = htonl(numOfBlks); - SSubmitBlk* blk = (SSubmitBlk*)((*pReq) + 1); - while (numOfBlks--) { - int32_t dataLen = blk->dataLen; - blk->uid = htobe64(blk->uid); - blk->suid = htobe64(blk->suid); - blk->sversion = htonl(blk->sversion); - blk->dataLen = htonl(blk->dataLen); - blk->schemaLen = htonl(blk->schemaLen); - blk->numOfRows = htonl(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + dataLen); - } - } else { - // no valid rows - taosMemoryFreeClear(*pReq); - } - - return TSDB_CODE_SUCCESS; -} -#endif - int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDataBlock, const STSchema* pTSchema, int64_t uid, int32_t vgId, tb_uid_t suid) { SSubmitReq2* pReq = *ppReq; @@ -2399,19 +2043,31 @@ _end: } char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { - if (stbFullName[0] == 0) { + char* pBuf = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); + if (!pBuf) { + return NULL; + } + int32_t code = buildCtbNameByGroupIdImpl(stbFullName, groupId, pBuf); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(pBuf); return NULL; } + return pBuf; +} + +int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, char* cname) { + if (stbFullName[0] == 0) { + return TSDB_CODE_FAILED; + } SArray* tags = taosArrayInit(0, sizeof(SSmlKv)); if (tags == NULL) { - return NULL; + return TSDB_CODE_FAILED; } - void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); if (cname == NULL) { taosArrayDestroy(tags); - return NULL; + return TSDB_CODE_FAILED; } SSmlKv pTag = {.key = "group_id", @@ -2433,9 +2089,9 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { taosArrayDestroy(tags); if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) { - return NULL; + return TSDB_CODE_FAILED; } - return rname.ctbShortName; + return TSDB_CODE_SUCCESS; } int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { @@ -2500,12 +2156,29 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { data += metaSize; dataLen += metaSize; - colSizes[col] = colDataGetLength(pColRes, numOfRows); - dataLen += colSizes[col]; - if (pColRes->pData != NULL) { - memmove(data, pColRes->pData, colSizes[col]); + if (pColRes->reassigned && IS_VAR_DATA_TYPE(pColRes->info.type)) { + colSizes[col] = 0; + for (int32_t row = 0; row < numOfRows; ++row) { + char* pColData = pColRes->pData + pColRes->varmeta.offset[row]; + int32_t colSize = 0; + if (pColRes->info.type == TSDB_DATA_TYPE_JSON) { + colSize = getJsonValueLen(pColData); + } else { + colSize = varDataTLen(pColData); + } + colSizes[col] += colSize; + dataLen += colSize; + memmove(data, pColData, colSize); + data += colSize; + } + } else { + colSizes[col] = colDataGetLength(pColRes, numOfRows); + dataLen += colSizes[col]; + if (pColRes->pData != NULL) { + memmove(data, pColRes->pData, colSizes[col]); + } + data += colSizes[col]; } - data += colSizes[col]; colSizes[col] = htonl(colSizes[col]); // uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type, htonl(colSizes[col]), colSizes[col]); @@ -2514,9 +2187,6 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { *actualLen = dataLen; *groupId = pBlock->info.id.groupId; ASSERT(dataLen > 0); - - uDebug("build data block, actualLen:%d, rows:%d, cols:%d", dataLen, *rows, *cols); - return dataLen; } @@ -2611,3 +2281,149 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { ASSERT(pStart - pData == dataLen); return pStart; } + +void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList) { +// int32_t totalRows = pBlock->info.rows; + int32_t bmLen = BitmapLen(totalRows); + char* pBitmap = NULL; + int32_t maxRows = 0; + + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); + // it is a reserved column for scalar function, and no data in this column yet. + if (pDst->pData == NULL) { + continue; + } + + int32_t numOfRows = 0; + if (IS_VAR_DATA_TYPE(pDst->info.type)) { + int32_t j = 0; + pDst->varmeta.length = 0; + + while (j < totalRows) { + if (pBoolList[j] == 0) { + j += 1; + continue; + } + + if (colDataIsNull_var(pDst, j)) { + colDataSetNull_var(pDst, numOfRows); + } else { + // fix address sanitizer error. p1 may point to memory that will change during realloc of colDataSetVal, first copy it to p2 + char* p1 = colDataGetVarData(pDst, j); + int32_t len = 0; + if (pDst->info.type == TSDB_DATA_TYPE_JSON) { + len = getJsonValueLen(p1); + } else { + len = varDataTLen(p1); + } + char* p2 = taosMemoryMalloc(len); + memcpy(p2, p1, len); + colDataSetVal(pDst, numOfRows, p2, false); + taosMemoryFree(p2); + } + numOfRows += 1; + j += 1; + } + + if (maxRows < numOfRows) { + maxRows = numOfRows; + } + } else { + if (pBitmap == NULL) { + pBitmap = taosMemoryCalloc(1, bmLen); + } + + memcpy(pBitmap, pDst->nullbitmap, bmLen); + memset(pDst->nullbitmap, 0, bmLen); + + int32_t j = 0; + + switch (pDst->info.type) { + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + while (j < totalRows) { + if (pBoolList[j] == 0) { + j += 1; + continue; + } + + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int64_t*)pDst->pData)[numOfRows] = ((int64_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + while (j < totalRows) { + if (pBoolList[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int32_t*)pDst->pData)[numOfRows] = ((int32_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + while (j < totalRows) { + if (pBoolList[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int16_t*)pDst->pData)[numOfRows] = ((int16_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + while (j < totalRows) { + if (pBoolList[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int8_t*)pDst->pData)[numOfRows] = ((int8_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + } + } + + if (maxRows < numOfRows) { + maxRows = numOfRows; + } + } + + pBlock->info.rows = maxRows; + if (pBitmap != NULL) { + taosMemoryFree(pBitmap); + } +} + +int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { + return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock); +} \ No newline at end of file diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 846ca44b83a2f5d0d1b6d0109f2da2200089c62b..7c6939635a290adae506c2b712ef209c7a888326 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1148,7 +1148,8 @@ static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const c switch (type) { case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_NCHAR: { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: { char tmpVal[32] = {0}; strncpy(tmpVal, val, vlen > 31 ? 31 : vlen); printf("%s:%d type:%d vlen:%d, val:\"%s\"\n", tag, ln, (int32_t)type, vlen, tmpVal); @@ -2244,15 +2245,18 @@ static int32_t tColDataUpdateValue72(SColData *pColData, uint8_t *pData, uint32_ } return 0; } +static FORCE_INLINE int32_t tColDataUpdateNothing(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + return 0; +} static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) = { - {NULL, NULL, NULL}, // 0 - {tColDataUpdateValue10, NULL, tColDataUpdateValue12}, // HAS_NONE - {tColDataUpdateValue20, NULL, NULL}, // HAS_NULL - {tColDataUpdateValue30, NULL, tColDataUpdateValue32}, // HAS_NULL|HAS_NONE - {tColDataUpdateValue40, NULL, tColDataUpdateValue42}, // HAS_VALUE - {tColDataUpdateValue50, NULL, tColDataUpdateValue52}, // HAS_VALUE|HAS_NONE - {tColDataUpdateValue60, NULL, tColDataUpdateValue62}, // HAS_VALUE|HAS_NULL - {tColDataUpdateValue70, NULL, tColDataUpdateValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE + {NULL, NULL, NULL}, // 0 + {tColDataUpdateValue10, tColDataUpdateNothing, tColDataUpdateValue12}, // HAS_NONE + {tColDataUpdateValue20, tColDataUpdateNothing, tColDataUpdateNothing}, // HAS_NULL + {tColDataUpdateValue30, tColDataUpdateNothing, tColDataUpdateValue32}, // HAS_NULL|HAS_NONE + {tColDataUpdateValue40, tColDataUpdateNothing, tColDataUpdateValue42}, // HAS_VALUE + {tColDataUpdateValue50, tColDataUpdateNothing, tColDataUpdateValue52}, // HAS_VALUE|HAS_NONE + {tColDataUpdateValue60, tColDataUpdateNothing, tColDataUpdateValue62}, // HAS_VALUE|HAS_NULL + {tColDataUpdateValue70, tColDataUpdateNothing, tColDataUpdateValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE // VALUE NONE NULL }; @@ -2503,18 +2507,21 @@ _exit: return code; } -int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen) { int32_t code = 0; if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { ASSERT(pColData->type == pBind->buffer_type); } - + if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type for (int32_t i = 0; i < pBind->num; ++i) { if (pBind->is_null && pBind->is_null[i]) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; + } else if (pBind->length[i] > buffMaxLen) { + uError("var data length too big, len:%d, max:%d", pBind->length[i], buffMaxLen); + return TSDB_CODE_INVALID_PARA; } else { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( pColData, (uint8_t *)pBind->buffer + pBind->buffer_length * i, pBind->length[i]); @@ -3522,6 +3529,43 @@ static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum } } +static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(uint64_t *)sum = 0; + *(uint64_t *)max = 0; + *(uint64_t *)min = 0; + *numOfNull = 0; + + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: + case (HAS_NONE | HAS_NULL): + *numOfNull = pColData->nVal; + break; + case HAS_VALUE: + *numOfNull = 0; + break; + case (HAS_VALUE | HAS_NULL): + case (HAS_VALUE | HAS_NONE): + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + if (GET_BIT1(pColData->pBitMap, iVal) == 0) { + (*numOfNull)++; + } + } + break; + case (HAS_VALUE | HAS_NONE | HAS_NULL): + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + if (GET_BIT2(pColData->pBitMap, iVal) != 2) { + (*numOfNull)++; + } + } + break; + default: + ASSERT(0); + break; + } +} + void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull) = { NULL, tColDataCalcSMABool, // TSDB_DATA_TYPE_BOOL @@ -3531,16 +3575,17 @@ void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_ tColDataCalcSMABigInt, // TSDB_DATA_TYPE_BIGINT tColDataCalcSMAFloat, // TSDB_DATA_TYPE_FLOAT tColDataCalcSMADouble, // TSDB_DATA_TYPE_DOUBLE - NULL, // TSDB_DATA_TYPE_VARCHAR + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_VARCHAR tColDataCalcSMABigInt, // TSDB_DATA_TYPE_TIMESTAMP - NULL, // TSDB_DATA_TYPE_NCHAR + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_NCHAR tColDataCalcSMAUTinyInt, // TSDB_DATA_TYPE_UTINYINT tColDataCalcSMATinyUSmallInt, // TSDB_DATA_TYPE_USMALLINT tColDataCalcSMAUInt, // TSDB_DATA_TYPE_UINT tColDataCalcSMAUBigInt, // TSDB_DATA_TYPE_UBIGINT - NULL, // TSDB_DATA_TYPE_JSON + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_JSON NULL, // TSDB_DATA_TYPE_VARBINARY NULL, // TSDB_DATA_TYPE_DECIMAL NULL, // TSDB_DATA_TYPE_BLOB - NULL // TSDB_DATA_TYPE_MEDIUMBLOB + NULL, // TSDB_DATA_TYPE_MEDIUMBLOB + NULL // TSDB_DATA_TYPE_GEOMETRY }; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 237a52efe5c141b663130d769d1adb24834c9971..a772efc33cf8ab1cbae6236921df6e3e624683bf 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "tglobal.h" +#include "os.h" #include "tconfig.h" #include "tgrant.h" #include "tlog.h" @@ -33,6 +34,7 @@ char tsFirst[TSDB_EP_LEN] = {0}; char tsSecond[TSDB_EP_LEN] = {0}; char tsLocalFqdn[TSDB_FQDN_LEN] = {0}; char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port +char tsVersionName[16] = "community"; uint16_t tsServerPort = 6030; int32_t tsVersion = 30000000; int32_t tsStatusInterval = 1; // second @@ -45,21 +47,25 @@ bool tsPrintAuth = false; // queue & threads int32_t tsNumOfRpcThreads = 1; -int32_t tsNumOfRpcSessions = 10000; +int32_t tsNumOfRpcSessions = 30000; int32_t tsTimeToGetAvailableConn = 500000; +int32_t tsKeepAliveIdle = 60; + int32_t tsNumOfCommitThreads = 2; int32_t tsNumOfTaskQueueThreads = 4; int32_t tsNumOfMnodeQueryThreads = 4; int32_t tsNumOfMnodeFetchThreads = 1; int32_t tsNumOfMnodeReadThreads = 1; int32_t tsNumOfVnodeQueryThreads = 4; -float tsRatioOfVnodeStreamThreads = 2.0; +float tsRatioOfVnodeStreamThreads = 4.0; int32_t tsNumOfVnodeFetchThreads = 4; int32_t tsNumOfVnodeRsmaThreads = 2; int32_t tsNumOfQnodeQueryThreads = 4; int32_t tsNumOfQnodeFetchThreads = 1; int32_t tsNumOfSnodeStreamThreads = 4; int32_t tsNumOfSnodeWriteThreads = 1; +int32_t tsMaxStreamBackendCache = 128; // M +int32_t tsPQSortMemThreshold = 16; // M // sync raft int32_t tsElectInterval = 25 * 1000; @@ -72,6 +78,14 @@ int64_t tsVndCommitMaxIntervalMs = 600 * 1000; // mnode int64_t tsMndSdbWriteDelta = 200; int64_t tsMndLogRetention = 2000; +int8_t tsGrant = 1; +int32_t tsMndGrantMode = 0; +bool tsMndSkipGrant = false; + +// dnode +int64_t tsDndStart = 0; +int64_t tsDndStartOsUptime = 0; +int64_t tsDndUpTime = 0; // monitor bool tsEnableMonitor = true; @@ -93,6 +107,8 @@ char *tsClientCrashReportUri = "/ccrashreport"; char *tsSvrCrashReportUri = "/dcrashreport"; // schemaless +bool tsSmlDot2Underline = true; +char tsSmlTsDefaultName[TSDB_COL_NAME_LEN] = "_ts"; char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value. // If set to empty system will generate table name using MD5 hash. @@ -100,12 +116,15 @@ char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table // bool tsSmlDataFormat = false; // int32_t tsSmlBatchSize = 10000; +// tmq +int32_t tmqMaxTopicNum = 20; // query int32_t tsQueryPolicy = 1; int32_t tsQueryRspPolicy = 0; int64_t tsQueryMaxConcurrentTables = 200; // unit is TSDB_TABLE_NUM_UNIT -bool tsEnableQueryHb = false; -bool tsEnableScience = false; // on taos-cli show float and doulbe with scientific notation if true +bool tsEnableQueryHb = true; +bool tsEnableScience = false; // on taos-cli show float and doulbe with scientific notation if true +bool tsTtlChangeOnWrite = false; // ttl delete time changes on last write if true int32_t tsQuerySmaOptimize = 0; int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data. bool tsQueryPlannerTrace = false; @@ -117,12 +136,10 @@ int32_t tsRedirectFactor = 2; int32_t tsRedirectMaxPeriod = 1000; int32_t tsMaxRetryWaitTime = 10000; bool tsUseAdapter = false; -int32_t tsSlowLogThreshold = 3; // seconds +int32_t tsMetaCacheMaxSize = -1; // MB +int32_t tsSlowLogThreshold = 3; // seconds int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL; - - - /* * denote if the server needs to compress response message at the application layer to client, including query rsp, * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server. @@ -143,8 +160,8 @@ int32_t tsCompressColData = -1; // count/hyperloglog function always return values in case of all NULL data or Empty data set. int32_t tsCountAlwaysReturnValue = 1; -// 10 ms for sliding time, the value will changed in case of time precision changed -int32_t tsMinSlidingTime = 10; +// 1 ms for sliding time, the value will changed in case of time precision changed +int32_t tsMinSlidingTime = 1; // the maxinum number of distict query result int32_t tsMaxNumOfDistinctResults = 1000 * 10000; @@ -152,8 +169,8 @@ int32_t tsMaxNumOfDistinctResults = 1000 * 10000; // 1 database precision unit for interval time range, changed accordingly int32_t tsMinIntervalTime = 1; -// maximum memory allowed to be allocated for a single csv load (in MB) -int32_t tsMaxMemUsedByInsert = 1024; +// maximum batch rows numbers imported from a single csv load +int32_t tsMaxInsertBatchRows = 1000000; float tsSelectivityRatio = 1.0; int32_t tsTagFilterResCacheSize = 1024 * 10; @@ -180,6 +197,13 @@ bool tsDeployOnSnode = true; * TSDB_TIME_PRECISION_NANO: 60000000000L */ int64_t tsTickPerMin[] = {60000L, 60000000L, 60000000000L}; +/* + * millisecond by default + * for TSDB_TIME_PRECISION_MILLI: 3600000L + * TSDB_TIME_PRECISION_MICRO: 3600000000L + * TSDB_TIME_PRECISION_NANO: 3600000000000L + */ +int64_t tsTickPerHour[] = {3600000L, 3600000000L, 3600000000000L}; // lossy compress 6 char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty @@ -210,6 +234,8 @@ char tsUdfdLdLibPath[512] = ""; bool tsDisableStream = false; int64_t tsStreamBufferSize = 128 * 1024 * 1024; int64_t tsCheckpointInterval = 3 * 60 * 60 * 1000; +bool tsFilterScalarMode = false; +int32_t tsKeepTimeOffset = 0; // latency of data migration #ifndef _STORAGE int32_t taosSetTfsCfg(SConfig *pCfg) { @@ -283,38 +309,38 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input } int32_t taosAddClientLogCfg(SConfig *pCfg) { - if (cfgAddDir(pCfg, "configDir", configDir, 1) != 0) return -1; - if (cfgAddDir(pCfg, "scriptDir", configDir, 1) != 0) return -1; - if (cfgAddDir(pCfg, "logDir", tsLogDir, 1) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, 1) != 0) return -1; - if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddDir(pCfg, "configDir", configDir, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, CFG_SCOPE_CLIENT) != 0) return -1; return 0; } static int32_t taosAddServerLogCfg(SConfig *pCfg) { - if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, 0) != CFG_SCOPE_SERVER) return -1; return 0; } @@ -325,51 +351,60 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { strcpy(defaultFqdn, "localhost"); } - if (cfgAddString(pCfg, "firstEp", "", 1) != 0) return -1; - if (cfgAddString(pCfg, "secondEp", "", 1) != 0) return -1; - if (cfgAddString(pCfg, "fqdn", defaultFqdn, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1; - if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, 1) != 0) return -1; - if (cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, false) != 0) return -1; - if (cfgAddBool(pCfg, "enableScience", tsEnableScience, false) != 0) return -1; - if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, 1) != 0) return -1; - if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1; - if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, true) != 0) return -1; - if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, true) != 0) return -1; - if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, true) != 0) return -1; - if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; - if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; - // if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; - // if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1; - if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1; - if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; - if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1; - if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1; - if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, true) != 0) return -1; - if (cfgAddString(pCfg, "slowLogScope", "", true) != 0) return -1; + if (cfgAddString(pCfg, "firstEp", "", CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlChildTableName", "", CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT) != 0) return -1; + // if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, CFG_SCOPE_CLIENT) != 0) return -1; + // if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, + CFG_SCOPE_CLIENT) != 0) + return -1; + if (cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "slowLogScope", "", CFG_SCOPE_CLIENT) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); - if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH) != 0) return -1; tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 100000); - if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1; tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 10000000); - if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH) != 0) + return -1; + + tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 72000); + if (cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4); if (tsNumOfTaskQueueThreads >= 10) { tsNumOfTaskQueueThreads = 10; } - if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT) != 0) return -1; return 0; } @@ -377,92 +412,98 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { static int32_t taosAddSystemCfg(SConfig *pCfg) { SysNameInfo info = taosGetSysNameInfo(); - if (cfgAddTimezone(pCfg, "timezone", tsTimezoneStr) != 0) return -1; - if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1; - if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1; - if (cfgAddBool(pCfg, "assert", 1, 1) != 0) return -1; - if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1; - if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1; - - if (cfgAddBool(pCfg, "SSE42", tsSSE42Enable, 0) != 0) return -1; - if (cfgAddBool(pCfg, "AVX", tsAVXEnable, 0) != 0) return -1; - if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, 0) != 0) return -1; - if (cfgAddBool(pCfg, "FMA", tsFMAEnable, 0) != 0) return -1; - if (cfgAddBool(pCfg, "SIMD-builtins", tsSIMDBuiltins, 0) != 0) return -1; - if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, 0) != 0) return -1; - - if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "assert", 1, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "enableCoreFile", 1, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1; + + if (cfgAddBool(pCfg, "SSE42", tsSSE42Enable, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "AVX", tsAVXEnable, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "FMA", tsFMAEnable, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "SIMD-builtins", tsSIMDBuiltins, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH) != 0) return -1; + + if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; #if !defined(_ALPINE) - if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; #endif - if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1; - if (cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, 1) != 0) return -1; - if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1; - if (cfgAddString(pCfg, "os nodename", info.nodename, 1) != 0) return -1; - if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1; - if (cfgAddString(pCfg, "os version", info.version, 1) != 0) return -1; - if (cfgAddString(pCfg, "os machine", info.machine, 1) != 0) return -1; - - if (cfgAddString(pCfg, "version", version, 1) != 0) return -1; - if (cfgAddString(pCfg, "compatible_version", compatible_version, 1) != 0) return -1; - if (cfgAddString(pCfg, "gitinfo", gitinfo, 1) != 0) return -1; - if (cfgAddString(pCfg, "buildinfo", buildinfo, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "os sysname", info.sysname, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "os nodename", info.nodename, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "os release", info.release, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "os version", info.version, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "os machine", info.machine, CFG_SCOPE_BOTH) != 0) return -1; + + if (cfgAddString(pCfg, "version", version, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "compatible_version", compatible_version, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "gitinfo", gitinfo, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "buildinfo", buildinfo, CFG_SCOPE_BOTH) != 0) return -1; return 0; } static int32_t taosAddServerCfg(SConfig *pCfg) { - if (cfgAddDir(pCfg, "dataDir", tsDataDir, 0) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0) != 0) return -1; + if (cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER) != 0) return -1; tsNumOfSupportVnodes = tsNumOfCores * 2; tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); - if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, 0) != 0) return -1; - - if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, 0) != 0) return -1; - if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER) != 0) return -1; + + if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, + CFG_SCOPE_SERVER) != 0) + return -1; + if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); - if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH) != 0) return -1; tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 10000); - if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1; tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 1000000); - if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsNumOfRpcSessions, 20, 1000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsNumOfRpcSessions, 20, 1000000, CFG_SCOPE_BOTH) != 0) return -1; + + tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 72000); + if (cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH) != 0) return -1; tsNumOfCommitThreads = tsNumOfCores / 2; tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER) != 0) return -1; tsNumOfMnodeReadThreads = tsNumOfCores / 8; tsNumOfMnodeReadThreads = TRANGE(tsNumOfMnodeReadThreads, 1, 4); - if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER) != 0) return -1; tsNumOfVnodeQueryThreads = tsNumOfCores * 2; tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 100, 0) != 0) return -1; + if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 100, CFG_SCOPE_SERVER) != 0) + return -1; tsNumOfVnodeFetchThreads = tsNumOfCores / 4; tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER) != 0) return -1; tsNumOfVnodeRsmaThreads = tsNumOfCores; tsNumOfVnodeRsmaThreads = TMAX(tsNumOfVnodeRsmaThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER) != 0) return -1; tsNumOfQnodeQueryThreads = tsNumOfCores * 2; tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 4); - if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 4, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER) != 0) return -1; // tsNumOfQnodeFetchThreads = tsNumOfCores / 2; // tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); @@ -470,58 +511,75 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfSnodeStreamThreads = tsNumOfCores / 4; tsNumOfSnodeStreamThreads = TRANGE(tsNumOfSnodeStreamThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER) != 0) + return -1; tsNumOfSnodeWriteThreads = tsNumOfCores / 4; tsNumOfSnodeWriteThreads = TRANGE(tsNumOfSnodeWriteThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER) != 0) return -1; tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); - if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, 0) != 0) + if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, + CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; - - if (cfgAddInt64(pCfg, "vndCommitMaxInterval", tsVndCommitMaxIntervalMs, 1000, 1000 * 60 * 60, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) + return -1; + if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) + return -1; - if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, 0) != 0) return -1; - if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, 0) != 0) return -1; + if (cfgAddInt64(pCfg, "vndCommitMaxInterval", tsVndCommitMaxIntervalMs, 1000, 1000 * 60 * 60, CFG_SCOPE_SERVER) != 0) + return -1; - if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1; - if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, 0) != 0) return -1; - if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, 0) != 0) return -1; + if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "grantMode", tsMndGrantMode, 0, 10000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER) != 0) return -1; + + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER) != 0) return -1; + + if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH) != 0) return -1; + + if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; + + if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER) != 0) return -1; + + if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, + CFG_SCOPE_SERVER) != 0) + return -1; - if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, 0) != 0) return -1; - if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, 0) != 0) return -1; - if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, 0) != 0) return -1; + if (cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 100000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, 0) != 0) return -1; + if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt64(pCfg, "checkpointInterval", tsCheckpointInterval, 0, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, 0) != 0) + if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1; - if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, 0) != 0) return -1; - if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, 0) != 0) return -1; - - if (cfgAddBool(pCfg, "disableStream", tsDisableStream, 0) != 0) return -1; - if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, 0) != 0) return -1; - if (cfgAddInt64(pCfg, "checkpointInterval", tsCheckpointInterval, 0, INT64_MAX, 0) != 0) return -1; - - if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, 0) != 0) return -1; + if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "keepTimeOffset", tsKeepTimeOffset, 0, 23, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER) != 0) return -1; GRANT_CFG_ADD; return 0; @@ -571,6 +629,13 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } + pItem = cfgGetItem(tsCfg, "keepAliveIdle"); + if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { + tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 720000); + pItem->i32 = tsKeepAliveIdle; + pItem->stype = stype; + } + pItem = cfgGetItem(tsCfg, "numOfCommitThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { tsNumOfCommitThreads = numOfCores / 2; @@ -768,10 +833,12 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); + tstrncpy(tsSmlTsDefaultName, cfgGetItem(pCfg, "smlTsDefaultName")->str, TSDB_COL_NAME_LEN); + tsSmlDot2Underline = cfgGetItem(pCfg, "smlDot2Underline")->bval; // tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; // tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32; - tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; + tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32; tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; @@ -779,7 +846,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval; - tsEnableScience = cfgGetItem(pCfg, "enableScience")->bval; + tsEnableScience = cfgGetItem(pCfg, "enableScience")->bval; tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; @@ -788,6 +855,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsUseAdapter = cfgGetItem(pCfg, "useAdapter")->bval; tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; tsQueryMaxConcurrentTables = cfgGetItem(pCfg, "queryMaxConcurrentTables")->i64; + tsMetaCacheMaxSize = cfgGetItem(pCfg, "metaCacheMaxSize")->i32; tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32; if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { return -1; @@ -799,6 +867,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32; tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32; + + tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32; return 0; } @@ -838,6 +908,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32; tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32; + tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32; + tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32; tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32; @@ -863,10 +935,13 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; + tsTtlChangeOnWrite = cfgGetItem(pCfg, "ttlChangeOnWrite")->bval; tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; + tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32; + tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32; tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32; @@ -884,6 +959,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMndSdbWriteDelta = cfgGetItem(pCfg, "mndSdbWriteDelta")->i64; tsMndLogRetention = cfgGetItem(pCfg, "mndLogRetention")->i64; + tsMndSkipGrant = cfgGetItem(pCfg, "skipGrant")->bval; + tsMndGrantMode = cfgGetItem(pCfg, "grantMode")->i32; tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs)); @@ -898,10 +975,21 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64; tsCheckpointInterval = cfgGetItem(pCfg, "checkpointInterval")->i64; + tsFilterScalarMode = cfgGetItem(pCfg, "filterScalarMode")->bval; + tsKeepTimeOffset = cfgGetItem(pCfg, "keepTimeOffset")->i32; + tsMaxStreamBackendCache = cfgGetItem(pCfg, "maxStreamBackendCache")->i32; + tsPQSortMemThreshold = cfgGetItem(pCfg, "pqSortMemThreshold")->i32; + GRANT_CFG_GET; return 0; } +#ifndef TD_ENTERPRISE +static int32_t taosSetReleaseCfg(SConfig *pCfg) { return 0; } +#else +int32_t taosSetReleaseCfg(SConfig *pCfg); +#endif + void taosLocalCfgForbiddenToChange(char *name, bool *forbidden) { int32_t len = strlen(name); char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; @@ -916,7 +1004,7 @@ void taosLocalCfgForbiddenToChange(char *name, bool *forbidden) { *forbidden = false; } -int32_t taosSetCfg(SConfig *pCfg, char *name) { +int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { int32_t len = strlen(name); char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); @@ -964,6 +1052,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { taosSetCoreDump(enableCore); } else if (strcasecmp("enableQueryHb", name) == 0) { tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval; + } else if (strcasecmp("ttlChangeOnWrite", name) == 0) { + tsTtlChangeOnWrite = cfgGetItem(pCfg, "ttlChangeOnWrite")->bval; } break; } @@ -1039,7 +1129,7 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { } else if (strcasecmp("maxNumOfDistinctRes", name) == 0) { tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; } else if (strcasecmp("maxMemUsedByInsert", name) == 0) { - tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; + tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32; } else if (strcasecmp("maxRetryWaitTime", name) == 0) { tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; } @@ -1051,6 +1141,12 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { } break; } + case 'e': { + if (strcasecmp("metaCacheMaxSize", name) == 0) { + atomic_store_32(&tsMetaCacheMaxSize, cfgGetItem(pCfg, "metaCacheMaxSize")->i32); + } + break; + } case 'i': { if (strcasecmp("minimalTmpDirGB", name) == 0) { tsTempSpace.reserved = (int64_t)(((double)cfgGetItem(pCfg, "minimalTmpDirGB")->fval) * 1024 * 1024 * 1024); @@ -1185,6 +1281,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { // tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; // } else if (strcasecmp("smlBatchSize", name) == 0) { // tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32; + } else if (strcasecmp("smlTsDefaultName", name) == 0) { + tstrncpy(tsSmlTsDefaultName, cfgGetItem(pCfg, "smlTsDefaultName")->str, TSDB_COL_NAME_LEN); + } else if (strcasecmp("smlDot2Underline", name) == 0) { + tsSmlDot2Underline = cfgGetItem(pCfg, "smlDot2Underline")->bval; } else if (strcasecmp("shellActivityTimer", name) == 0) { tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; } else if (strcasecmp("supportVnodes", name) == 0) { @@ -1400,6 +1500,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile if (taosSetClientCfg(tsCfg)) return -1; if (taosUpdateServerCfg(tsCfg)) return -1; if (taosSetServerCfg(tsCfg)) return -1; + if (taosSetReleaseCfg(tsCfg)) return -1; if (taosSetTfsCfg(tsCfg) != 0) return -1; } taosSetSystemCfg(tsCfg); @@ -1444,6 +1545,13 @@ void taosCfgDynamicOptions(const char *option, const char *value) { return; } + if (strcasecmp(option, "keepTimeOffset") == 0) { + int32_t newKeepTimeOffset = atoi(value); + uInfo("keepTimeOffset set from %d to %d", tsKeepTimeOffset, newKeepTimeOffset); + tsKeepTimeOffset = newKeepTimeOffset; + return; + } + const char *options[] = { "dDebugFlag", "vDebugFlag", "mDebugFlag", "wDebugFlag", "sDebugFlag", "tsdbDebugFlag", "tqDebugFlag", "fsDebugFlag", "udfDebugFlag", "smaDebugFlag", "idxDebugFlag", "tdbDebugFlag", "tmrDebugFlag", "uDebugFlag", @@ -1507,3 +1615,5 @@ void taosSetAllDebugFlag(int32_t flag, bool rewrite) { taosSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, rewrite); uInfo("all debug flag are set to %d", flag); } + +int8_t taosGranted() { return atomic_load_8(&tsGrant); } diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index c195f5387cb2e01fb128895465980a2569b4d04a..95a5c27cf16835a60945263c5b4dd282d9ee53b6 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -109,7 +109,7 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t taosGetAppName(tmp, NULL); tjsonAddStringToObject(pJson, "appName", tmp); - if (taosGetOsReleaseName(tmp, sizeof(tmp)) == 0) { + if (taosGetOsReleaseName(tmp, NULL, NULL, sizeof(tmp)) == 0) { tjsonAddStringToObject(pJson, "os", tmp); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index aff213fea3412b5f925cdca0e2103426867bbc61..ef0006e7abb1ee386516c185ccf48357f200916c 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -30,6 +30,9 @@ #include "tlog.h" +static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq); +static int32_t tDecodeSBatchDeleteReqCommon(SDecoder *pDecoder, SBatchDeleteReq *pReq); + int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL) { terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; @@ -224,6 +227,7 @@ static int32_t tSerializeSClientHbReq(SEncoder *pEncoder, const SClientHbReq *pR if (tEncodeI64(pEncoder, desc->stime) < 0) return -1; if (tEncodeI64(pEncoder, desc->reqRid) < 0) return -1; if (tEncodeI8(pEncoder, desc->stableQuery) < 0) return -1; + if (tEncodeI8(pEncoder, desc->isSubQuery) < 0) return -1; if (tEncodeCStr(pEncoder, desc->fqdn) < 0) return -1; if (tEncodeI32(pEncoder, desc->subPlanNum) < 0) return -1; @@ -291,6 +295,7 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) if (tDecodeI64(pDecoder, &desc.stime) < 0) return -1; if (tDecodeI64(pDecoder, &desc.reqRid) < 0) return -1; if (tDecodeI8(pDecoder, (int8_t *)&desc.stableQuery) < 0) return -1; + if (tDecodeI8(pDecoder, (int8_t *)&desc.isSubQuery) < 0) return -1; if (tDecodeCStrTo(pDecoder, desc.fqdn) < 0) return -1; if (tDecodeI32(pDecoder, &desc.subPlanNum) < 0) return -1; @@ -830,6 +835,7 @@ int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq if (tEncodeBinary(&encoder, pReq->ast, pReq->astLen) < 0) return -1; } if (tEncodeI64(&encoder, pReq->deleteMark) < 0) return -1; + if (tEncodeI64(&encoder, pReq->lastTs) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -879,6 +885,7 @@ int32_t tDeserializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pR if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; } if (tDecodeI64(&decoder, &pReq->deleteMark) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->lastTs) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -1094,6 +1101,9 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tEncodeI64(&encoder, pReq->qload.timeInFetchQueue) < 0) return -1; if (tEncodeI32(&encoder, pReq->statusSeq) < 0) return -1; + if (tEncodeI64(&encoder, pReq->mload.syncTerm) < 0) return -1; + if (tEncodeI64(&encoder, pReq->mload.roleTimeMs) < 0) return -1; + if (tEncodeI8(&encoder, pReq->clusterCfg.ttlChangeOnWrite) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1138,7 +1148,8 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { for (int32_t i = 0; i < vlen; ++i) { SVnodeLoad vload = {0}; - int64_t reserved = 0; + int64_t reserved64 = 0; + int32_t reserved32 = 0; if (tDecodeI32(&decoder, &vload.vgId) < 0) return -1; if (tDecodeI8(&decoder, &vload.syncState) < 0) return -1; if (tDecodeI8(&decoder, &vload.syncRestore) < 0) return -1; @@ -1150,9 +1161,9 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tDecodeI64(&decoder, &vload.compStorage) < 0) return -1; if (tDecodeI64(&decoder, &vload.pointsWritten) < 0) return -1; if (tDecodeI32(&decoder, &vload.numOfCachedTables) < 0) return -1; - if (tDecodeI32(&decoder, (int32_t *)&reserved) < 0) return -1; - if (tDecodeI64(&decoder, &reserved) < 0) return -1; - if (tDecodeI64(&decoder, &reserved) < 0) return -1; + if (tDecodeI32(&decoder, (int32_t *)&reserved32) < 0) return -1; + if (tDecodeI64(&decoder, &reserved64) < 0) return -1; + if (tDecodeI64(&decoder, &reserved64) < 0) return -1; if (taosArrayPush(pReq->pVloads, &vload) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -1176,6 +1187,19 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tDecodeI64(&decoder, &pReq->qload.timeInFetchQueue) < 0) return -1; if (tDecodeI32(&decoder, &pReq->statusSeq) < 0) return -1; + + pReq->mload.syncTerm = -1; + pReq->mload.roleTimeMs = 0; + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI64(&decoder, &pReq->mload.syncTerm) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->mload.roleTimeMs) < 0) return -1; + } + + pReq->clusterCfg.ttlChangeOnWrite = false; + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->clusterCfg.ttlChangeOnWrite) < 0) return -1; + } + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -1518,6 +1542,9 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) useDb = taosHashIterate(pRsp->useDbs, useDb); } + // since 3.0.7.0 + if (tEncodeI32(pEncoder, pRsp->passVer) < 0) return -1; + return 0; } @@ -1537,6 +1564,7 @@ int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pR } int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRsp) { + char *key = NULL, *value = NULL; pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); @@ -1545,103 +1573,122 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs pRsp->useDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pRsp->createdDbs == NULL || pRsp->readDbs == NULL || pRsp->writeDbs == NULL || pRsp->readTbs == NULL || pRsp->writeTbs == NULL || pRsp->useDbs == NULL) { - return -1; + goto _err; } - if (tDecodeCStrTo(pDecoder, pRsp->user) < 0) return -1; - if (tDecodeI8(pDecoder, &pRsp->superAuth) < 0) return -1; - if (tDecodeI8(pDecoder, &pRsp->sysInfo) < 0) return -1; - if (tDecodeI8(pDecoder, &pRsp->enable) < 0) return -1; - if (tDecodeI8(pDecoder, &pRsp->reserve) < 0) return -1; - if (tDecodeI32(pDecoder, &pRsp->version) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pRsp->user) < 0) goto _err; + if (tDecodeI8(pDecoder, &pRsp->superAuth) < 0) goto _err; + if (tDecodeI8(pDecoder, &pRsp->sysInfo) < 0) goto _err; + if (tDecodeI8(pDecoder, &pRsp->enable) < 0) goto _err; + if (tDecodeI8(pDecoder, &pRsp->reserve) < 0) goto _err; + if (tDecodeI32(pDecoder, &pRsp->version) < 0) goto _err; int32_t numOfCreatedDbs = 0; int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; - if (tDecodeI32(pDecoder, &numOfCreatedDbs) < 0) return -1; - if (tDecodeI32(pDecoder, &numOfReadDbs) < 0) return -1; - if (tDecodeI32(pDecoder, &numOfWriteDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfCreatedDbs) < 0) goto _err; + if (tDecodeI32(pDecoder, &numOfReadDbs) < 0) goto _err; + if (tDecodeI32(pDecoder, &numOfWriteDbs) < 0) goto _err; for (int32_t i = 0; i < numOfCreatedDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(pDecoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) goto _err; int32_t len = strlen(db); - taosHashPut(pRsp->createdDbs, db, len, db, len); + taosHashPut(pRsp->createdDbs, db, len, db, len + 1); } for (int32_t i = 0; i < numOfReadDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(pDecoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) goto _err; int32_t len = strlen(db); - taosHashPut(pRsp->readDbs, db, len, db, len); + taosHashPut(pRsp->readDbs, db, len, db, len + 1); } for (int32_t i = 0; i < numOfWriteDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(pDecoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) goto _err; int32_t len = strlen(db); - taosHashPut(pRsp->writeDbs, db, len, db, len); + taosHashPut(pRsp->writeDbs, db, len, db, len + 1); } if (!tDecodeIsEnd(pDecoder)) { int32_t numOfReadTbs = 0; int32_t numOfWriteTbs = 0; int32_t numOfUseDbs = 0; - if (tDecodeI32(pDecoder, &numOfReadTbs) < 0) return -1; - if (tDecodeI32(pDecoder, &numOfWriteTbs) < 0) return -1; - if (tDecodeI32(pDecoder, &numOfUseDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfReadTbs) < 0) goto _err; + if (tDecodeI32(pDecoder, &numOfWriteTbs) < 0) goto _err; + if (tDecodeI32(pDecoder, &numOfUseDbs) < 0) goto _err; for (int32_t i = 0; i < numOfReadTbs; ++i) { int32_t keyLen = 0; - if (tDecodeI32(pDecoder, &keyLen) < 0) return -1; + if (tDecodeI32(pDecoder, &keyLen) < 0) goto _err; - char *key = taosMemoryCalloc(keyLen + 1, sizeof(char)); - if (tDecodeCStrTo(pDecoder, key) < 0) return -1; + key = taosMemoryCalloc(keyLen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, key) < 0) goto _err; int32_t valuelen = 0; - if (tDecodeI32(pDecoder, &valuelen) < 0) return -1; - char *value = taosMemoryCalloc(valuelen + 1, sizeof(char)); - if (tDecodeCStrTo(pDecoder, value) < 0) return -1; + if (tDecodeI32(pDecoder, &valuelen) < 0) goto _err; + + value = taosMemoryCalloc(valuelen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, value) < 0) goto _err; taosHashPut(pRsp->readTbs, key, strlen(key), value, valuelen + 1); - taosMemoryFree(key); - taosMemoryFree(value); + taosMemoryFreeClear(key); + taosMemoryFreeClear(value); } for (int32_t i = 0; i < numOfWriteTbs; ++i) { int32_t keyLen = 0; - if (tDecodeI32(pDecoder, &keyLen) < 0) return -1; + if (tDecodeI32(pDecoder, &keyLen) < 0) goto _err; - char *key = taosMemoryCalloc(keyLen + 1, sizeof(char)); - if (tDecodeCStrTo(pDecoder, key) < 0) return -1; + key = taosMemoryCalloc(keyLen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, key) < 0) goto _err; int32_t valuelen = 0; - if (tDecodeI32(pDecoder, &valuelen) < 0) return -1; - char *value = taosMemoryCalloc(valuelen + 1, sizeof(char)); - if (tDecodeCStrTo(pDecoder, value) < 0) return -1; + if (tDecodeI32(pDecoder, &valuelen) < 0) goto _err; + + value = taosMemoryCalloc(valuelen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, value) < 0) goto _err; taosHashPut(pRsp->writeTbs, key, strlen(key), value, valuelen + 1); - taosMemoryFree(key); - taosMemoryFree(value); + taosMemoryFreeClear(key); + taosMemoryFreeClear(value); } for (int32_t i = 0; i < numOfUseDbs; ++i) { int32_t keyLen = 0; - if (tDecodeI32(pDecoder, &keyLen) < 0) return -1; + if (tDecodeI32(pDecoder, &keyLen) < 0) goto _err; - char *key = taosMemoryCalloc(keyLen + 1, sizeof(char)); - if (tDecodeCStrTo(pDecoder, key) < 0) return -1; + key = taosMemoryCalloc(keyLen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, key) < 0) goto _err; int32_t ref = 0; - if (tDecodeI32(pDecoder, &ref) < 0) return -1; + if (tDecodeI32(pDecoder, &ref) < 0) goto _err; + taosHashPut(pRsp->useDbs, key, strlen(key), &ref, sizeof(ref)); - taosMemoryFree(key); + taosMemoryFreeClear(key); + } + // since 3.0.7.0 + if (!tDecodeIsEnd(pDecoder)) { + if (tDecodeI32(pDecoder, &pRsp->passVer) < 0) goto _err; + } else { + pRsp->passVer = 0; } } - return 0; +_err: + taosHashCleanup(pRsp->createdDbs); + taosHashCleanup(pRsp->readDbs); + taosHashCleanup(pRsp->writeDbs); + taosHashCleanup(pRsp->writeTbs); + taosHashCleanup(pRsp->readTbs); + taosHashCleanup(pRsp->useDbs); + + taosMemoryFreeClear(key); + taosMemoryFreeClear(value); + return -1; } int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { @@ -1701,6 +1748,7 @@ int32_t tSerializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq) if (tEncodeCStr(&encoder, pReq->fqdn) < 0) return -1; if (tEncodeI32(&encoder, pReq->port) < 0) return -1; if (tEncodeI8(&encoder, pReq->force) < 0) return -1; + if (tEncodeI8(&encoder, pReq->unsafe) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1717,6 +1765,12 @@ int32_t tDeserializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq if (tDecodeCStrTo(&decoder, pReq->fqdn) < 0) return -1; if (tDecodeI32(&decoder, &pReq->port) < 0) return -1; if (tDecodeI8(&decoder, &pReq->force) < 0) return -1; + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->unsafe) < 0) return -1; + } else { + pReq->unsafe = false; + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -2805,7 +2859,25 @@ int32_t tSerializeSUseDbRsp(void *buf, int32_t bufLen, const SUseDbRsp *pRsp) { return tlen; } -int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp) { +int32_t tSerializeSDbHbRspImp(SEncoder *pEncoder, const SDbHbRsp *pRsp) { + if (pRsp->useDbRsp) { + if (tEncodeI8(pEncoder, 1) < 0) return -1; + if (tSerializeSUseDbRspImp(pEncoder, pRsp->useDbRsp) < 0) return -1; + } else { + if (tEncodeI8(pEncoder, 0) < 0) return -1; + } + + if (pRsp->cfgRsp) { + if (tEncodeI8(pEncoder, 1) < 0) return -1; + if (tSerializeSDbCfgRspImpl(pEncoder, pRsp->cfgRsp) < 0) return -1; + } else { + if (tEncodeI8(pEncoder, 0) < 0) return -1; + } + + return 0; +} + +int32_t tSerializeSDbHbBatchRsp(void *buf, int32_t bufLen, SDbHbBatchRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2814,8 +2886,8 @@ int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; for (int32_t i = 0; i < numOfBatch; ++i) { - SUseDbRsp *pUsedbRsp = taosArrayGet(pRsp->pArray, i); - if (tSerializeSUseDbRspImp(&encoder, pUsedbRsp) < 0) return -1; + SDbHbRsp *pDbRsp = taosArrayGet(pRsp->pArray, i); + if (tSerializeSDbHbRspImp(&encoder, pDbRsp) < 0) return -1; } tEndEncode(&encoder); @@ -2868,7 +2940,25 @@ int32_t tDeserializeSUseDbRsp(void *buf, int32_t bufLen, SUseDbRsp *pRsp) { return 0; } -int32_t tDeserializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp) { +int32_t tDeserializeSDbHbRspImp(SDecoder *decoder, SDbHbRsp *pRsp) { + int8_t flag = 0; + if (tDecodeI8(decoder, &flag) < 0) return -1; + if (flag) { + pRsp->useDbRsp = taosMemoryCalloc(1, sizeof(SUseDbRsp)); + if (NULL == pRsp->useDbRsp) return -1; + if (tDeserializeSUseDbRspImp(decoder, pRsp->useDbRsp) < 0) return -1; + } + if (tDecodeI8(decoder, &flag) < 0) return -1; + if (flag) { + pRsp->cfgRsp = taosMemoryCalloc(1, sizeof(SDbCfgRsp)); + if (NULL == pRsp->cfgRsp) return -1; + if (tDeserializeSDbCfgRspImpl(decoder, pRsp->cfgRsp) < 0) return -1; + } + + return 0; +} + +int32_t tDeserializeSDbHbBatchRsp(void *buf, int32_t bufLen, SDbHbBatchRsp *pRsp) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -2877,19 +2967,19 @@ int32_t tDeserializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pR int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; - pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SUseDbRsp)); + pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SDbHbRsp)); if (pRsp->pArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } for (int32_t i = 0; i < numOfBatch; ++i) { - SUseDbRsp usedbRsp = {0}; - if (tDeserializeSUseDbRspImp(&decoder, &usedbRsp) < 0) { + SDbHbRsp rsp = {0}; + if (tDeserializeSDbHbRspImp(&decoder, &rsp) < 0) { tDecoderClear(&decoder); return -1; } - taosArrayPush(pRsp->pArray, &usedbRsp); + taosArrayPush(pRsp->pArray, &rsp); } tEndDecode(&decoder); @@ -2899,11 +2989,27 @@ int32_t tDeserializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pR void tFreeSUsedbRsp(SUseDbRsp *pRsp) { taosArrayDestroy(pRsp->pVgroupInfos); } -void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { +void tFreeSDbHbRsp(SDbHbRsp *pDbRsp) { + if (NULL == pDbRsp) { + return; + } + + if (pDbRsp->useDbRsp) { + tFreeSUsedbRsp(pDbRsp->useDbRsp); + taosMemoryFree(pDbRsp->useDbRsp); + } + + if (pDbRsp->cfgRsp) { + tFreeSDbCfgRsp(pDbRsp->cfgRsp); + taosMemoryFree(pDbRsp->cfgRsp); + } +} + +void tFreeSDbHbBatchRsp(SDbHbBatchRsp *pRsp) { int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); for (int32_t i = 0; i < numOfBatch; ++i) { - SUseDbRsp *pUsedbRsp = taosArrayGet(pRsp->pArray, i); - tFreeSUsedbRsp(pUsedbRsp); + SDbHbRsp *pDbRsp = taosArrayGet(pRsp->pArray, i); + tFreeSDbHbRsp(pDbRsp); } taosArrayDestroy(pRsp->pArray); @@ -2964,59 +3070,6 @@ void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp *pRsp) { taosArrayDestroy(pRsp->pArray); } -int32_t tSerializeSUserPassBatchRsp(void *buf, int32_t bufLen, SUserPassBatchRsp *pRsp) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - if (tStartEncode(&encoder) < 0) return -1; - - int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); - if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; - for (int32_t i = 0; i < numOfBatch; ++i) { - SGetUserPassRsp *pUserPassRsp = taosArrayGet(pRsp->pArray, i); - if (tEncodeCStr(&encoder, pUserPassRsp->user) < 0) return -1; - if (tEncodeI32(&encoder, pUserPassRsp->version) < 0) return -1; - } - tEndEncode(&encoder); - - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} - -int32_t tDeserializeSUserPassBatchRsp(void *buf, int32_t bufLen, SUserPassBatchRsp *pRsp) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - - int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); - if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; - - pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SGetUserPassRsp)); - if (pRsp->pArray == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - for (int32_t i = 0; i < numOfBatch; ++i) { - SGetUserPassRsp rsp = {0}; - if (tDecodeCStrTo(&decoder, rsp.user) < 0) return -1; - if (tDecodeI32(&decoder, &rsp.version) < 0) return -1; - taosArrayPush(pRsp->pArray, &rsp); - } - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; -} - -void tFreeSUserPassBatchRsp(SUserPassBatchRsp *pRsp) { - if(pRsp) { - taosArrayDestroy(pRsp->pArray); - } -} - int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3099,7 +3152,7 @@ int32_t tSerializeSVDropTtlTableReq(void *buf, int32_t bufLen, SVDropTtlTableReq tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI32(&encoder, pReq->timestamp) < 0) return -1; + if (tEncodeI32(&encoder, pReq->timestampSec) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -3112,92 +3165,100 @@ int32_t tDeserializeSVDropTtlTableReq(void *buf, int32_t bufLen, SVDropTtlTableR tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->timestamp) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->timestampSec) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); return 0; } +int32_t tSerializeSDbCfgRspImpl(SEncoder *encoder, const SDbCfgRsp *pRsp) { + if (tEncodeCStr(encoder, pRsp->db) < 0) return -1; + if (tEncodeI64(encoder, pRsp->dbId) < 0) return -1; + if (tEncodeI32(encoder, pRsp->cfgVersion) < 0) return -1; + if (tEncodeI32(encoder, pRsp->numOfVgroups) < 0) return -1; + if (tEncodeI32(encoder, pRsp->numOfStables) < 0) return -1; + if (tEncodeI32(encoder, pRsp->buffer) < 0) return -1; + if (tEncodeI32(encoder, pRsp->cacheSize) < 0) return -1; + if (tEncodeI32(encoder, pRsp->pageSize) < 0) return -1; + if (tEncodeI32(encoder, pRsp->pages) < 0) return -1; + if (tEncodeI32(encoder, pRsp->daysPerFile) < 0) return -1; + if (tEncodeI32(encoder, pRsp->daysToKeep0) < 0) return -1; + if (tEncodeI32(encoder, pRsp->daysToKeep1) < 0) return -1; + if (tEncodeI32(encoder, pRsp->daysToKeep2) < 0) return -1; + if (tEncodeI32(encoder, pRsp->minRows) < 0) return -1; + if (tEncodeI32(encoder, pRsp->maxRows) < 0) return -1; + if (tEncodeI32(encoder, pRsp->walFsyncPeriod) < 0) return -1; + if (tEncodeI16(encoder, pRsp->hashPrefix) < 0) return -1; + if (tEncodeI16(encoder, pRsp->hashSuffix) < 0) return -1; + if (tEncodeI8(encoder, pRsp->walLevel) < 0) return -1; + if (tEncodeI8(encoder, pRsp->precision) < 0) return -1; + if (tEncodeI8(encoder, pRsp->compression) < 0) return -1; + if (tEncodeI8(encoder, pRsp->replications) < 0) return -1; + if (tEncodeI8(encoder, pRsp->strict) < 0) return -1; + if (tEncodeI8(encoder, pRsp->cacheLast) < 0) return -1; + if (tEncodeI32(encoder, pRsp->tsdbPageSize) < 0) return -1; + if (tEncodeI32(encoder, pRsp->walRetentionPeriod) < 0) return -1; + if (tEncodeI32(encoder, pRsp->walRollPeriod) < 0) return -1; + if (tEncodeI64(encoder, pRsp->walRetentionSize) < 0) return -1; + if (tEncodeI64(encoder, pRsp->walSegmentSize) < 0) return -1; + if (tEncodeI32(encoder, pRsp->numOfRetensions) < 0) return -1; + for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { + SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i); + if (tEncodeI64(encoder, pRetension->freq) < 0) return -1; + if (tEncodeI64(encoder, pRetension->keep) < 0) return -1; + if (tEncodeI8(encoder, pRetension->freqUnit) < 0) return -1; + if (tEncodeI8(encoder, pRetension->keepUnit) < 0) return -1; + } + if (tEncodeI8(encoder, pRsp->schemaless) < 0) return -1; + if (tEncodeI16(encoder, pRsp->sstTrigger) < 0) return -1; + + return 0; +} + int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->numOfVgroups) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->numOfStables) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->buffer) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->cacheSize) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->pageSize) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->pages) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->daysPerFile) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->daysToKeep0) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->daysToKeep1) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->daysToKeep2) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->minRows) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->maxRows) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->walFsyncPeriod) < 0) return -1; - if (tEncodeI16(&encoder, pRsp->hashPrefix) < 0) return -1; - if (tEncodeI16(&encoder, pRsp->hashSuffix) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->walLevel) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->precision) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->compression) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->replications) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->strict) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->cacheLast) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->tsdbPageSize) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->walRetentionPeriod) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->walRollPeriod) < 0) return -1; - if (tEncodeI64(&encoder, pRsp->walRetentionSize) < 0) return -1; - if (tEncodeI64(&encoder, pRsp->walSegmentSize) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->numOfRetensions) < 0) return -1; - for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { - SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i); - if (tEncodeI64(&encoder, pRetension->freq) < 0) return -1; - if (tEncodeI64(&encoder, pRetension->keep) < 0) return -1; - if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; - if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; - } - if (tEncodeI8(&encoder, pRsp->schemaless) < 0) return -1; - if (tEncodeI16(&encoder, pRsp->sstTrigger) < 0) return -1; + tSerializeSDbCfgRspImpl(&encoder, pRsp); tEndEncode(&encoder); int32_t tlen = encoder.pos; tEncoderClear(&encoder); return tlen; } -int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->numOfVgroups) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->numOfStables) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->buffer) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->cacheSize) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->pageSize) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->pages) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->daysPerFile) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->daysToKeep0) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->daysToKeep1) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->daysToKeep2) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->minRows) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->maxRows) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->walFsyncPeriod) < 0) return -1; - if (tDecodeI16(&decoder, &pRsp->hashPrefix) < 0) return -1; - if (tDecodeI16(&decoder, &pRsp->hashSuffix) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->walLevel) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->precision) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->compression) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->replications) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->strict) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->cacheLast) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->tsdbPageSize) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->walRetentionPeriod) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->walRollPeriod) < 0) return -1; - if (tDecodeI64(&decoder, &pRsp->walRetentionSize) < 0) return -1; - if (tDecodeI64(&decoder, &pRsp->walSegmentSize) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->numOfRetensions) < 0) return -1; +int32_t tDeserializeSDbCfgRspImpl(SDecoder *decoder, SDbCfgRsp *pRsp) { + if (tDecodeCStrTo(decoder, pRsp->db) < 0) return -1; + if (tDecodeI64(decoder, &pRsp->dbId) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->cfgVersion) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->numOfVgroups) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->numOfStables) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->buffer) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->cacheSize) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->pageSize) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->pages) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->daysPerFile) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->daysToKeep0) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->daysToKeep1) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->daysToKeep2) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->minRows) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->maxRows) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->walFsyncPeriod) < 0) return -1; + if (tDecodeI16(decoder, &pRsp->hashPrefix) < 0) return -1; + if (tDecodeI16(decoder, &pRsp->hashSuffix) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->walLevel) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->precision) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->compression) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->replications) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->strict) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->cacheLast) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->tsdbPageSize) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->walRetentionPeriod) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->walRollPeriod) < 0) return -1; + if (tDecodeI64(decoder, &pRsp->walRetentionSize) < 0) return -1; + if (tDecodeI64(decoder, &pRsp->walSegmentSize) < 0) return -1; + if (tDecodeI32(decoder, &pRsp->numOfRetensions) < 0) return -1; if (pRsp->numOfRetensions > 0) { pRsp->pRetensions = taosArrayInit(pRsp->numOfRetensions, sizeof(SRetention)); if (pRsp->pRetensions == NULL) { @@ -3208,23 +3269,41 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { SRetention rentension = {0}; - if (tDecodeI64(&decoder, &rentension.freq) < 0) return -1; - if (tDecodeI64(&decoder, &rentension.keep) < 0) return -1; - if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; - if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; + if (tDecodeI64(decoder, &rentension.freq) < 0) return -1; + if (tDecodeI64(decoder, &rentension.keep) < 0) return -1; + if (tDecodeI8(decoder, &rentension.freqUnit) < 0) return -1; + if (tDecodeI8(decoder, &rentension.keepUnit) < 0) return -1; if (taosArrayPush(pRsp->pRetensions, &rentension) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } } - if (tDecodeI8(&decoder, &pRsp->schemaless) < 0) return -1; - if (tDecodeI16(&decoder, &pRsp->sstTrigger) < 0) return -1; + if (tDecodeI8(decoder, &pRsp->schemaless) < 0) return -1; + if (tDecodeI16(decoder, &pRsp->sstTrigger) < 0) return -1; + + return 0; +} + +int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDeserializeSDbCfgRspImpl(&decoder, pRsp) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); return 0; } +void tFreeSDbCfgRsp(SDbCfgRsp *pRsp) { + if (NULL == pRsp) { + return; + } + + taosArrayDestroy(pRsp->pRetensions); +} + int32_t tSerializeSUserIndexReq(void *buf, int32_t bufLen, SUserIndexReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3330,6 +3409,7 @@ int32_t tSerializeSTableIndexRsp(void *buf, int32_t bufLen, const STableIndexRsp if (tEncodeCStr(&encoder, pRsp->dbFName) < 0) return -1; if (tEncodeU64(&encoder, pRsp->suid) < 0) return -1; if (tEncodeI32(&encoder, pRsp->version) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->indexSize) < 0) return -1; int32_t num = taosArrayGetSize(pRsp->pIndex); if (tEncodeI32(&encoder, num) < 0) return -1; if (num > 0) { @@ -3375,6 +3455,7 @@ int32_t tDeserializeSTableIndexRsp(void *buf, int32_t bufLen, STableIndexRsp *pR if (tDecodeCStrTo(&decoder, pRsp->dbFName) < 0) return -1; if (tDecodeU64(&decoder, &pRsp->suid) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->version) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->indexSize) < 0) return -1; int32_t num = 0; if (tDecodeI32(&decoder, &num) < 0) return -1; if (num > 0) { @@ -3433,12 +3514,14 @@ int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesR int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) { if (tEncodeCStr(pEncoder, pInfo->name) < 0) return -1; if (tEncodeCStr(pEncoder, pInfo->value) < 0) return -1; + if (tEncodeCStr(pEncoder, pInfo->scope) < 0) return -1; return 0; } int32_t tDecodeSVariablesInfo(SDecoder *pDecoder, SVariablesInfo *pInfo) { if (tDecodeCStrTo(pDecoder, pInfo->name) < 0) return -1; if (tDecodeCStrTo(pDecoder, pInfo->value) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pInfo->scope) < 0) return -1; return 0; } @@ -3649,10 +3732,11 @@ int32_t tSerializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { if (tEncodeCStr(&encoder, pIndexRsp->dbFName) < 0) return -1; if (tEncodeU64(&encoder, pIndexRsp->suid) < 0) return -1; if (tEncodeI32(&encoder, pIndexRsp->version) < 0) return -1; + if (tEncodeI32(&encoder, pIndexRsp->indexSize) < 0) return -1; int32_t num = taosArrayGetSize(pIndexRsp->pIndex); if (tEncodeI32(&encoder, num) < 0) return -1; - for (int32_t i = 0; i < num; ++i) { - STableIndexInfo *pInfo = (STableIndexInfo *)taosArrayGet(pIndexRsp->pIndex, i); + for (int32_t j = 0; j < num; ++j) { + STableIndexInfo *pInfo = (STableIndexInfo *)taosArrayGet(pIndexRsp->pIndex, j); if (tSerializeSTableIndexInfo(&encoder, pInfo) < 0) return -1; } } @@ -3711,13 +3795,14 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { if (tDecodeCStrTo(&decoder, tableIndexRsp.dbFName) < 0) return -1; if (tDecodeU64(&decoder, &tableIndexRsp.suid) < 0) return -1; if (tDecodeI32(&decoder, &tableIndexRsp.version) < 0) return -1; + if (tDecodeI32(&decoder, &tableIndexRsp.indexSize) < 0) return -1; int32_t num = 0; if (tDecodeI32(&decoder, &num) < 0) return -1; if (num > 0) { tableIndexRsp.pIndex = taosArrayInit(num, sizeof(STableIndexInfo)); if (NULL == tableIndexRsp.pIndex) return -1; STableIndexInfo info; - for (int32_t i = 0; i < num; ++i) { + for (int32_t j = 0; j < num; ++j) { if (tDeserializeSTableIndexInfo(&decoder, &info) < 0) return -1; if (NULL == taosArrayPush(tableIndexRsp.pIndex, &info)) { taosMemoryFree(info.expr); @@ -3912,11 +3997,16 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo if (tEncodeI8(&encoder, pReq->withMeta) < 0) return -1; if (tEncodeCStr(&encoder, pReq->subDbName) < 0) return -1; if (TOPIC_SUB_TYPE__DB == pReq->subType) { - } else if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { - if (tEncodeCStr(&encoder, pReq->subStbName) < 0) return -1; } else { - if (tEncodeI32(&encoder, strlen(pReq->ast)) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->ast) < 0) return -1; + if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { + if (tEncodeCStr(&encoder, pReq->subStbName) < 0) return -1; + } + if (pReq->ast && strlen(pReq->ast) > 0) { + if (tEncodeI32(&encoder, strlen(pReq->ast)) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->ast) < 0) return -1; + } else { + if (tEncodeI32(&encoder, 0) < 0) return -1; + } } if (tEncodeI32(&encoder, strlen(pReq->sql)) < 0) return -1; if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; @@ -3942,9 +4032,10 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeI8(&decoder, &pReq->withMeta) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->subDbName) < 0) return -1; if (TOPIC_SUB_TYPE__DB == pReq->subType) { - } else if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { - if (tDecodeCStrTo(&decoder, pReq->subStbName) < 0) return -1; } else { + if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { + if (tDecodeCStrTo(&decoder, pReq->subStbName) < 0) return -1; + } if (tDecodeI32(&decoder, &astLen) < 0) return -1; if (astLen > 0) { pReq->ast = taosMemoryCalloc(1, astLen + 1); @@ -3967,7 +4058,7 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) { taosMemoryFreeClear(pReq->sql); - if (TOPIC_SUB_TYPE__COLUMN == pReq->subType) { + if (TOPIC_SUB_TYPE__DB != pReq->subType) { taosMemoryFreeClear(pReq->ast); } } @@ -4058,6 +4149,7 @@ int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tEncodeCStr(&encoder, pRsp->sVer) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->sDetailVer) < 0) return -1; if (tEncodeI32(&encoder, pRsp->passVer) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->authVer) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -4087,6 +4179,12 @@ int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { } else { pRsp->passVer = 0; } + // since 3.0.7.0 + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI32(&decoder, &pRsp->authVer) < 0) return -1; + } else { + pRsp->authVer = 0; + } tEndDecode(&decoder); @@ -4573,7 +4671,7 @@ int32_t tDeserializeSAlterVnodeReplicaReq(void *buf, int32_t bufLen, SAlterVnode if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; } } - + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -5242,6 +5340,15 @@ int32_t tDeserializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq) { return 0; } +int32_t tDeatroySMqHbReq(SMqHbReq *pReq) { + for (int i = 0; i < taosArrayGetSize(pReq->topics); i++) { + TopicOffsetRows *vgs = taosArrayGet(pReq->topics, i); + if (vgs) taosArrayDestroy(vgs->offsetRows); + } + taosArrayDestroy(pReq->topics); + return 0; +} + int32_t tSerializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -5250,6 +5357,21 @@ int32_t tSerializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) { if (tEncodeI64(&encoder, pReq->consumerId) < 0) return -1; if (tEncodeI32(&encoder, pReq->epoch) < 0) return -1; + int32_t sz = taosArrayGetSize(pReq->topics); + if (tEncodeI32(&encoder, sz) < 0) return -1; + for (int32_t i = 0; i < sz; ++i) { + TopicOffsetRows *vgs = (TopicOffsetRows *)taosArrayGet(pReq->topics, i); + if (tEncodeCStr(&encoder, vgs->topicName) < 0) return -1; + int32_t szVgs = taosArrayGetSize(vgs->offsetRows); + if (tEncodeI32(&encoder, szVgs) < 0) return -1; + for (int32_t j = 0; j < szVgs; ++j) { + OffsetRows *offRows = taosArrayGet(vgs->offsetRows, j); + if (tEncodeI32(&encoder, offRows->vgId) < 0) return -1; + if (tEncodeI64(&encoder, offRows->rows) < 0) return -1; + if (tEncodeSTqOffsetVal(&encoder, &offRows->offset) < 0) return -1; + } + } + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -5266,6 +5388,68 @@ int32_t tDeserializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) { if (tDecodeI64(&decoder, &pReq->consumerId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->epoch) < 0) return -1; + int32_t sz = 0; + if (tDecodeI32(&decoder, &sz) < 0) return -1; + if (sz > 0) { + pReq->topics = taosArrayInit(sz, sizeof(TopicOffsetRows)); + if (NULL == pReq->topics) return -1; + for (int32_t i = 0; i < sz; ++i) { + TopicOffsetRows *data = taosArrayReserve(pReq->topics, 1); + tDecodeCStrTo(&decoder, data->topicName); + int32_t szVgs = 0; + if (tDecodeI32(&decoder, &szVgs) < 0) return -1; + if (szVgs > 0) { + data->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows)); + if (NULL == data->offsetRows) return -1; + for (int32_t j = 0; j < szVgs; ++j) { + OffsetRows *offRows = taosArrayReserve(data->offsetRows, 1); + if (tDecodeI32(&decoder, &offRows->vgId) < 0) return -1; + if (tDecodeI64(&decoder, &offRows->rows) < 0) return -1; + if (tDecodeSTqOffsetVal(&decoder, &offRows->offset) < 0) return -1; + } + } + } + } + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + +int32_t tSerializeSMqSeekReq(void *buf, int32_t bufLen, SMqSeekReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + if (buf != NULL) { + buf = (char *)buf + headLen; + bufLen -= headLen; + } + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pReq->consumerId) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->subKey) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + + if (buf != NULL) { + SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen); + pHead->vgId = htonl(pReq->head.vgId); + pHead->contLen = htonl(tlen + headLen); + } + + return tlen + headLen; +} + +int32_t tDeserializeSMqSeekReq(void *buf, int32_t bufLen, SMqSeekReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + + SDecoder decoder = {0}; + tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->consumerId) < 0) return -1; + tDecodeCStrTo(&decoder, pReq->subKey); tEndDecode(&decoder); @@ -5459,9 +5643,9 @@ int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { int32_t headLen = sizeof(SMsgHead); -// SMsgHead *pHead = buf; -// pHead->vgId = pReq->head.vgId; -// pHead->contLen = pReq->head.contLen; + // SMsgHead *pHead = buf; + // pHead->vgId = pReq->head.vgId; + // pHead->contLen = pReq->head.contLen; SDecoder decoder = {0}; tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); @@ -6026,6 +6210,7 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS } if (tEncodeI64(&encoder, pReq->deleteMark) < 0) return -1; if (tEncodeI8(&encoder, pReq->igUpdate) < 0) return -1; + if (tEncodeI64(&encoder, pReq->lastTs) < 0) return -1; tEndEncode(&encoder); @@ -6111,6 +6296,7 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea if (tDecodeI64(&decoder, &pReq->deleteMark) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igUpdate) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->lastTs) < 0) return -1; tEndDecode(&decoder); @@ -6177,6 +6363,9 @@ int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStrea } void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { + if (NULL == pReq) { + return; + } taosArrayDestroy(pReq->pTags); taosMemoryFreeClear(pReq->sql); taosMemoryFreeClear(pReq->ast); @@ -6261,7 +6450,7 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { if (tEncodeI32v(pCoder, pReq->flags) < 0) return -1; if (tEncodeCStr(pCoder, pReq->name) < 0) return -1; if (tEncodeI64(pCoder, pReq->uid) < 0) return -1; - if (tEncodeI64(pCoder, pReq->ctime) < 0) return -1; + if (tEncodeI64(pCoder, pReq->btime) < 0) return -1; if (tEncodeI32(pCoder, pReq->ttl) < 0) return -1; if (tEncodeI8(pCoder, pReq->type) < 0) return -1; if (tEncodeI32(pCoder, pReq->commentLen) < 0) return -1; @@ -6296,7 +6485,7 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1; if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1; if (tDecodeI64(pCoder, &pReq->uid) < 0) return -1; - if (tDecodeI64(pCoder, &pReq->ctime) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->btime) < 0) return -1; if (tDecodeI32(pCoder, &pReq->ttl) < 0) return -1; if (tDecodeI8(pCoder, &pReq->type) < 0) return -1; if (tDecodeI32(pCoder, &pReq->commentLen) < 0) return -1; @@ -6761,14 +6950,13 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) { default: break; } + if (tEncodeI64(pEncoder, pReq->ctimeMs) < 0) return -1; tEndEncode(pEncoder); return 0; } -int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) { - if (tStartDecode(pDecoder) < 0) return -1; - +static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq) { if (tDecodeCStr(pDecoder, &pReq->tbName) < 0) return -1; if (tDecodeI8(pDecoder, &pReq->action) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->colId) < 0) return -1; @@ -6812,6 +7000,31 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) { default: break; } + return 0; +} + +int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeSVAlterTbReqCommon(pDecoder, pReq) < 0) return -1; + + pReq->ctimeMs = 0; + if (!tDecodeIsEnd(pDecoder)) { + if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1; + } + + tEndDecode(pDecoder); + return 0; +} + +int32_t tDecodeSVAlterTbReqSetCtime(SDecoder *pDecoder, SVAlterTbReq *pReq, int64_t ctimeMs) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeSVAlterTbReqCommon(pDecoder, pReq) < 0) return -1; + + pReq->ctimeMs = 0; + if (!tDecodeIsEnd(pDecoder)) { + *(int64_t *)(pDecoder->data + pDecoder->pos) = ctimeMs; + if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1; + } tEndDecode(pDecoder); return 0; @@ -6990,15 +7203,15 @@ int32_t tDecodeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffsetVal) { int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) { if (pVal->type == TMQ_OFFSET__RESET_NONE) { - snprintf(buf, maxLen, "offset(reset to none)"); - } else if (pVal->type == TMQ_OFFSET__RESET_EARLIEAST) { - snprintf(buf, maxLen, "offset(reset to earlieast)"); + snprintf(buf, maxLen, "none"); + } else if (pVal->type == TMQ_OFFSET__RESET_EARLIEST) { + snprintf(buf, maxLen, "earliest"); } else if (pVal->type == TMQ_OFFSET__RESET_LATEST) { - snprintf(buf, maxLen, "offset(reset to latest)"); + snprintf(buf, maxLen, "latest"); } else if (pVal->type == TMQ_OFFSET__LOG) { - snprintf(buf, maxLen, "offset(log) ver:%" PRId64, pVal->version); + snprintf(buf, maxLen, "wal:%" PRId64, pVal->version); } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA || pVal->type == TMQ_OFFSET__SNAPSHOT_META) { - snprintf(buf, maxLen, "offset(snapshot) uid:%" PRId64 " ts:%" PRId64, pVal->uid, pVal->ts); + snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64, pVal->uid, pVal->ts); } else { return TSDB_CODE_INVALID_PARA; } @@ -7015,10 +7228,8 @@ bool tOffsetEqual(const STqOffsetVal *pLeft, const STqOffsetVal *pRight) { } else if (pLeft->type == TMQ_OFFSET__SNAPSHOT_META) { return pLeft->uid == pRight->uid; } else { + uError("offset type:%d", pLeft->type); ASSERT(0); - /*ASSERT(pLeft->type == TMQ_OFFSET__RESET_NONE || pLeft->type == TMQ_OFFSET__RESET_EARLIEAST ||*/ - /*pLeft->type == TMQ_OFFSET__RESET_LATEST);*/ - /*return true;*/ } } return false; @@ -7036,13 +7247,13 @@ int32_t tDecodeSTqOffset(SDecoder *pDecoder, STqOffset *pOffset) { return 0; } -int32_t tEncodeMqVgOffset(SEncoder* pEncoder, const SMqVgOffset* pOffset) { +int32_t tEncodeMqVgOffset(SEncoder *pEncoder, const SMqVgOffset *pOffset) { if (tEncodeSTqOffset(pEncoder, &pOffset->offset) < 0) return -1; if (tEncodeI64(pEncoder, pOffset->consumerId) < 0) return -1; return 0; } -int32_t tDecodeMqVgOffset(SDecoder* pDecoder, SMqVgOffset* pOffset) { +int32_t tDecodeMqVgOffset(SDecoder *pDecoder, SMqVgOffset *pOffset) { if (tDecodeSTqOffset(pDecoder, &pOffset->offset) < 0) return -1; if (tDecodeI64(pDecoder, &pOffset->consumerId) < 0) return -1; return 0; @@ -7090,6 +7301,7 @@ int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) { if (tEncodeCStr(pCoder, pRes->tableFName) < 0) return -1; if (tEncodeCStr(pCoder, pRes->tsColName) < 0) return -1; + if (tEncodeI64(pCoder, pRes->ctimeMs) < 0) return -1; return 0; } @@ -7109,6 +7321,11 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) { if (tDecodeCStrTo(pCoder, pRes->tableFName) < 0) return -1; if (tDecodeCStrTo(pCoder, pRes->tsColName) < 0) return -1; + + pRes->ctimeMs = 0; + if (!tDecodeIsEnd(pCoder)) { + if (tDecodeI64(pCoder, &pRes->ctimeMs) < 0) return -1; + } return 0; } @@ -7207,27 +7424,8 @@ void tDeleteMqDataRsp(SMqDataRsp *pRsp) { } int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { - if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1; - if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; - if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1; - if (pRsp->blockNum != 0) { - if (tEncodeI8(pEncoder, pRsp->withTbName) < 0) return -1; - if (tEncodeI8(pEncoder, pRsp->withSchema) < 0) return -1; + if (tEncodeMqDataRsp(pEncoder, (const SMqDataRsp *)pRsp) < 0) return -1; - for (int32_t i = 0; i < pRsp->blockNum; i++) { - int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i); - void *data = taosArrayGetP(pRsp->blockData, i); - if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1; - if (pRsp->withSchema) { - SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i); - if (tEncodeSSchemaWrapper(pEncoder, pSW) < 0) return -1; - } - if (pRsp->withTbName) { - char *tbName = (char *)taosArrayGetP(pRsp->blockTbName, i); - if (tEncodeCStr(pEncoder, tbName) < 0) return -1; - } - } - } if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { for (int32_t i = 0; i < pRsp->createTableNum; i++) { @@ -7240,46 +7438,8 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { } int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) { - if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1; - if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; - if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1; - if (pRsp->blockNum != 0) { - pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void *)); - pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t)); - if (tDecodeI8(pDecoder, &pRsp->withTbName) < 0) return -1; - if (tDecodeI8(pDecoder, &pRsp->withSchema) < 0) return -1; - if (pRsp->withTbName) { - pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void *)); - } - if (pRsp->withSchema) { - pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void *)); - } - - for (int32_t i = 0; i < pRsp->blockNum; i++) { - void *data; - uint64_t bLen; - if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1; - taosArrayPush(pRsp->blockData, &data); - int32_t len = bLen; - taosArrayPush(pRsp->blockDataLen, &len); + if (tDecodeMqDataRsp(pDecoder, (SMqDataRsp *)pRsp) < 0) return -1; - if (pRsp->withSchema) { - SSchemaWrapper *pSW = (SSchemaWrapper *)taosMemoryCalloc(1, sizeof(SSchemaWrapper)); - if (pSW == NULL) return -1; - if (tDecodeSSchemaWrapper(pDecoder, pSW) < 0) { - taosMemoryFree(pSW); - return -1; - } - taosArrayPush(pRsp->blockSchema, &pSW); - } - - if (pRsp->withTbName) { - char *tbName; - if (tDecodeCStrAlloc(pDecoder, &tbName) < 0) return -1; - taosArrayPush(pRsp->blockTbName, &tbName); - } - } - } if (tDecodeI32(pDecoder, &pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t)); @@ -7332,10 +7492,11 @@ int32_t tEncodeSBatchDeleteReq(SEncoder *pEncoder, const SBatchDeleteReq *pReq) SSingleDeleteReq *pOneReq = taosArrayGet(pReq->deleteReqs, i); if (tEncodeSSingleDeleteReq(pEncoder, pOneReq) < 0) return -1; } + if (tEncodeI64(pEncoder, pReq->ctimeMs) < 0) return -1; return 0; } -int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) { +static int32_t tDecodeSBatchDeleteReqCommon(SDecoder *pDecoder, SBatchDeleteReq *pReq) { if (tDecodeI64(pDecoder, &pReq->suid) < 0) return -1; int32_t sz; if (tDecodeI32(pDecoder, &sz) < 0) return -1; @@ -7349,6 +7510,27 @@ int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) { return 0; } +int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) { + if (tDecodeSBatchDeleteReqCommon(pDecoder, pReq)) return -1; + + pReq->ctimeMs = 0; + if (!tDecodeIsEnd(pDecoder)) { + if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1; + } + return 0; +} + +int32_t tDecodeSBatchDeleteReqSetCtime(SDecoder *pDecoder, SBatchDeleteReq *pReq, int64_t ctimeMs) { + if (tDecodeSBatchDeleteReqCommon(pDecoder, pReq)) return -1; + + pReq->ctimeMs = 0; + if (!tDecodeIsEnd(pDecoder)) { + *(int64_t *)(pDecoder->data + pDecoder->pos) = ctimeMs; + if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1; + } + return 0; +} + static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubmitTbData) { if (tStartEncode(pCoder) < 0) return -1; @@ -7383,6 +7565,7 @@ static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubm pCoder->pos += rows[iRow]->len; } } + if (tEncodeI64(pCoder, pSubmitTbData->ctimeMs) < 0) return -1; tEndEncode(pCoder); return 0; @@ -7463,6 +7646,14 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa } } + pSubmitTbData->ctimeMs = 0; + if (!tDecodeIsEnd(pCoder)) { + if (tDecodeI64(pCoder, &pSubmitTbData->ctimeMs) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + tEndDecode(pCoder); _exit: diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 9d381ce15fa762b74cecf23eeec1fa6667a10b99..039f436505f11d63f27f74af18edca60b0c09456 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -126,6 +126,9 @@ void tdSCellValPrint(SCellVal *pVal, int8_t colType) { case TSDB_DATA_TYPE_JSON: printf("JSON "); break; + case TSDB_DATA_TYPE_GEOMETRY: + printf("GEOMETRY "); + break; case TSDB_DATA_TYPE_VARBINARY: printf("VARBIN "); break; @@ -353,7 +356,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { } } else { varDataLen += sizeof(VarDataLenT); - if (pTColumn->type == TSDB_DATA_TYPE_VARCHAR) { + if (pTColumn->type == TSDB_DATA_TYPE_VARCHAR || pTColumn->type == TSDB_DATA_TYPE_GEOMETRY) { varDataLen += CHAR_BYTES; if (maxVarDataLen < CHAR_BYTES + sizeof(VarDataLenT)) { maxVarDataLen = CHAR_BYTES + sizeof(VarDataLenT); @@ -420,7 +423,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { val = (const void *)&pColVal->value.val; } } else { - pColVal = NULL; + // pColVal = NULL; valType = TD_VTYPE_NONE; } diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index dcd539bd91e11c570a9f779f9c1c9a3811765c41..e9313e0591dd6f01f67844f85ba439fe72be1e57 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -25,46 +25,6 @@ #include "tlog.h" -/* - * mktime64 - Converts date to seconds. - * Converts Gregorian date to seconds since 1970-01-01 00:00:00. - * Assumes input in normal date format, i.e. 1980-12-31 23:59:59 - * => year=1980, mon=12, day=31, hour=23, min=59, sec=59. - * - * [For the Julian calendar (which was used in Russia before 1917, - * Britain & colonies before 1752, anywhere else before 1582, - * and is still in use by some communities) leave out the - * -year/100+year/400 terms, and add 10.] - * - * This algorithm was first published by Gauss (I think). - * - * A leap second can be indicated by calling this function with sec as - * 60 (allowable under ISO 8601). The leap second is treated the same - * as the following second since they don't exist in UNIX time. - * - * An encoding of midnight at the end of the day as 24:00:00 - ie. midnight - * tomorrow - (allowable under ISO 8601) is supported. - */ -static int64_t user_mktime64(const uint32_t year0, const uint32_t mon0, const uint32_t day, const uint32_t hour, - const uint32_t min, const uint32_t sec, int64_t time_zone) { - uint32_t mon = mon0, year = year0; - - /* 1..12 -> 11,12,1..10 */ - if (0 >= (int32_t)(mon -= 2)) { - mon += 12; /* Puts Feb last since it has leap day */ - year -= 1; - } - - // int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) + - // year*365 - 719499)*24 + hour)*60 + min)*60 + sec); - int64_t res; - res = 367 * ((int64_t)mon) / 12; - res += year / 4 - year / 100 + year / 400 + day + ((int64_t)year) * 365 - 719499; - res = res * 24; - res = ((res + hour) * 60 + min) * 60 + sec; - - return (res + time_zone); -} // ==== mktime() kernel code =================// static int64_t m_deltaUtc = 0; @@ -82,6 +42,7 @@ static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_ static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static char* forwardToTimeStringEnd(char* str); static bool checkTzPresent(const char* str, int32_t len); +static int32_t parseTimezone(char* str, int64_t* tzOffset); static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) = { parseLocaltime, parseLocaltimeDst}; @@ -92,13 +53,13 @@ int32_t taosParseTime(const char* timestr, int64_t* utime, int32_t len, int32_t if (checkTzPresent(timestr, len)) { return parseTimeWithTz(timestr, utime, timePrec, 'T'); } else { - return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 'T'); + return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 'T'); } } else { if (checkTzPresent(timestr, len)) { return parseTimeWithTz(timestr, utime, timePrec, 0); } else { - return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 0); + return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 0); } } } @@ -713,16 +674,12 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { return t; } - if (unit != 'n' && unit != 'y') { + if (!IS_CALENDAR_TIME_DURATION(unit)) { return t + duration; } // The following code handles the y/n time duration - int64_t numOfMonth = duration; - if (unit == 'y') { - numOfMonth *= 12; - } - + int64_t numOfMonth = (unit == 'y')? duration*12:duration; int64_t fraction = t % TSDB_TICK_PER_SECOND(precision); struct tm tm; @@ -741,6 +698,7 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char ekey = skey; skey = tmp; } + if (unit != 'n' && unit != 'y') { return (int32_t)((ekey - skey) / interval); } @@ -764,13 +722,16 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char return (emon - smon) / (int32_t)interval; } -int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision) { +int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { if (pInterval->sliding == 0 && pInterval->interval == 0) { - return t; + return ts; } - int64_t start = t; - if (pInterval->slidingUnit == 'n' || pInterval->slidingUnit == 'y') { + int64_t start = ts; + int32_t precision = pInterval->precision; + + if (IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { + start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; time_t tt = (time_t)start; @@ -792,44 +753,72 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio start = (int64_t)(taosMktime(&tm) * TSDB_TICK_PER_SECOND(precision)); } else { - int64_t delta = t - pInterval->interval; - int32_t factor = (delta >= 0) ? 1 : -1; + if (IS_CALENDAR_TIME_DURATION(pInterval->intervalUnit)) { + int64_t news = (ts / pInterval->sliding) * pInterval->sliding; + ASSERT(news <= ts); + + if (news <= ts) { + int64_t prev = news; + int64_t newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + + if (newe < ts) { // move towards the greater endpoint + while(newe < ts && news < ts) { + news += pInterval->sliding; + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } + + prev = news; + } else { + while (newe >= ts) { + prev = news; + news -= pInterval->sliding; + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } + } + + return prev; + } + } else { + int64_t delta = ts - pInterval->interval; + int32_t factor = (delta >= 0) ? 1 : -1; - start = (delta / pInterval->sliding + factor) * pInterval->sliding; + start = (delta / pInterval->sliding + factor) * pInterval->sliding; - if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { - /* - * here we revised the start time of day according to the local time zone, - * but in case of DST, the start time of one day need to be dynamically decided. - */ - // todo refactor to extract function that is available for Linux/Windows/Mac platform + if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { + /* + * here we revised the start time of day according to the local time zone, + * but in case of DST, the start time of one day need to be dynamically decided. + */ + // todo refactor to extract function that is available for Linux/Windows/Mac platform #if defined(WINDOWS) && _MSC_VER >= 1900 - // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char** tzname = _tzname; + // see + // https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 + int64_t timezone = _timezone; + int32_t daylight = _daylight; + char** tzname = _tzname; #endif - start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); - } + start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); + } - int64_t end = 0; + int64_t end = 0; - // not enough time range - if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - while (end < t) { // move forward to the correct time window - start += pInterval->sliding; + // not enough time range + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { + end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; + while (end < ts) { // move forward to the correct time window + start += pInterval->sliding; - if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = start + pInterval->interval - 1; - } else { - end = INT64_MAX; - break; + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { + end = start + pInterval->interval - 1; + } else { + end = INT64_MAX; + break; + } } + } else { + end = INT64_MAX; } - } else { - end = INT64_MAX; } } @@ -841,10 +830,10 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio // try to move current window to the left-hande-side, due to the offset effect. int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - int64_t newEnd = end; - while (newEnd >= t) { - end = newEnd; - newEnd = taosTimeAdd(newEnd, -pInterval->sliding, pInterval->slidingUnit, precision); + int64_t newe = end; + while (newe >= ts) { + end = newe; + newe = taosTimeAdd(newe, -pInterval->sliding, pInterval->slidingUnit, precision); } start = taosTimeAdd(end, -pInterval->interval, pInterval->intervalUnit, precision) + 1; @@ -940,7 +929,7 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) default: fractionLen = 0; - ASSERT(false); + return; } if (taosLocalTime(", &ptm, buf) == NULL) { diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index f415bd20cd3a1a98d0478dd5ae281d02f89cd39d..aabbff6f04d602e2fc78236bf7b0fcf8df4aee42 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -296,7 +296,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType)); int32_t trueLen = pBlock->tag.nLen; - if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR || pBlock->tag.nType == TSDB_DATA_TYPE_GEOMETRY) { metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen); } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { @@ -378,7 +378,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { // NOTE: mix types tags are not supported size_t sz = 0; - if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR || pBlock->tag.nType == TSDB_DATA_TYPE_GEOMETRY) { char* tp = taosMemoryRealloc(pBlock->tag.pz, pBlock->tag.nLen + 1); ASSERT(tp != NULL); diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index c83bdc0e3293e97c01a2eb7aed24652e9fd9775d..39255cff3a8d22136fd785914e131d6c2c7b2d92 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -17,7 +17,7 @@ #include "ttypes.h" #include "tcompression.h" -const int32_t TYPE_BYTES[16] = { +const int32_t TYPE_BYTES[21] = { -1, // TSDB_DATA_TYPE_NULL CHAR_BYTES, // TSDB_DATA_TYPE_BOOL CHAR_BYTES, // TSDB_DATA_TYPE_TINYINT @@ -34,6 +34,11 @@ const int32_t TYPE_BYTES[16] = { INT_BYTES, // TSDB_DATA_TYPE_UINT sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT TSDB_MAX_JSON_TAG_LEN, // TSDB_DATA_TYPE_JSON + TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_VARBINARY: placeholder, not implemented + TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_DECIMAL: placeholder, not implemented + TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_BLOB: placeholder, not implemented + TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_MEDIUMBLOB: placeholder, not implemented + sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_GEOMETRY }; tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { @@ -56,6 +61,11 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt}, {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint}, {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString}, + {TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, NULL, NULL}, // placeholder, not implemented + {TSDB_DATA_TYPE_DECIMAL, 7, 1, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented + {TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented + {TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented + {TSDB_DATA_TYPE_GEOMETRY, 8, 1, "GEOMETRY", 0, 0, tsCompressString, tsDecompressString}, }; static float floatMin = -FLT_MAX, floatMax = FLT_MAX; @@ -125,6 +135,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { *((int64_t *)val) = GET_INT64_VAL(src); break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_GEOMETRY: varDataCopy(val, src); break; case TSDB_DATA_TYPE_NCHAR: diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index db69fe9d48ae14e880a82039bc084abbb80b7d74..cb4a42eb109a1b680de774074cfe7806ba7e3f42 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -121,7 +121,8 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin break; } - case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_GEOMETRY: { // todo refactor, extract a method pVar->pz = taosMemoryCalloc(len + 1, sizeof(char)); memcpy(pVar->pz, pz, len); pVar->nLen = (int32_t)len; @@ -140,7 +141,7 @@ void taosVariantDestroy(SVariant *pVar) { if (pVar == NULL) return; if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR || - pVar->nType == TSDB_DATA_TYPE_JSON) { + pVar->nType == TSDB_DATA_TYPE_JSON || pVar->nType == TSDB_DATA_TYPE_GEOMETRY) { taosMemoryFreeClear(pVar->pz); pVar->nLen = 0; } @@ -152,7 +153,7 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { pDst->nType = pSrc->nType; if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR || - pSrc->nType == TSDB_DATA_TYPE_JSON) { + pSrc->nType == TSDB_DATA_TYPE_JSON || pSrc->nType == TSDB_DATA_TYPE_GEOMETRY) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; char *p = taosMemoryRealloc(pDst->pz, len); ASSERT(p); @@ -184,7 +185,7 @@ int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) { return 1; } - if (p1->nType == TSDB_DATA_TYPE_BINARY || p1->nType == TSDB_DATA_TYPE_NCHAR) { + if (p1->nType == TSDB_DATA_TYPE_BINARY || p1->nType == TSDB_DATA_TYPE_NCHAR || p1->nType == TSDB_DATA_TYPE_GEOMETRY) { if (p1->nLen == p2->nLen) { return memcmp(p1->pz, p2->pz, p1->nLen); } else { @@ -237,6 +238,7 @@ char *taosVariantGet(SVariant *pVar, int32_t type) { return (char *)&pVar->f; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_GEOMETRY: return (char *)pVar->pz; case TSDB_DATA_TYPE_NCHAR: return (char *)pVar->ucs4; diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index 4f8652d02cc0602583edff7f59354d9d093275fc..b1083e45905ca5b1850022036e6e5978fbbcc9a4 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -235,7 +235,8 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) { case TSDB_DATA_TYPE_DOUBLE: printf("%lf ", cv->value.d); break; - case TSDB_DATA_TYPE_VARCHAR: { + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_GEOMETRY: { char tv[15] = {0}; snprintf(tv, 15, "%s", cv->value.pData); printf("%s ", tv); @@ -337,7 +338,8 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) { sscanf(rawVal, "%lf", &rawSVal.d); EXPECT_DOUBLE_EQ(cv->value.d, rawSVal.d); } break; - case TSDB_DATA_TYPE_VARCHAR: { + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_GEOMETRY: { EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData); } break; case TSDB_DATA_TYPE_TIMESTAMP: { diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 989bff39849f1157a65b0ea606ea1763c337daf5..3c08714218dfa861958e27080a23a407e7323eb5 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -19,6 +19,9 @@ #include "tconfig.h" #include "tglobal.h" #include "version.h" +#ifdef TD_JEMALLOC_ENABLED +#include "jemalloc/jemalloc.h" +#endif #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -87,6 +90,18 @@ static void dmStopDnode(int signum, void *sigInfo, void *context) { } void dmLogCrash(int signum, void *sigInfo, void *context) { + // taosIgnSignal(SIGTERM); + // taosIgnSignal(SIGHUP); + // taosIgnSignal(SIGINT); + // taosIgnSignal(SIGBREAK); + +#ifndef WINDOWS + taosIgnSignal(SIGBUS); +#endif + taosIgnSignal(SIGABRT); + taosIgnSignal(SIGFPE); + taosIgnSignal(SIGSEGV); + char *pMsg = NULL; const char *flags = "UTL FATAL "; ELogLevel level = DEBUG_FATAL; @@ -243,6 +258,10 @@ static void taosCleanupArgs() { } int main(int argc, char const *argv[]) { +#ifdef TD_JEMALLOC_ENABLED + bool jeBackgroundThread = true; + mallctl("background_thread", NULL, NULL, &jeBackgroundThread, sizeof(bool)); +#endif if (!taosCheckSystemIsLittleEnd()) { printf("failed to start since on non-little-end machines\n"); return -1; @@ -340,7 +359,11 @@ int mainWindows(int argc, char **argv) { taosCleanupArgs(); if (dmInit() != 0) { - dError("failed to init dnode since %s", terrstr()); + if (terrno == TSDB_CODE_NOT_FOUND) { + dError("failed to init dnode since unsupported platform, please visit https://www.taosdata.com for support"); + } else { + dError("failed to init dnode since %s", terrstr()); + } taosCleanupCfg(); taosCloseLog(); @@ -350,6 +373,8 @@ int mainWindows(int argc, char **argv) { dInfo("start to init service"); dmSetSignalHandle(); + tsDndStart = taosGetTimestampMs(); + tsDndStartOsUptime = taosGetOsUptime(); int32_t code = dmRun(); dInfo("shutting down the service"); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 19982698968c8c91bc3f53edd3868f173b070005..1bce20ff44627c814c9ce677408e7ae3e12ed3ba 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -90,6 +90,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.clusterCfg.statusInterval = tsStatusInterval; req.clusterCfg.checkTime = 0; + req.clusterCfg.ttlChangeOnWrite = tsTtlChangeOnWrite; char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); @@ -265,6 +266,12 @@ int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) { pColInfo = taosArrayGet(pBlock->pDataBlock, c++); colDataSetVal(pColInfo, i, value, false); + char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0}; + cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen); + varDataSetLen(scope, valueLen); + pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + colDataSetVal(pColInfo, i, scope, false); + numOfRows++; } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 89c394fdd0889a30b737b43f534aec96e3fb3afa..76cb65b53a850b5b597c17806b501c88174380be 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -24,12 +24,16 @@ static void *dmStatusThreadFp(void *param) { const static int16_t TRIM_FREQ = 30; int32_t trimCount = 0; + int32_t upTimeCount = 0; + int64_t upTime = 0; + while (1) { taosMsleep(200); if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; int64_t curTime = taosGetTimestampMs(); - float interval = (curTime - lastTime) / 1000.0f; + if (curTime < lastTime) lastTime = curTime; + float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsStatusInterval) { dmSendStatusReq(pMgmt); lastTime = curTime; @@ -38,6 +42,11 @@ static void *dmStatusThreadFp(void *param) { if (trimCount == 0) { taosMemoryTrim(0); } + + if ((upTimeCount = ((upTimeCount + 1) & 63)) == 0) { + upTime = taosGetOsUptime() - tsDndStartOsUptime; + tsDndUpTime = TMAX(tsDndUpTime, upTime); + } } } @@ -54,7 +63,8 @@ static void *dmMonitorThreadFp(void *param) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; int64_t curTime = taosGetTimestampMs(); - float interval = (curTime - lastTime) / 1000.0f; + if (curTime < lastTime) lastTime = curTime; + float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsMonitorInterval) { (*pMgmt->sendMonitorReportFp)(); lastTime = curTime; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 7f8f6a48fa86c7c2ddd10c54fd58e7d4c8ce24c4..d975eb1cd13dd0487ee66635bb8dbe75837a380e 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -163,7 +163,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_DROP_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_SUBSCRIBE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_ASK_EP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_HB, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_HB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_DROP_CGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_DROP_CGROUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 0152e5d0b1a02041b2d70961a7bf85e7a516dd82..bbd77c0828d8a7901433afbc5ae351aa737f96c2 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -120,6 +120,11 @@ int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + if (NULL == pMgmt->pMnode) { + const STraceId *trace = &pMsg->info.traceId; + dGError("msg:%p, stop to pre-process in mnode since mnode is NULL, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + return -1; + } pMsg->info.node = pMgmt->pMnode; if (mndPreProcessQueryMsg(pMsg) != 0) { const STraceId *trace = &pMsg->info.traceId; diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index c098d546b693b88eee078dcdc6a67a6a606663d0..8206b4e4258a0eacb7eb91b5cde66a4bcb53f3d2 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -76,6 +76,10 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_SCAN_HISTORY_FINISH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_SCAN_HISTORY_FINISH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; code = 0; _OVER: diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index 83fb331dbd5fc368f39c8ab866426200c99ab6bc..5d08320fab973bec1c8a0bf3f93df9e78c0d4f50 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -46,6 +46,8 @@ typedef struct { int32_t vgId; int32_t vgVersion; int8_t dropped; + int32_t diskPrimary; + int32_t toVgId; char path[PATH_MAX + 20]; } SWrapperCfg; @@ -55,6 +57,8 @@ typedef struct { int32_t refCount; int8_t dropped; int8_t disable; + int32_t diskPrimary; + int32_t toVgId; char *path; SVnode *pImpl; SMultiWorker pWriteW; @@ -70,6 +74,7 @@ typedef struct { int32_t vnodeNum; int32_t opened; int32_t failed; + bool updateVnodesList; int32_t threadIndex; TdThread thread; SVnodeMgmt *pMgmt; @@ -78,6 +83,7 @@ typedef struct { } SVnodeThread; // vmInt.c +int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId); SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId); void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode); int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index bf176ebb40574b5806fe67f5a0b6fb2519886d05..da7f4d4a564379cea22dd01e934179aea8bf6851 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -71,6 +71,10 @@ static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg ** if (code < 0) goto _OVER; tjsonGetInt32ValueFromDouble(vnode, "vgVersion", pCfg->vgVersion, code); if (code < 0) goto _OVER; + tjsonGetInt32ValueFromDouble(vnode, "diskPrimary", pCfg->diskPrimary, code); + if (code < 0) goto _OVER; + tjsonGetInt32ValueFromDouble(vnode, "toVgId", pCfg->toVgId, code); + if (code < 0) goto _OVER; snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId); } @@ -165,6 +169,8 @@ static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t num if (tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId) < 0) return -1; if (tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped) < 0) return -1; if (tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion) < 0) return -1; + if (tjsonAddDoubleToObject(vnode, "diskPrimary", pVnode->diskPrimary) < 0) return -1; + if (pVnode->toVgId && tjsonAddDoubleToObject(vnode, "toVgId", pVnode->toVgId) < 0) return -1; if (tjsonAddItemToArray(vnodes, vnode) < 0) return -1; } @@ -179,7 +185,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { SVnodeObj **ppVnodes = NULL; char file[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0}; - snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%svnodes_tmp.json", pMgmt->path, TD_DIRSEP); snprintf(realfile, sizeof(realfile), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); int32_t numOfVnodes = 0; @@ -226,4 +232,4 @@ _OVER: dError("failed to write vnodes file:%s since %s, vnodes:%d", realfile, terrstr(), numOfVnodes); } return code; -} \ No newline at end of file +} diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 814a155cfba6d11d4dff61935a0cd8b0898d765c..bed9a6730326437a8b67c475d4da45c1b5588e71 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -263,30 +263,19 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } - snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId); + wrapperCfg.diskPrimary = vmAllocPrimaryDisk(pMgmt, vnodeCfg.vgId); + int32_t diskPrimary = wrapperCfg.diskPrimary; - if (pMgmt->pTfs) { - if (tfsDirExistAt(pMgmt->pTfs, path, (SDiskID){0})) { - terrno = TSDB_CODE_VND_DIR_ALREADY_EXIST; - dError("vgId:%d, failed to restore vnode since %s", req.vgId, terrstr()); - return -1; - } - } else { - if (taosDirExist(path)) { - terrno = TSDB_CODE_VND_DIR_ALREADY_EXIST; - dError("vgId:%d, failed to restore vnode since %s", req.vgId, terrstr()); - return -1; - } - } + snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId); -if (vnodeCreate(path, &vnodeCfg, pMgmt->pTfs) < 0) { + if (vnodeCreate(path, &vnodeCfg, diskPrimary, pMgmt->pTfs) < 0) { tFreeSCreateVnodeReq(&req); dError("vgId:%d, failed to create vnode since %s", req.vgId, terrstr()); code = terrno; goto _OVER; } - SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); + SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode since %s", req.vgId, terrstr()); code = terrno; @@ -417,21 +406,23 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { .dropped = pVnode->dropped, .vgId = pVnode->vgId, .vgVersion = pVnode->vgVersion, + .diskPrimary = pVnode->diskPrimary, }; tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path)); vmCloseVnode(pMgmt, pVnode, false); + int32_t diskPrimary = wrapperCfg.diskPrimary; char path[TSDB_FILENAME_LEN] = {0}; snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vgId); dInfo("vgId:%d, start to alter vnode replica at %s", vgId, path); - if (vnodeAlterReplica(path, &req, pMgmt->pTfs) < 0) { + if (vnodeAlterReplica(path, &req, diskPrimary, pMgmt->pTfs) < 0) { dError("vgId:%d, failed to alter vnode at %s since %s", vgId, path, terrstr()); return -1; } dInfo("vgId:%d, begin to open vnode", vgId); - SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); + SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr()); return -1; @@ -482,10 +473,18 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t srcVgId = req.srcVgId; int32_t dstVgId = req.dstVgId; + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, dstVgId); + if (pVnode != NULL) { + dError("vgId:%d, vnode already exist", dstVgId); + vmReleaseVnode(pMgmt, pVnode); + terrno = TSDB_CODE_VND_ALREADY_EXIST; + return -1; + } + dInfo("vgId:%d, start to alter vnode hashrange:[%u, %u], dstVgId:%d", req.srcVgId, req.hashBegin, req.hashEnd, req.dstVgId); - - SVnodeObj *pVnode = vmAcquireVnode(pMgmt, srcVgId); + pVnode = vmAcquireVnode(pMgmt, srcVgId); if (pVnode == NULL) { dError("vgId:%d, failed to alter hashrange since %s", srcVgId, terrstr()); terrno = TSDB_CODE_VND_NOT_EXIST; @@ -496,25 +495,34 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { .dropped = pVnode->dropped, .vgId = dstVgId, .vgVersion = pVnode->vgVersion, + .diskPrimary = pVnode->diskPrimary, }; tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path)); + // prepare alter + pVnode->toVgId = dstVgId; + if (vmWriteVnodeListToFile(pMgmt) != 0) { + dError("vgId:%d, failed to write vnode list since %s", dstVgId, terrstr()); + return -1; + } + dInfo("vgId:%d, close vnode", srcVgId); vmCloseVnode(pMgmt, pVnode, true); + int32_t diskPrimary = wrapperCfg.diskPrimary; char srcPath[TSDB_FILENAME_LEN] = {0}; char dstPath[TSDB_FILENAME_LEN] = {0}; snprintf(srcPath, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, srcVgId); snprintf(dstPath, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, dstVgId); dInfo("vgId:%d, alter vnode hashrange at %s", srcVgId, srcPath); - if (vnodeAlterHashRange(srcPath, dstPath, &req, pMgmt->pTfs) < 0) { + if (vnodeAlterHashRange(srcPath, dstPath, &req, diskPrimary, pMgmt->pTfs) < 0) { dError("vgId:%d, failed to alter vnode hashrange since %s", srcVgId, terrstr()); return -1; } dInfo("vgId:%d, open vnode", dstVgId); - SVnode *pImpl = vnodeOpen(dstPath, pMgmt->pTfs, pMgmt->msgCb); + SVnode *pImpl = vnodeOpen(dstPath, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode at %s since %s", dstVgId, dstPath, terrstr()); return -1; @@ -530,6 +538,7 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } + // complete alter if (vmWriteVnodeListToFile(pMgmt) != 0) { dError("vgId:%d, failed to write vnode list since %s", dstVgId, terrstr()); return -1; @@ -600,21 +609,23 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { .dropped = pVnode->dropped, .vgId = pVnode->vgId, .vgVersion = pVnode->vgVersion, + .diskPrimary = pVnode->diskPrimary, }; tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path)); vmCloseVnode(pMgmt, pVnode, false); + int32_t diskPrimary = wrapperCfg.diskPrimary; char path[TSDB_FILENAME_LEN] = {0}; snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vgId); dInfo("vgId:%d, start to alter vnode replica at %s", vgId, path); - if (vnodeAlterReplica(path, &alterReq, pMgmt->pTfs) < 0) { + if (vnodeAlterReplica(path, &alterReq, diskPrimary, pMgmt->pTfs) < 0) { dError("vgId:%d, failed to alter vnode at %s since %s", vgId, path, terrstr()); return -1; } dInfo("vgId:%d, begin to open vnode", vgId); - SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); + SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr()); return -1; @@ -708,11 +719,13 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DELETE_SUB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SEEK_TO_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SEEK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_ADD_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DEL_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_CONSUME, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_CONSUME_PUSH, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_VG_WALINFO, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_VG_COMMITTEDINFO, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_BATCH_DEL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -727,9 +740,11 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_RECOVER_FINISH, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_SCAN_HISTORY_FINISH, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_SCAN_HISTORY_FINISH_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TRANSFER_STATE, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK_RSP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 16e7ffc5367c857dad1414c788dc46d9f6263b42..0ff2537e4c192530ac283a429edf7db34b72efc0 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -15,8 +15,64 @@ #define _DEFAULT_SOURCE #include "vmInt.h" +#include "tfs.h" #include "vnd.h" +int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { + STfs *pTfs = pMgmt->pTfs; + int32_t diskId = 0; + if (!pTfs) { + return diskId; + } + + // search fs + char vnodePath[TSDB_FILENAME_LEN] = {0}; + snprintf(vnodePath, TSDB_FILENAME_LEN - 1, "vnode%svnode%d", TD_DIRSEP, vgId); + char fname[TSDB_FILENAME_LEN] = {0}; + char fnameTmp[TSDB_FILENAME_LEN] = {0}; + snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s", vnodePath, TD_DIRSEP, VND_INFO_FNAME); + snprintf(fnameTmp, TSDB_FILENAME_LEN - 1, "%s%s%s", vnodePath, TD_DIRSEP, VND_INFO_FNAME_TMP); + + diskId = tfsSearch(pTfs, 0, fname); + if (diskId >= 0) { + return diskId; + } + diskId = tfsSearch(pTfs, 0, fnameTmp); + if (diskId >= 0) { + return diskId; + } + + // alloc + int32_t disks[TFS_MAX_DISKS_PER_TIER] = {0}; + int32_t numOfVnodes = 0; + SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); + for (int32_t v = 0; v < numOfVnodes; v++) { + SVnodeObj *pVnode = ppVnodes[v]; + disks[pVnode->diskPrimary] += 1; + } + + int32_t minVal = INT_MAX; + int32_t ndisk = tfsGetDisksAtLevel(pTfs, 0); + diskId = 0; + for (int32_t id = 0; id < ndisk; id++) { + if (minVal > disks[id]) { + minVal = disks[id]; + diskId = id; + } + } + + for (int32_t i = 0; i < numOfVnodes; ++i) { + if (ppVnodes == NULL || ppVnodes[i] == NULL) continue; + vmReleaseVnode(pMgmt, ppVnodes[i]); + } + if (ppVnodes != NULL) { + taosMemoryFree(ppVnodes); + } + + dInfo("vgId:%d, alloc disk:%d of level 0. ndisk:%d, vnodes: %d", vgId, diskId, ndisk, numOfVnodes); + return diskId; +} + SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { SVnodeObj *pVnode = NULL; @@ -52,6 +108,7 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { pVnode->vgId = pCfg->vgId; pVnode->vgVersion = pCfg->vgVersion; + pVnode->diskPrimary = pCfg->diskPrimary; pVnode->refCount = 0; pVnode->dropped = 0; pVnode->path = taosStrdup(pCfg->path); @@ -158,6 +215,29 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) taosMemoryFree(pVnode); } +static int32_t vmRestoreVgroupId(SWrapperCfg *pCfg, STfs *pTfs) { + int32_t srcVgId = pCfg->vgId; + int32_t dstVgId = pCfg->toVgId; + if (dstVgId == 0) return 0; + + char srcPath[TSDB_FILENAME_LEN]; + char dstPath[TSDB_FILENAME_LEN]; + + snprintf(srcPath, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, srcVgId); + snprintf(dstPath, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, dstVgId); + + int32_t diskPrimary = pCfg->diskPrimary; + int32_t vgId = vnodeRestoreVgroupId(srcPath, dstPath, srcVgId, dstVgId, diskPrimary, pTfs); + if (vgId <= 0) { + dError("vgId:%d, failed to restore vgroup id. srcPath: %s", pCfg->vgId, srcPath); + return -1; + } + + pCfg->vgId = vgId; + pCfg->toVgId = 0; + return 0; +} + static void *vmOpenVnodeInThread(void *param) { SVnodeThread *pThread = param; SVnodeMgmt *pMgmt = pThread->pMgmt; @@ -174,17 +254,34 @@ static void *vmOpenVnodeInThread(void *param) { pMgmt->state.openVnodes, pMgmt->state.totalVnodes); tmsgReportStartup("vnode-open", stepDesc); + if (pCfg->toVgId) { + if (vmRestoreVgroupId(pCfg, pMgmt->pTfs) != 0) { + dError("vgId:%d, failed to restore vgroup id by thread:%d", pCfg->vgId, pThread->threadIndex); + pThread->failed++; + continue; + } + pThread->updateVnodesList = true; + } + + int32_t diskPrimary = pCfg->diskPrimary; snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId); - SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); + + SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { + dError("vgId:%d, failed to open vnode by thread:%d since %s", pCfg->vgId, pThread->threadIndex, terrstr()); + pThread->failed++; + continue; + } + + if (vmOpenVnode(pMgmt, pCfg, pImpl) != 0) { dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->failed++; - } else { - vmOpenVnode(pMgmt, pCfg, pImpl); - dInfo("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); - pThread->opened++; - atomic_add_fetch_32(&pMgmt->state.openVnodes, 1); + continue; } + + dInfo("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); + pThread->opened++; + atomic_add_fetch_32(&pMgmt->state.openVnodes, 1); } dInfo("thread:%d, numOfVnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, @@ -242,6 +339,8 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { taosThreadAttrDestroy(&thAttr); } + bool updateVnodesList = false; + for (int32_t t = 0; t < threadNum; ++t) { SVnodeThread *pThread = &threads[t]; if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { @@ -249,17 +348,24 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { taosThreadClear(&pThread->thread); } taosMemoryFree(pThread->pCfgs); + if (pThread->updateVnodesList) updateVnodesList = true; } taosMemoryFree(threads); taosMemoryFree(pCfgs); if (pMgmt->state.openVnodes != pMgmt->state.totalVnodes) { dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes); + terrno = TSDB_CODE_VND_INIT_FAILED; + return -1; + } + + if (updateVnodesList && vmWriteVnodeListToFile(pMgmt) != 0) { + dError("failed to write vnode list since %s", terrstr()); return -1; - } else { - dInfo("successfully opened %d vnodes", pMgmt->state.totalVnodes); - return 0; } + + dInfo("successfully opened %d vnodes", pMgmt->state.totalVnodes); + return 0; } static void *vmCloseVnodeInThread(void *param) { @@ -354,7 +460,6 @@ static void vmCleanup(SVnodeMgmt *pMgmt) { vmCloseVnodes(pMgmt); vmStopWorker(pMgmt); vnodeCleanup(); - tfsClose(pMgmt->pTfs); taosThreadRwlockDestroy(&pMgmt->lock); taosMemoryFree(pMgmt); } @@ -429,20 +534,9 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->msgCb.mgmt = pMgmt; taosThreadRwlockInit(&pMgmt->lock, NULL); - SDiskCfg dCfg = {0}; - tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN); - dCfg.level = 0; - dCfg.primary = 1; - SDiskCfg *pDisks = tsDiskCfg; - int32_t numOfDisks = tsDiskCfgNum; - if (numOfDisks <= 0 || pDisks == NULL) { - pDisks = &dCfg; - numOfDisks = 1; - } - - pMgmt->pTfs = tfsOpen(pDisks, numOfDisks); + pMgmt->pTfs = pInput->pTfs; if (pMgmt->pTfs == NULL) { - dError("failed to init tfs since %s", terrstr()); + dError("tfs is null."); goto _OVER; } tmsgReportStartup("vnode-tfs", "initialized"); @@ -472,7 +566,7 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { tmsgReportStartup("vnode-worker", "initialized"); if (vmOpenVnodes(pMgmt) != 0) { - dError("failed to open vnode since %s", terrstr()); + dError("failed to open all vnodes since %s", terrstr()); goto _OVER; } tmsgReportStartup("vnode-vnodes", "initialized"); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index a5f53c87030b49bc321dd53f0fc91bf7ded0a977..cae2a7d2be581abfcbcfb160104cf55d78ee827e 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" +#include "vnodeInt.h" static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) { if (pMsg->info.handle == NULL) return; @@ -78,7 +79,7 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, pMsg); if (code != 0) { if (terrno != 0) code = terrno; - dGError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, terrstr(code)); + dGError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, tstrerror(code)); vmSendRsp(pMsg, code); } @@ -92,7 +93,7 @@ static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { const STraceId *trace = &pMsg->info.traceId; dGTrace("vgId:%d, msg:%p get from vnode-stream queue", pVnode->vgId, pMsg); - int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); + int32_t code = vnodeProcessStreamMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType), @@ -114,9 +115,10 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO const STraceId *trace = &pMsg->info.traceId; dGTrace("vgId:%d, msg:%p get from vnode-fetch queue", pVnode->vgId, pMsg); + terrno = 0; int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { - if (terrno != 0) { + if (code == -1 && terrno != 0) { code = terrno; } @@ -158,6 +160,15 @@ static void vmSendResponse(SRpcMsg *pMsg) { } } +static bool vmDataSpaceSufficient(SVnodeObj *pVnode) { + STfs *pTfs = pVnode->pImpl->pTfs; + if (pTfs) { + return tfsDiskSpaceSufficient(pTfs, 0, pVnode->diskPrimary); + } else { + return osDataSpaceSufficient(); + } +} + static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) { const STraceId *trace = &pMsg->info.traceId; if (pMsg->contLen < sizeof(SMsgHead)) { @@ -203,7 +214,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - if (!osDataSpaceSufficient()) { + if (!vmDataSpaceSufficient(pVnode)) { terrno = TSDB_CODE_NO_ENOUGH_DISKSPACE; code = terrno; dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code)); @@ -218,6 +229,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp if (pMsg->msgType != TDMT_VND_ALTER_CONFIRM && pVnode->disable) { dDebug("vgId:%d, msg:%p put into vnode-write queue failed since its disable", pVnode->vgId, pMsg); terrno = TSDB_CODE_VND_STOPPED; + code = terrno; break; } dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index 02cd6784331b0f7c9f49238f537d70bf36bf0a12..98489433b948b68a4dd6cdae798f8b6aec2b1fda 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -20,6 +20,7 @@ #include "uv.h" #include "dmInt.h" +#include "tfs.h" #ifdef __cplusplus extern "C" { @@ -79,6 +80,7 @@ typedef struct SDnode { TdThreadMutex mutex; TdFilePtr lockfile; SDnodeData data; + STfs *pTfs; SMgmtWrapper wrappers[NODE_END]; } SDnode; @@ -124,4 +126,4 @@ void dmGetQnodeLoads(SQnodeLoad *pInfo); } #endif -#endif /*_TD_DND_MGMT_H_*/ \ No newline at end of file +#endif /*_TD_DND_MGMT_H_*/ diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 56bff0c760b042c8dd1bf389ff93ce371df6a9df..a34002161d820e2f7df4551b434df568be422f8a 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -16,7 +16,33 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" -static SDnode globalDnode = {0}; +#define STR_CASE_CMP(s, d) (0 == strcasecmp((s), (d))) +#define STR_STR_CMP(s, d) (strstr((s), (d))) +#define STR_INT_CMP(s, d, c) (taosStr2Int32(s, 0, 10) c(d)) +#define STR_STR_SIGN ("ia") +#define DM_INIT_MON() \ + do { \ + code = (int32_t)(2147483648 | 298); \ + strncpy(stName, tsVersionName, 64); \ + monCfg.maxLogs = tsMonitorMaxLogs; \ + monCfg.port = tsMonitorPort; \ + monCfg.server = tsMonitorFqdn; \ + monCfg.comp = tsMonitorComp; \ + if (monInit(&monCfg) != 0) { \ + if (terrno != 0) code = terrno; \ + goto _exit; \ + } \ + } while (0) + +#define DM_ERR_RTN(c) \ + do { \ + code = (c); \ + goto _exit; \ + } while (0) + +static SDnode globalDnode = {0}; +static const char *dmOS[10] = {"Ubuntu", "CentOS Linux", "Red Hat", "Debian GNU", "CoreOS", + "FreeBSD", "openSUSE", "SLES", "Fedora", "macOS"}; SDnode *dmInstance() { return &globalDnode; } @@ -37,40 +63,56 @@ static int32_t dmInitSystem() { } static int32_t dmInitMonitor() { + int32_t code = 0; SMonCfg monCfg = {0}; - monCfg.maxLogs = tsMonitorMaxLogs; - monCfg.port = tsMonitorPort; - monCfg.server = tsMonitorFqdn; - monCfg.comp = tsMonitorComp; - if (monInit(&monCfg) != 0) { - dError("failed to init monitor since %s", terrstr()); - return -1; + char reName[64] = {0}; + char stName[64] = {0}; + char ver[64] = {0}; + + DM_INIT_MON(); + + if (STR_STR_CMP(stName, STR_STR_SIGN)) { + DM_ERR_RTN(0); } - return 0; + if (taosGetOsReleaseName(reName, stName, ver, 64) != 0) { + DM_ERR_RTN(code); + } + if (STR_CASE_CMP(stName, dmOS[0])) { + if (STR_INT_CMP(ver, 17, >)) { + DM_ERR_RTN(0); + } + } else if (STR_CASE_CMP(stName, dmOS[1])) { + if (STR_INT_CMP(ver, 6, >)) { + DM_ERR_RTN(0); + } + } else if (STR_STR_CMP(stName, dmOS[2]) || STR_STR_CMP(stName, dmOS[3]) || STR_STR_CMP(stName, dmOS[4]) || + STR_STR_CMP(stName, dmOS[5]) || STR_STR_CMP(stName, dmOS[6]) || STR_STR_CMP(stName, dmOS[7]) || + STR_STR_CMP(stName, dmOS[8]) || STR_STR_CMP(stName, dmOS[9])) { + DM_ERR_RTN(0); + } + +_exit: + if (code) terrno = code; + return code; } -static bool dmCheckDiskSpace() { - osUpdate(); - // sufficiency - if (!osDataSpaceSufficient()) { - dWarn("free data disk size: %f GB, not sufficient, expected %f GB at least", - (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); - } - if (!osLogSpaceSufficient()) { - dWarn("free log disk size: %f GB, not sufficient, expected %f GB at least", - (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0); +static bool dmDataSpaceAvailable() { + SDnode *pDnode = dmInstance(); + if (pDnode->pTfs) { + return tfsDiskSpaceAvailable(pDnode->pTfs, 0); } - if (!osTempSpaceSufficient()) { - dWarn("free temp disk size: %f GB, not sufficient, expected %f GB at least", - (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); + if (!osDataSpaceAvailable()) { + dError("data disk space unavailable, i.e. %s", tsDataDir); + return false; } + return true; +} + +static bool dmCheckDiskSpace() { + osUpdate(); // availability bool ret = true; - if (!osDataSpaceAvailable()) { - dError("data disk space unavailable, i.e. %s", tsDataDir); + if (!dmDataSpaceAvailable()) { terrno = TSDB_CODE_NO_DISKSPACE; ret = false; } @@ -87,6 +129,34 @@ static bool dmCheckDiskSpace() { return ret; } +int32_t dmDiskInit() { + SDnode *pDnode = dmInstance(); + SDiskCfg dCfg = {0}; + tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN); + dCfg.level = 0; + dCfg.primary = 1; + SDiskCfg *pDisks = tsDiskCfg; + int32_t numOfDisks = tsDiskCfgNum; + if (numOfDisks <= 0 || pDisks == NULL) { + pDisks = &dCfg; + numOfDisks = 1; + } + + pDnode->pTfs = tfsOpen(pDisks, numOfDisks); + if (pDnode->pTfs == NULL) { + dError("failed to init tfs since %s", terrstr()); + return -1; + } + return 0; +} + +int32_t dmDiskClose() { + SDnode *pDnode = dmInstance(); + tfsClose(pDnode->pTfs); + pDnode->pTfs = NULL; + return 0; +} + static bool dmCheckDataDirVersion() { char checkDataDirJsonFileName[PATH_MAX] = {0}; snprintf(checkDataDirJsonFileName, PATH_MAX, "%s/dnode/dnodeCfg.json", tsDataDir); @@ -100,6 +170,7 @@ static bool dmCheckDataDirVersion() { int32_t dmInit() { dInfo("start to init dnode env"); + if (dmDiskInit() != 0) return -1; if (!dmCheckDataDirVersion()) return -1; if (!dmCheckDiskSpace()) return -1; if (dmCheckRepeatInit(dmInstance()) != 0) return -1; @@ -130,6 +201,7 @@ void dmCleanup() { udfcClose(); udfStopUdfd(); taosStopCacheRefreshWorker(); + dmDiskClose(); dInfo("dnode env is cleaned up"); taosCleanupCfg(); @@ -320,6 +392,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) { SMgmtInputOpt opt = { .path = pWrapper->path, .name = pWrapper->name, + .pTfs = pWrapper->pDnode->pTfs, .pData = &pWrapper->pDnode->data, .processCreateNodeFp = dmProcessCreateNodeReq, .processAlterNodeTypeFp = dmProcessAlterNodeTypeReq, diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 19d5e06c5b6d118feaa8bf6d50fe222e7a73007f..a8bf5be3e21136ddb290adbd6215e35e5a7f2d3b 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -41,7 +41,7 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) { pWrapper->pMgmt = output.pMgmt; } - dmReportStartup(pWrapper->name, "openned"); + dmReportStartup(pWrapper->name, "opened"); return 0; } @@ -159,7 +159,7 @@ int32_t dmRunDnode(SDnode *pDnode) { } else { count++; } - + taosMsleep(100); } } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index ea46b7069385fba0c204c5e78f9f7e7983053d85..e0f7da3ac49fb74838ddc75c2e5403f807bf8b63 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" #include "qworker.h" +#include "tversion.h" static inline void dmSendRsp(SRpcMsg *pMsg) { rpcSendResponse(pMsg); } @@ -73,6 +74,13 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { dGTrace("msg:%s is received, handle:%p len:%d code:0x%x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType), pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId); + int32_t svrVer = 0; + taosVersionStrToInt(version, &svrVer); + if (0 != taosCheckVersionCompatible(pRpc->info.cliVer, svrVer, 3)) { + dError("Version not compatible, cli ver: %d, svr ver: %d", pRpc->info.cliVer, svrVer); + goto _OVER; + } + switch (pRpc->msgType) { case TDMT_DND_NET_TEST: dmProcessNetTestReq(pDnode, pRpc); @@ -282,6 +290,7 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = TSDB_DEFAULT_USER; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.parent = pDnode; rpcInit.rfp = rpcRfp; @@ -290,7 +299,7 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.retryMinInterval = tsRedirectPeriod; rpcInit.retryStepFactor = tsRedirectFactor; rpcInit.retryMaxInterval = tsRedirectMaxPeriod; - rpcInit.retryMaxTimouet = tsMaxRetryWaitTime; + rpcInit.retryMaxTimeout = tsMaxRetryWaitTime; rpcInit.failFastInterval = 5000; // interval threshold(ms) rpcInit.failFastThreshold = 3; // failed threshold @@ -305,6 +314,7 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.supportBatch = 1; rpcInit.batchSize = 8 * 1024; rpcInit.timeToGetConn = tsTimeToGetAvailableConn; + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); pTrans->clientRpc = rpcOpen(&rpcInit); if (pTrans->clientRpc == NULL) { @@ -339,7 +349,7 @@ int32_t dmInitServer(SDnode *pDnode) { rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.parent = pDnode; rpcInit.compressSize = tsCompressMsgSize; - + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); pTrans->serverRpc = rpcOpen(&rpcInit); if (pTrans->serverRpc == NULL) { dError("failed to init dnode rpc server"); diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 98ef8cd95ba9c201ec23ff9fd8515948016eec4f..32c3d225062c5c9630d370064fe653c3e799a839 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -37,6 +37,7 @@ #include "monitor.h" #include "qnode.h" #include "sync.h" +#include "tfs.h" #include "wal.h" #include "libs/function/tudf.h" @@ -105,11 +106,13 @@ typedef struct { SHashObj *dnodeHash; TdThreadRwlock lock; SMsgCb msgCb; + bool validMnodeEps; } SDnodeData; typedef struct { const char *path; const char *name; + STfs *pTfs; SDnodeData *pData; SMsgCb msgCb; ProcessCreateNodeFp processCreateNodeFp; diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 45cc4bb7112fec0c06899a2426b774c5984af268..1564a090357717b4733a5c557fd2c1c248f753ba 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -288,6 +288,8 @@ static void dmResetEps(SDnodeData *pData, SArray *dnodeEps) { taosHashPut(pData->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp)); } + pData->validMnodeEps = true; + dmPrintEps(pData); } @@ -348,6 +350,7 @@ void dmRotateMnodeEpSet(SDnodeData *pData) { } void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet) { + if(!pData->validMnodeEps) return; dmGetMnodeEpSet(pData, pEpSet); dTrace("msg is redirected, handle:%p num:%d use:%d", pMsg->info.handle, pEpSet->numOfEps, pEpSet->inUse); for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { diff --git a/source/dnode/mgmt/test/sut/src/client.cpp b/source/dnode/mgmt/test/sut/src/client.cpp index a27a511651795fc864a24bef6db534793e74649d..95eea2359d081e8b98db190fe45c24c2d72954bd 100644 --- a/source/dnode/mgmt/test/sut/src/client.cpp +++ b/source/dnode/mgmt/test/sut/src/client.cpp @@ -16,6 +16,7 @@ #include "sut.h" #include "tdatablock.h" #include "tmisce.h" +#include "tversion.h" static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { TestClient* client = (TestClient*)parent; @@ -53,6 +54,7 @@ void TestClient::DoInit() { rpcInit.parent = this; // rpcInit.secret = (char*)secretEncrypt; // rpcInit.spi = 1; + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); clientRpc = rpcOpen(&rpcInit); ASSERT(clientRpc); diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 2cb41edd7c1d37c8dab6f0e276259e9cc530fea8..e33ffdb372d0c317f5478add89b587bbe91562a9 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -27,7 +27,7 @@ void mndCleanupCluster(SMnode *pMnode); int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len); int64_t mndGetClusterId(SMnode *pMnode); int64_t mndGetClusterCreateTime(SMnode *pMnode); -float mndGetClusterUpTime(SMnode *pMnode); +int64_t mndGetClusterUpTime(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index 96401511d2cd4832ad6d548a4b7286ba62227a7d..a3a31cfc5a5cfd0fdde3830ab015d2ca8cd72c98 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -25,14 +25,15 @@ extern "C" { enum { MQ_CONSUMER_STATUS_REBALANCE = 1, // MQ_CONSUMER_STATUS__MODIFY_IN_REB, // this value is not used anymore - MQ_CONSUMER_STATUS__READY, - MQ_CONSUMER_STATUS__LOST, + MQ_CONSUMER_STATUS_READY, + MQ_CONSUMER_STATUS_LOST, // MQ_CONSUMER_STATUS__LOST_IN_REB, // this value is not used anymore - MQ_CONSUMER_STATUS__LOST_REBD, -}; +// MQ_CONSUMER_STATUS__LOST_REBD, +};\ int32_t mndInitConsumer(SMnode *pMnode); void mndCleanupConsumer(SMnode *pMnode); +void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId); SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId); void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer); diff --git a/source/dnode/mnode/impl/inc/mndDb.h b/source/dnode/mnode/impl/inc/mndDb.h index 97d047d7a3db51b0893ad2fd4a45c76cf1ed8424..5569a6ec9fb8ccc003f171024f045c56cb3aefc3 100644 --- a/source/dnode/mnode/impl/inc/mndDb.h +++ b/source/dnode/mnode/impl/inc/mndDb.h @@ -26,12 +26,13 @@ int32_t mndInitDb(SMnode *pMnode); void mndCleanupDb(SMnode *pMnode); SDbObj *mndAcquireDb(SMnode *pMnode, const char *db); void mndReleaseDb(SMnode *pMnode, SDbObj *pDb); -int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen); +int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen); int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUseDbReq *pReq); bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb); SSdbRaw *mndDbActionEncode(SDbObj *pDb); const char *mndGetDbStr(const char *src); +const char *mndGetStableStr(const char *src); int32_t mndProcessCompactDbReq(SRpcMsg *pReq); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index b14f7a9023fae8bd7a0fa1458a177064b0fb619c..c83a40c25dc9079f2e07dba2e1c8ca6796228cdb 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -108,7 +108,7 @@ typedef enum { TRN_STAGE_UNDO_ACTION = 3, TRN_STAGE_COMMIT = 4, TRN_STAGE_COMMIT_ACTION = 5, - TRN_STAGE_FINISHED = 6, + TRN_STAGE_FINISH = 6, TRN_STAGE_PRE_FINISH = 7 } ETrnStage; @@ -133,16 +133,16 @@ typedef enum { DND_REASON_TIME_ZONE_NOT_MATCH, DND_REASON_LOCALE_NOT_MATCH, DND_REASON_CHARSET_NOT_MATCH, + DND_REASON_TTL_CHANGE_ON_WRITE_NOT_MATCH, DND_REASON_OTHERS } EDndReason; typedef enum { - CONSUMER_UPDATE__TOUCH = 1, // rebalance req do not need change consume topic - CONSUMER_UPDATE__ADD, - CONSUMER_UPDATE__REMOVE, - CONSUMER_UPDATE__LOST, - CONSUMER_UPDATE__RECOVER, - CONSUMER_UPDATE__REBALANCE, // subscribe req need change consume topic + CONSUMER_UPDATE_REB = 1, // update after rebalance + CONSUMER_ADD_REB, // add after rebalance + CONSUMER_REMOVE_REB, // remove after rebalance + CONSUMER_UPDATE_REC, // update after recover + CONSUMER_UPDATE_SUB, // update after subscribe req } ECsmUpdateType; typedef struct { @@ -157,6 +157,7 @@ typedef struct { void* rpcRsp; int32_t rpcRspLen; int32_t redoActionPos; + SArray* prepareActions; SArray* redoActions; SArray* undoActions; SArray* commitActions; @@ -215,8 +216,9 @@ typedef struct { int64_t createdTime; int64_t updateTime; ESyncState syncState; + SyncTerm syncTerm; bool syncRestore; - int64_t stateStartTime; + int64_t roleTimeMs; SDnodeObj* pDnode; int32_t role; SyncIndex lastIndex; @@ -467,6 +469,7 @@ typedef struct { int8_t replica; int16_t numOfColumns; int32_t numOfRows; + int32_t curIterPackedRows; void* pIter; SMnode* pMnode; STableMetaRsp* pMeta; @@ -521,6 +524,7 @@ typedef struct { char* physicalPlan; SSchemaWrapper schema; int64_t stbUid; + char stbName[TSDB_TABLE_FNAME_LEN]; // forbid condition int64_t ntbUid; SArray* ntbColIds; @@ -546,36 +550,42 @@ typedef struct { // data for display int32_t pid; SEpSet ep; - int64_t upTime; + int64_t createTime; int64_t subscribeTime; int64_t rebalanceTime; + + int8_t withTbName; + int8_t autoCommit; + int32_t autoCommitInterval; + int32_t resetOffsetCfg; } SMqConsumerObj; SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]); -void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer); +void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer, bool delete); int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer); -void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer); +void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer, int8_t sver); typedef struct { int32_t vgId; - char* qmsg; // SubPlanToString +// char* qmsg; // SubPlanToString SEpSet epSet; } SMqVgEp; SMqVgEp* tCloneSMqVgEp(const SMqVgEp* pVgEp); void tDeleteSMqVgEp(SMqVgEp* pVgEp); int32_t tEncodeSMqVgEp(void** buf, const SMqVgEp* pVgEp); -void* tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp); +void* tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp, int8_t sver); typedef struct { int64_t consumerId; // -1 for unassigned SArray* vgs; // SArray + SArray* offsetRows; // SArray } SMqConsumerEp; -SMqConsumerEp* tCloneSMqConsumerEp(const SMqConsumerEp* pEp); -void tDeleteSMqConsumerEp(void* pEp); +//SMqConsumerEp* tCloneSMqConsumerEp(const SMqConsumerEp* pEp); +//void tDeleteSMqConsumerEp(void* pEp); int32_t tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp* pEp); -void* tDecodeSMqConsumerEp(const void* buf, SMqConsumerEp* pEp); +void* tDecodeSMqConsumerEp(const void* buf, SMqConsumerEp* pEp, int8_t sver); typedef struct { char key[TSDB_SUBSCRIBE_KEY_LEN]; @@ -587,34 +597,36 @@ typedef struct { int64_t stbUid; SHashObj* consumerHash; // consumerId -> SMqConsumerEp SArray* unassignedVgs; // SArray + SArray* offsetRows; char dbName[TSDB_DB_FNAME_LEN]; + char* qmsg; // SubPlanToString } SMqSubscribeObj; SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]); SMqSubscribeObj* tCloneSubscribeObj(const SMqSubscribeObj* pSub); void tDeleteSubscribeObj(SMqSubscribeObj* pSub); int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub); -void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub); - -typedef struct { - int32_t epoch; - SArray* consumers; // SArray -} SMqSubActionLogEntry; - -SMqSubActionLogEntry* tCloneSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry); -void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry); -int32_t tEncodeSMqSubActionLogEntry(void** buf, const SMqSubActionLogEntry* pEntry); -void* tDecodeSMqSubActionLogEntry(const void* buf, SMqSubActionLogEntry* pEntry); - -typedef struct { - char key[TSDB_SUBSCRIBE_KEY_LEN]; - SArray* logs; // SArray -} SMqSubActionLogObj; - -SMqSubActionLogObj* tCloneSMqSubActionLogObj(SMqSubActionLogObj* pLog); -void tDeleteSMqSubActionLogObj(SMqSubActionLogObj* pLog); -int32_t tEncodeSMqSubActionLogObj(void** buf, const SMqSubActionLogObj* pLog); -void* tDecodeSMqSubActionLogObj(const void* buf, SMqSubActionLogObj* pLog); +void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub, int8_t sver); + +//typedef struct { +// int32_t epoch; +// SArray* consumers; // SArray +//} SMqSubActionLogEntry; + +//SMqSubActionLogEntry* tCloneSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry); +//void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry); +//int32_t tEncodeSMqSubActionLogEntry(void** buf, const SMqSubActionLogEntry* pEntry); +//void* tDecodeSMqSubActionLogEntry(const void* buf, SMqSubActionLogEntry* pEntry); +// +//typedef struct { +// char key[TSDB_SUBSCRIBE_KEY_LEN]; +// SArray* logs; // SArray +//} SMqSubActionLogObj; +// +//SMqSubActionLogObj* tCloneSMqSubActionLogObj(SMqSubActionLogObj* pLog); +//void tDeleteSMqSubActionLogObj(SMqSubActionLogObj* pLog); +//int32_t tEncodeSMqSubActionLogObj(void** buf, const SMqSubActionLogObj* pLog); +//void* tDecodeSMqSubActionLogObj(const void* buf, SMqSubActionLogObj* pLog); typedef struct { int32_t oldConsumerNum; @@ -633,9 +645,17 @@ typedef struct { SArray* removedConsumers; // SArray SArray* modifyConsumers; // SArray SMqSubscribeObj* pSub; - SMqSubActionLogEntry* pLogEntry; +// SMqSubActionLogEntry* pLogEntry; } SMqRebOutputObj; +typedef struct SStreamConf { + int8_t igExpired; + int8_t trigger; + int8_t fillHistory; + int64_t triggerParam; + int64_t watermark; +} SStreamConf; + typedef struct { char name[TSDB_STREAM_FNAME_LEN]; // ctl @@ -649,12 +669,7 @@ typedef struct { // info int64_t uid; int8_t status; - // config - int8_t igExpired; - int8_t trigger; - int8_t fillHistory; - int64_t triggerParam; - int64_t watermark; + SStreamConf conf; // source and target int64_t sourceDbUid; int64_t targetDbUid; @@ -662,15 +677,20 @@ typedef struct { char targetDb[TSDB_DB_FNAME_LEN]; char targetSTbName[TSDB_TABLE_FNAME_LEN]; int64_t targetStbUid; - int32_t fixedSinkVgId; // 0 for shuffle + // fixedSinkVg is not applicable for encode and decode - SVgObj fixedSinkVg; + SVgObj fixedSinkVg; + int32_t fixedSinkVgId; // 0 for shuffle // transformation char* sql; char* ast; char* physicalPlan; - SArray* tasks; // SArray> + SArray* tasks; // SArray> + + SArray* pHTasksList; // generate the results for already stored ts data + int64_t hTaskUid; // stream task for history ts data + SSchemaWrapper outputSchema; SSchemaWrapper tagSchema; @@ -685,12 +705,12 @@ int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj); int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj, int32_t sver); void tFreeStreamObj(SStreamObj* pObj); -typedef struct { - char streamName[TSDB_STREAM_FNAME_LEN]; - int64_t uid; - int64_t streamUid; - SArray* childInfo; // SArray -} SStreamCheckpointObj; +//typedef struct { +// char streamName[TSDB_STREAM_FNAME_LEN]; +// int64_t uid; +// int64_t streamUid; +// SArray* childInfo; // SArray +//} SStreamCheckpointObj; #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndScheduler.h b/source/dnode/mnode/impl/inc/mndScheduler.h index 23085c53eed7ef6234da38f146f962ea96d9fdde..cba52c6b45a14cc6c5fa434279b2c3f1ccecc4ff 100644 --- a/source/dnode/mnode/impl/inc/mndScheduler.h +++ b/source/dnode/mnode/impl/inc/mndScheduler.h @@ -22,15 +22,12 @@ extern "C" { #endif -int32_t mndInitScheduler(SMnode* pMnode); -void mndCleanupScheduler(SMnode* pMnode); int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub); - int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, int64_t watermark, int64_t deleteMark); -int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream); +int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t nextWindowSkey); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 66d0ed1d12160b5ea6e9c30f145e0db6faa22b31..db960d790f708e76ce2921f4ad6b6fefba82e441 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -39,6 +39,7 @@ int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char *dbFName, char *stbFName, vo void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst); void mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst); +void mndExtractShortDbNameFromDbFullName(const char *stbFullName, char *dst); void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize); const char *mndGetStbStr(const char *src); @@ -47,6 +48,7 @@ int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew); int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char *stbFullName, int64_t suid, col_id_t colId); void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen, void *alterOriData, int32_t alterOriDataLen); +int32_t mndSetForceDropCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, SStbObj *pStb); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index d873df621e8b44dca5adc9b4c4affaddc78730c2..05adc17d64fed3a87e48ac3ec4e4eaaff869a7f8 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -38,7 +38,7 @@ int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); -int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); +int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndSubscribe.h b/source/dnode/mnode/impl/inc/mndSubscribe.h index fad316ea12edce96bde4c21694b5402d97bf4ae0..ba4328b8fe821e0b8f858fb69d2deb687c36ac93 100644 --- a/source/dnode/mnode/impl/inc/mndSubscribe.h +++ b/source/dnode/mnode/impl/inc/mndSubscribe.h @@ -25,6 +25,7 @@ extern "C" { int32_t mndInitSubscribe(SMnode *pMnode); void mndCleanupSubscribe(SMnode *pMnode); +int32_t mndGetGroupNumByTopic(SMnode *pMnode, const char *topicName); SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *CGroup, const char *topicName); SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key); void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub); diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 03434573c4f3b10ab971039fb4d71637b7d1dfa7..625546aa552c77035b4b86dae30d2c0964b66118 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -70,6 +70,7 @@ int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendNullLog(STrans *pTrans); +int32_t mndTransAppendPrepareAction(STrans *pTrans, STransAction *pAction); int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction); int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); @@ -78,15 +79,23 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbnam void mndTransSetSerial(STrans *pTrans); void mndTransSetParallel(STrans *pTrans); void mndTransSetOper(STrans *pTrans, EOperType oper); -int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans); - +int32_t mndTransCheckConflict(SMnode *pMnode, STrans *pTrans); +static int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans) { + return mndTransCheckConflict(pMnode, pTrans); +} int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); int32_t mndTransProcessRsp(SRpcMsg *pRsp); void mndTransPullup(SMnode *pMnode); int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans); -void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader); +void mndTransExecute(SMnode *pMnode, STrans *pTrans); +void mndTransRefresh(SMnode *pMnode, STrans *pTrans); int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, const char *dbname); +SSdbRaw *mndTransEncode(STrans *pTrans); +SSdbRow *mndTransDecode(SSdbRaw *pRaw); +void mndTransDropData(STrans *pTrans); + +bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans); #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index aa7f97f0870dfd83ba4c3296ba1293e091b6fdba..8b930e7f18588a70b8bb144f53417069477e32af 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -35,11 +35,11 @@ SHashObj *mndDupTableHash(SHashObj *pOld); SHashObj *mndDupTopicHash(SHashObj *pOld); int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); -int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int32_t numOfUses, void **ppRsp, - int32_t *pRspLen); int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db); int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic); +int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew); +void mndUserFreeObj(SUserObj *pUser); #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 2ece0da5ebb03f609b085067856c1e39db0f0fb6..7c2f8b5b65386b7a2e27541cf8d34b0f11c5e1ec 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -27,6 +27,7 @@ void mndCleanupVgroup(SMnode *pMnode); SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); +SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); void mndSortVnodeGid(SVgObj *pVgroup); @@ -36,11 +37,12 @@ int64_t mndGetVgroupMemory(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup); SArray *mndBuildDnodesArray(SMnode *, int32_t exceptDnodeId); int32_t mndAllocSmaVgroup(SMnode *, SDbObj *pDb, SVgObj *pVgroup); int32_t mndAllocVgroup(SMnode *, SDbObj *pDb, SVgObj **ppVgroups); +int32_t mndAddPrepareNewVgAction(SMnode *, STrans *pTrans, SVgObj *pVg); int32_t mndAddCreateVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid); int32_t mndAddAlterVnodeConfirmAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup); int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType); int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo); -int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId, bool force); +int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId, bool force, bool unsafe); int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup, SArray *pArray); int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs, diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 4d05637a2b3ab872d5a0ba99c022e21f79ca9b05..4c799e1e1ea4b4ba54811cf3df5ef663afb80e7c 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -20,7 +20,6 @@ #define CLUSTER_VER_NUMBE 1 #define CLUSTER_RESERVE_SIZE 60 -char tsVersionName[16] = "community"; int64_t tsExpireTime = 0; static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); @@ -77,7 +76,6 @@ static SClusterObj *mndAcquireCluster(SMnode *pMnode, void **ppIter) { if (pIter == NULL) break; *ppIter = pIter; - return pCluster; } @@ -124,7 +122,7 @@ static int32_t mndGetClusterUpTimeImp(SClusterObj *pCluster) { #endif } -float mndGetClusterUpTime(SMnode *pMnode) { +int64_t mndGetClusterUpTime(SMnode *pMnode) { int64_t upTime = 0; void *pIter = NULL; SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); @@ -133,7 +131,7 @@ float mndGetClusterUpTime(SMnode *pMnode) { mndReleaseCluster(pMnode, pCluster, pIter); } - return upTime / 86400.0f; + return upTime; } static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 117c1082a58fc9dfcf9cea19d7e1832536a8de5e..82492f930ef29f54722a4dccc410dc828db6e70a 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -23,9 +23,10 @@ #include "tcompare.h" #include "tname.h" -#define MND_CONSUMER_VER_NUMBER 1 +#define MND_CONSUMER_VER_NUMBER 2 #define MND_CONSUMER_RESERVE_SIZE 64 +#define MND_MAX_GROUP_PER_TOPIC 100 #define MND_CONSUMER_LOST_HB_CNT 6 #define MND_CONSUMER_LOST_CLEAR_THRESHOLD 43200 @@ -63,7 +64,7 @@ int32_t mndInitConsumer(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_TMQ_HB, mndProcessMqHbReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_ASK_EP, mndProcessAskEpReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessMqTimerMsg); - mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CONSUMER_LOST, mndProcessConsumerLostMsg); +// mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CONSUMER_LOST, mndProcessConsumerLostMsg); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, mndProcessConsumerClearMsg); @@ -75,6 +76,22 @@ int32_t mndInitConsumer(SMnode *pMnode) { void mndCleanupConsumer(SMnode *pMnode) {} +void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId){ + SMqConsumerClearMsg *pClearMsg = rpcMallocCont(sizeof(SMqConsumerClearMsg)); + if (pClearMsg == NULL) { + mError("consumer:0x%"PRIx64" failed to clear consumer due to out of memory. alloc size:%d", consumerId, (int32_t)sizeof(SMqConsumerClearMsg)); + return; + } + + pClearMsg->consumerId = consumerId; + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, .pCont = pClearMsg, .contLen = sizeof(SMqConsumerClearMsg)}; + + mInfo("consumer:0x%" PRIx64 " drop from sdb", consumerId); + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + return; +} + bool mndRebTryStart() { int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1); mDebug("tq timer, rebalance counter old val:%d", old); @@ -105,50 +122,48 @@ void mndRebCntDec() { } } -static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; - SMqConsumerLostMsg *pLostMsg = pMsg->pCont; - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId); - if (pConsumer == NULL) { - return 0; - } - - mInfo("process consumer lost msg, consumer:0x%" PRIx64 " status:%d(%s)", pLostMsg->consumerId, pConsumer->status, - mndConsumerStatusName(pConsumer->status)); - - if (pConsumer->status != MQ_CONSUMER_STATUS__READY) { - mndReleaseConsumer(pMnode, pConsumer); - return -1; - } - - SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup); - pConsumerNew->updateType = CONSUMER_UPDATE__LOST; - - mndReleaseConsumer(pMnode, pConsumer); - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "lost-csm"); - if (pTrans == NULL) { - goto FAIL; - } - - if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { - goto FAIL; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - goto FAIL; - } - - tDeleteSMqConsumerObj(pConsumerNew); - taosMemoryFree(pConsumerNew); - mndTransDrop(pTrans); - return 0; -FAIL: - tDeleteSMqConsumerObj(pConsumerNew); - taosMemoryFree(pConsumerNew); - mndTransDrop(pTrans); - return -1; -} +//static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) { +// SMnode *pMnode = pMsg->info.node; +// SMqConsumerLostMsg *pLostMsg = pMsg->pCont; +// SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId); +// if (pConsumer == NULL) { +// return 0; +// } +// +// mInfo("process consumer lost msg, consumer:0x%" PRIx64 " status:%d(%s)", pLostMsg->consumerId, pConsumer->status, +// mndConsumerStatusName(pConsumer->status)); +// +// if (pConsumer->status != MQ_CONSUMER_STATUS_READY) { +// mndReleaseConsumer(pMnode, pConsumer); +// return -1; +// } +// +// SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup); +// pConsumerNew->updateType = CONSUMER_UPDATE_TIMER_LOST; +// +// mndReleaseConsumer(pMnode, pConsumer); +// +// STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "lost-csm"); +// if (pTrans == NULL) { +// goto FAIL; +// } +// +// if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { +// goto FAIL; +// } +// +// if (mndTransPrepare(pMnode, pTrans) != 0) { +// goto FAIL; +// } +// +// tDeleteSMqConsumerObj(pConsumerNew, true); +// mndTransDrop(pTrans); +// return 0; +//FAIL: +// tDeleteSMqConsumerObj(pConsumerNew, true); +// mndTransDrop(pTrans); +// return -1; +//} static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; @@ -162,14 +177,14 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { mInfo("receive consumer recover msg, consumer:0x%" PRIx64 " status:%d(%s)", pRecoverMsg->consumerId, pConsumer->status, mndConsumerStatusName(pConsumer->status)); - if (pConsumer->status != MQ_CONSUMER_STATUS__LOST_REBD) { + if (pConsumer->status != MQ_CONSUMER_STATUS_LOST) { mndReleaseConsumer(pMnode, pConsumer); terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; return -1; } SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup); - pConsumerNew->updateType = CONSUMER_UPDATE__RECOVER; + pConsumerNew->updateType = CONSUMER_UPDATE_REC; mndReleaseConsumer(pMnode, pConsumer); @@ -181,13 +196,13 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL; if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL; - tDeleteSMqConsumerObj(pConsumerNew); - taosMemoryFree(pConsumerNew); + tDeleteSMqConsumerObj(pConsumerNew, true); + mndTransDrop(pTrans); return 0; FAIL: - tDeleteSMqConsumerObj(pConsumerNew); - taosMemoryFree(pConsumerNew); + tDeleteSMqConsumerObj(pConsumerNew, true); + mndTransDrop(pTrans); return -1; } @@ -206,13 +221,13 @@ static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) { mInfo("consumer:0x%" PRIx64 " needs to be cleared, status %s", pClearMsg->consumerId, mndConsumerStatusName(pConsumer->status)); - if (pConsumer->status != MQ_CONSUMER_STATUS__LOST_REBD) { - mndReleaseConsumer(pMnode, pConsumer); - return -1; - } +// if (pConsumer->status != MQ_CONSUMER_STATUS_LOST) { +// mndReleaseConsumer(pMnode, pConsumer); +// return -1; +// } SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup); - pConsumerNew->updateType = CONSUMER_UPDATE__LOST; +// pConsumerNew->updateType = CONSUMER_UPDATE_TIMER_LOST; mndReleaseConsumer(pMnode, pConsumer); @@ -223,14 +238,14 @@ static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) { if (mndSetConsumerDropLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL; if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL; - tDeleteSMqConsumerObj(pConsumerNew); - taosMemoryFree(pConsumerNew); + tDeleteSMqConsumerObj(pConsumerNew, true); + mndTransDrop(pTrans); return 0; FAIL: - tDeleteSMqConsumerObj(pConsumerNew); - taosMemoryFree(pConsumerNew); + tDeleteSMqConsumerObj(pConsumerNew, true); + mndTransDrop(pTrans); return -1; } @@ -266,7 +281,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { // rebalance cannot be parallel if (!mndRebTryStart()) { - mDebug("mq rebalance already in progress, do nothing"); + mInfo("mq rebalance already in progress, do nothing"); return 0; } @@ -297,56 +312,29 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); int32_t status = atomic_load_32(&pConsumer->status); - mDebug("check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", uptime:%" PRId64 ", hbstatus:%d", - pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->upTime, + mInfo("check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64 ", hbstatus:%d", + pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->createTime, hbStatus); - if (status == MQ_CONSUMER_STATUS__READY) { - if (hbStatus > MND_CONSUMER_LOST_HB_CNT) { - SMqConsumerLostMsg *pLostMsg = rpcMallocCont(sizeof(SMqConsumerLostMsg)); - if (pLostMsg == NULL) { - mError("consumer:0x%"PRIx64" failed to transfer consumer status to lost due to out of memory. alloc size:%d", - pConsumer->consumerId, (int32_t)sizeof(SMqConsumerLostMsg)); - continue; - } - - pLostMsg->consumerId = pConsumer->consumerId; - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_TMQ_CONSUMER_LOST, .pCont = pLostMsg, .contLen = sizeof(SMqConsumerLostMsg)}; - - mDebug("consumer:0x%"PRIx64" hb not received beyond threshold %d, set to lost", pConsumer->consumerId, - MND_CONSUMER_LOST_HB_CNT); - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); - } - } else if (status == MQ_CONSUMER_STATUS__LOST_REBD) { - // if the client is lost longer than one day, clear it. Otherwise, do nothing about the lost consumers. - if (hbStatus > MND_CONSUMER_LOST_CLEAR_THRESHOLD) { - SMqConsumerClearMsg *pClearMsg = rpcMallocCont(sizeof(SMqConsumerClearMsg)); - if (pClearMsg == NULL) { - mError("consumer:0x%"PRIx64" failed to clear consumer due to out of memory. alloc size:%d", - pConsumer->consumerId, (int32_t)sizeof(SMqConsumerClearMsg)); - continue; + if (status == MQ_CONSUMER_STATUS_READY) { + if (taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close + mndDropConsumerFromSdb(pMnode, pConsumer->consumerId); + } else if (hbStatus > MND_CONSUMER_LOST_HB_CNT) { + taosRLockLatch(&pConsumer->lock); + int32_t topicNum = taosArrayGetSize(pConsumer->currentTopics); + for (int32_t i = 0; i < topicNum; i++) { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + char *removedTopic = taosArrayGetP(pConsumer->currentTopics, i); + mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); + SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); + taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); } - - pClearMsg->consumerId = pConsumer->consumerId; - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, .pCont = pClearMsg, .contLen = sizeof(SMqConsumerClearMsg)}; - - mDebug("consumer:0x%" PRIx64 " lost beyond threshold %d, clear it", pConsumer->consumerId, - MND_CONSUMER_LOST_CLEAR_THRESHOLD); - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + taosRUnLockLatch(&pConsumer->lock); } - } else if (status == MQ_CONSUMER_STATUS__LOST) { - taosRLockLatch(&pConsumer->lock); - int32_t topicNum = taosArrayGetSize(pConsumer->currentTopics); - for (int32_t i = 0; i < topicNum; i++) { - char key[TSDB_SUBSCRIBE_KEY_LEN]; - char *removedTopic = taosArrayGetP(pConsumer->currentTopics, i); - mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); - SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); - taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); + } else if (status == MQ_CONSUMER_STATUS_LOST) { + if (hbStatus > MND_CONSUMER_LOST_CLEAR_THRESHOLD) { // clear consumer if lost a day + mndDropConsumerFromSdb(pMnode, pConsumer->consumerId); } - taosRUnLockLatch(&pConsumer->lock); } else { // MQ_CONSUMER_STATUS_REBALANCE taosRLockLatch(&pConsumer->lock); @@ -374,7 +362,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { } if (taosHashGetSize(pRebMsg->rebSubHash) != 0) { - mInfo("mq rebalance will be triggered"); + mInfo("mq rebalance will be triggered"); SRpcMsg rpcMsg = { .msgType = TDMT_MND_TMQ_DO_REBALANCE, .pCont = pRebMsg, @@ -391,12 +379,13 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { } static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { + int32_t code = 0; SMnode *pMnode = pMsg->info.node; SMqHbReq req = {0}; - if (tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req) < 0) { + if ((code = tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req)) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + goto end; } int64_t consumerId = req.consumerId; @@ -404,14 +393,15 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { if (pConsumer == NULL) { mError("consumer:0x%" PRIx64 " not exist", consumerId); terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; - return -1; + code = -1; + goto end; } atomic_store_32(&pConsumer->hbStatus, 0); int32_t status = atomic_load_32(&pConsumer->status); - if (status == MQ_CONSUMER_STATUS__LOST_REBD) { + if (status == MQ_CONSUMER_STATUS_LOST) { mInfo("try to recover consumer:0x%" PRIx64 "", consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); @@ -424,9 +414,32 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &pRpcMsg); } + for(int i = 0; i < taosArrayGetSize(req.topics); i++){ + TopicOffsetRows* data = taosArrayGet(req.topics, i); + mInfo("heartbeat report offset rows.%s:%s", pConsumer->cgroup, data->topicName); + + SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, data->topicName); + if(pSub == NULL){ + ASSERT(0); + continue; + } + taosWLockLatch(&pSub->lock); + SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t)); + if(pConsumerEp){ + taosArrayDestroy(pConsumerEp->offsetRows); + pConsumerEp->offsetRows = data->offsetRows; + data->offsetRows = NULL; + } + taosWUnLockLatch(&pSub->lock); + + mndReleaseSubscribe(pMnode, pSub); + } + mndReleaseConsumer(pMnode, pConsumer); - return 0; +end: + tDeatroySMqHbReq(&req); + return code; } static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { @@ -454,7 +467,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { mError("consumer:0x%" PRIx64 " group:%s not consistent with data in sdb, saved cgroup:%s", consumerId, req.cgroup, pConsumer->cgroup); terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; - return -1; + goto FAIL; } atomic_store_32(&pConsumer->hbStatus, 0); @@ -462,7 +475,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { // 1. check consumer status int32_t status = atomic_load_32(&pConsumer->status); - if (status == MQ_CONSUMER_STATUS__LOST_REBD) { + if (status == MQ_CONSUMER_STATUS_LOST) { mInfo("try to recover consumer:0x%" PRIx64, consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); @@ -476,10 +489,10 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &pRpcMsg); } - if (status != MQ_CONSUMER_STATUS__READY) { + if (status != MQ_CONSUMER_STATUS_READY) { mInfo("consumer:0x%" PRIx64 " not ready, status: %s", consumerId, mndConsumerStatusName(status)); terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; - return -1; + goto FAIL; } int32_t serverEpoch = atomic_load_32(&pConsumer->epoch); @@ -503,7 +516,10 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { char *topic = taosArrayGetP(pConsumer->currentTopics, i); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); // txn guarantees pSub is created - + if(pSub == NULL) { + ASSERT(0); + continue; + } taosRLockLatch(&pSub->lock); SMqSubTopicEp topicEp = {0}; @@ -511,6 +527,12 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { // 2.1 fetch topic schema SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); + if(pTopic == NULL) { + ASSERT(0); + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); + continue; + } taosRLockLatch(&pTopic->lock); tstrncpy(topicEp.db, pTopic->db, TSDB_DB_FNAME_LEN); topicEp.schema.nCols = pTopic->schema.nCols; @@ -561,7 +583,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { void *buf = rpcMallocCont(tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + goto FAIL; } SMqRspHead* pHead = buf; @@ -644,10 +666,11 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { SCMSubscribeReq subscribe = {0}; tDeserializeSCMSubscribeReq(msgStr, &subscribe); - uint64_t consumerId = subscribe.consumerId; + int64_t consumerId = subscribe.consumerId; char *cgroup = subscribe.cgroup; SMqConsumerObj *pExistedConsumer = NULL; SMqConsumerObj *pConsumerNew = NULL; + STrans *pTrans = NULL; int32_t code = -1; SArray *pTopicList = subscribe.topicNames; @@ -655,9 +678,17 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem); int32_t newTopicNum = taosArrayGetSize(pTopicList); + for(int i = 0; i < newTopicNum; i++){ + int32_t gNum = mndGetGroupNumByTopic(pMnode, (const char*)taosArrayGetP(pTopicList, i)); + if(gNum >= MND_MAX_GROUP_PER_TOPIC){ + terrno = TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE; + code = terrno; + goto _over; + } + } // check topic existence - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe"); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe"); if (pTrans == NULL) { goto _over; } @@ -675,8 +706,12 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); tstrncpy(pConsumerNew->clientId, subscribe.clientId, tListLen(pConsumerNew->clientId)); - // set the update type - pConsumerNew->updateType = CONSUMER_UPDATE__REBALANCE; + pConsumerNew->withTbName = subscribe.withTbName; + pConsumerNew->autoCommit = subscribe.autoCommit; + pConsumerNew->autoCommitInterval = subscribe.autoCommitInterval; + pConsumerNew->resetOffsetCfg = subscribe.resetOffsetCfg; + +// pConsumerNew->updateType = CONSUMER_UPDATE_SUB; // use insert logic taosArrayDestroy(pConsumerNew->assignedTopics); pConsumerNew->assignedTopics = taosArrayDup(pTopicList, topicNameDup); @@ -695,7 +730,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { " cgroup:%s, current status:%d(%s), subscribe topic num: %d", consumerId, subscribe.cgroup, status, mndConsumerStatusName(status), newTopicNum); - if (status != MQ_CONSUMER_STATUS__READY) { + if (status != MQ_CONSUMER_STATUS_READY) { terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; goto _over; } @@ -706,11 +741,11 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { } // set the update type - pConsumerNew->updateType = CONSUMER_UPDATE__REBALANCE; + pConsumerNew->updateType = CONSUMER_UPDATE_SUB; taosArrayDestroy(pConsumerNew->assignedTopics); pConsumerNew->assignedTopics = taosArrayDup(pTopicList, topicNameDup); - int32_t oldTopicNum = (pExistedConsumer->currentTopics) ? taosArrayGetSize(pExistedConsumer->currentTopics) : 0; + int32_t oldTopicNum = taosArrayGetSize(pExistedConsumer->currentTopics); int32_t i = 0, j = 0; while (i < oldTopicNum || j < newTopicNum) { @@ -765,10 +800,7 @@ _over: mndReleaseConsumer(pMnode, pExistedConsumer); } - if (pConsumerNew) { - tDeleteSMqConsumerObj(pConsumerNew); - taosMemoryFree(pConsumerNew); - } + tDeleteSMqConsumerObj(pConsumerNew, true); // TODO: replace with destroy subscribe msg taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree); @@ -822,7 +854,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { goto CM_DECODE_OVER; } - if (sver != MND_CONSUMER_VER_NUMBER) { + if (sver < 1 || sver > MND_CONSUMER_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto CM_DECODE_OVER; } @@ -849,7 +881,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, buf, len, CM_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_DECODE_OVER); - if (tDecodeSMqConsumerObj(buf, pConsumer) == NULL) { + if (tDecodeSMqConsumerObj(buf, pConsumer, sver) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; // TODO set correct error code goto CM_DECODE_OVER; } @@ -868,17 +900,17 @@ CM_DECODE_OVER: } static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) { - mDebug("consumer:0x%" PRIx64 " cgroup:%s status:%d(%s) epoch:%d load from sdb, perform insert action", + mInfo("consumer:0x%" PRIx64 " sub insert, cgroup:%s status:%d(%s) epoch:%d", pConsumer->consumerId, pConsumer->cgroup, pConsumer->status, mndConsumerStatusName(pConsumer->status), pConsumer->epoch); - pConsumer->subscribeTime = pConsumer->upTime; + pConsumer->subscribeTime = pConsumer->createTime; return 0; } static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { - mDebug("consumer:0x%" PRIx64 " perform delete action, status:(%d)%s", pConsumer->consumerId, pConsumer->status, + mInfo("consumer:0x%" PRIx64 " perform delete action, status:(%d)%s", pConsumer->consumerId, pConsumer->status, mndConsumerStatusName(pConsumer->status)); - tDeleteSMqConsumerObj(pConsumer); + tDeleteSMqConsumerObj(pConsumer, false); return 0; } @@ -887,10 +919,9 @@ static void updateConsumerStatus(SMqConsumerObj *pConsumer) { if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) { if (status == MQ_CONSUMER_STATUS_REBALANCE) { - pConsumer->status = MQ_CONSUMER_STATUS__READY; - } else if (status == MQ_CONSUMER_STATUS__LOST) { - ASSERT(taosArrayGetSize(pConsumer->currentTopics) == 0); - pConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; + pConsumer->status = MQ_CONSUMER_STATUS_READY; + } else if (status == MQ_CONSUMER_STATUS_READY) { + pConsumer->status = MQ_CONSUMER_STATUS_LOST; } } } @@ -904,7 +935,7 @@ static void removeFromNewTopicList(SMqConsumerObj *pConsumer, const char *pTopic taosArrayRemove(pConsumer->rebNewTopics, i); taosMemoryFree(p); - mDebug("consumer:0x%" PRIx64 " remove new topic:%s in the topic list, remain newTopics:%d", pConsumer->consumerId, + mInfo("consumer:0x%" PRIx64 " remove new topic:%s in the topic list, remain newTopics:%d", pConsumer->consumerId, pTopic, (int)taosArrayGetSize(pConsumer->rebNewTopics)); break; } @@ -920,7 +951,7 @@ static void removeFromRemoveTopicList(SMqConsumerObj *pConsumer, const char *pTo taosArrayRemove(pConsumer->rebRemovedTopics, i); taosMemoryFree(p); - mDebug("consumer:0x%" PRIx64 " remove topic:%s in the removed topic list, remain removedTopics:%d", + mInfo("consumer:0x%" PRIx64 " remove topic:%s in the removed topic list, remain removedTopics:%d", pConsumer->consumerId, pTopic, (int)taosArrayGetSize(pConsumer->rebRemovedTopics)); break; } @@ -935,7 +966,7 @@ static void removeFromCurrentTopicList(SMqConsumerObj *pConsumer, const char *pT taosArrayRemove(pConsumer->currentTopics, i); taosMemoryFree(topic); - mDebug("consumer:0x%" PRIx64 " remove topic:%s in the current topic list, remain currentTopics:%d", + mInfo("consumer:0x%" PRIx64 " remove topic:%s in the current topic list, remain currentTopics:%d", pConsumer->consumerId, pTopic, (int)taosArrayGetSize(pConsumer->currentTopics)); break; } @@ -958,47 +989,46 @@ static bool existInCurrentTopicList(const SMqConsumerObj* pConsumer, const char* } static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { - mDebug("consumer:0x%" PRIx64 " perform update action, update type:%d, subscribe-time:%" PRId64 ", uptime:%" PRId64, - pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->upTime); + mInfo("consumer:0x%" PRIx64 " perform update action, update type:%d, subscribe-time:%" PRId64 ", createTime:%" PRId64, + pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->createTime); taosWLockLatch(&pOldConsumer->lock); - if (pNewConsumer->updateType == CONSUMER_UPDATE__REBALANCE) { + if (pNewConsumer->updateType == CONSUMER_UPDATE_SUB) { TSWAP(pOldConsumer->rebNewTopics, pNewConsumer->rebNewTopics); TSWAP(pOldConsumer->rebRemovedTopics, pNewConsumer->rebRemovedTopics); TSWAP(pOldConsumer->assignedTopics, pNewConsumer->assignedTopics); - pOldConsumer->subscribeTime = pNewConsumer->upTime; + pOldConsumer->subscribeTime = taosGetTimestampMs(); pOldConsumer->status = MQ_CONSUMER_STATUS_REBALANCE; - } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { - int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); - for (int32_t i = 0; i < sz; i++) { - char *topic = taosStrdup(taosArrayGetP(pOldConsumer->currentTopics, i)); - taosArrayPush(pOldConsumer->rebRemovedTopics, &topic); - } - - pOldConsumer->rebalanceTime = pNewConsumer->upTime; - - int32_t prevStatus = pOldConsumer->status; - pOldConsumer->status = MQ_CONSUMER_STATUS__LOST; - mDebug("consumer:0x%" PRIx64 " state %s -> %s, reb-time:%" PRId64 ", reb-removed-topics:%d", - pOldConsumer->consumerId, mndConsumerStatusName(prevStatus), mndConsumerStatusName(pOldConsumer->status), - pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); - } else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) { + mInfo("consumer:0x%" PRIx64 " sub update, modify existed consumer",pOldConsumer->consumerId); +// } else if (pNewConsumer->updateType == CONSUMER_UPDATE_TIMER_LOST) { +// int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); +// for (int32_t i = 0; i < sz; i++) { +// char *topic = taosStrdup(taosArrayGetP(pOldConsumer->currentTopics, i)); +// taosArrayPush(pOldConsumer->rebRemovedTopics, &topic); +// } +// +// int32_t prevStatus = pOldConsumer->status; +// pOldConsumer->status = MQ_CONSUMER_STATUS_LOST; +// mInfo("consumer:0x%" PRIx64 " timer update, timer lost. state %s -> %s, reb-time:%" PRId64 ", reb-removed-topics:%d", +// pOldConsumer->consumerId, mndConsumerStatusName(prevStatus), mndConsumerStatusName(pOldConsumer->status), +// pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); + } else if (pNewConsumer->updateType == CONSUMER_UPDATE_REC) { int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics); for (int32_t i = 0; i < sz; i++) { char *topic = taosStrdup(taosArrayGetP(pOldConsumer->assignedTopics, i)); taosArrayPush(pOldConsumer->rebNewTopics, &topic); } - pOldConsumer->rebalanceTime = pNewConsumer->upTime; pOldConsumer->status = MQ_CONSUMER_STATUS_REBALANCE; - } else if (pNewConsumer->updateType == CONSUMER_UPDATE__TOUCH) { + mInfo("consumer:0x%" PRIx64 " timer update, timer recover",pOldConsumer->consumerId); + } else if (pNewConsumer->updateType == CONSUMER_UPDATE_REB) { atomic_add_fetch_32(&pOldConsumer->epoch, 1); - pOldConsumer->rebalanceTime = pNewConsumer->upTime; - - } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) { + pOldConsumer->rebalanceTime = taosGetTimestampMs(); + mInfo("consumer:0x%" PRIx64 " reb update, only rebalance time", pOldConsumer->consumerId); + } else if (pNewConsumer->updateType == CONSUMER_ADD_REB) { char *pNewTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); // check if exist in current topic @@ -1007,6 +1037,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // add to current topic bool existing = existInCurrentTopicList(pOldConsumer, pNewTopic); if (existing) { + mError("consumer:0x%" PRIx64 "new topic:%s should not in currentTopics", pOldConsumer->consumerId, pNewTopic); taosMemoryFree(pNewTopic); } else { // added into current topic list taosArrayPush(pOldConsumer->currentTopics, &pNewTopic); @@ -1018,17 +1049,17 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, updateConsumerStatus(pOldConsumer); // the re-balance is triggered when the new consumer is launched. - pOldConsumer->rebalanceTime = pNewConsumer->upTime; + pOldConsumer->rebalanceTime = taosGetTimestampMs(); atomic_add_fetch_32(&pOldConsumer->epoch, 1); - mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 + mInfo("consumer:0x%" PRIx64 " reb update add, state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d, newTopics:%d, removeTopics:%d", pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status, mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics), (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); - } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) { + } else if (pNewConsumer->updateType == CONSUMER_REMOVE_REB) { char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0); // remove from removed topic @@ -1041,10 +1072,10 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, int32_t status = pOldConsumer->status; updateConsumerStatus(pOldConsumer); - pOldConsumer->rebalanceTime = pNewConsumer->upTime; + pOldConsumer->rebalanceTime = taosGetTimestampMs(); atomic_add_fetch_32(&pOldConsumer->epoch, 1); - mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 + mInfo("consumer:0x%" PRIx64 " reb update remove, state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d, newTopics:%d, removeTopics:%d", pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status, mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime, @@ -1083,13 +1114,13 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * } if (taosArrayGetSize(pConsumer->assignedTopics) == 0) { - mDebug("showing consumer:0x%" PRIx64 " no assigned topic, skip", pConsumer->consumerId); + mInfo("showing consumer:0x%" PRIx64 " no assigned topic, skip", pConsumer->consumerId); sdbRelease(pSdb, pConsumer); continue; } taosRLockLatch(&pConsumer->lock); - mDebug("showing consumer:0x%" PRIx64, pConsumer->consumerId); + mInfo("showing consumer:0x%" PRIx64, pConsumer->consumerId); int32_t topicSz = taosArrayGetSize(pConsumer->assignedTopics); bool hasTopic = true; @@ -1107,8 +1138,12 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * int32_t cols = 0; // consumer id + char consumerIdHex[32] = {0}; + sprintf(varDataVal(consumerIdHex), "0x%"PRIx64, pConsumer->consumerId); + varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex))); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->consumerId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)consumerIdHex, false); // consumer group char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -1149,7 +1184,7 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * // up time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->upTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->createTime, false); // subscribe time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -1159,6 +1194,17 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0); + char buf[TSDB_OFFSET_LEN] = {0}; + STqOffsetVal pVal = {.type = pConsumer->resetOffsetCfg}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &pVal); + + char parasStr[64 + TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0}; + sprintf(varDataVal(parasStr), "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName, pConsumer->autoCommit, pConsumer->autoCommitInterval, buf); + varDataSetLen(parasStr, strlen(varDataVal(parasStr))); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)parasStr, false); + numOfRows++; } @@ -1179,10 +1225,9 @@ static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { static const char *mndConsumerStatusName(int status) { switch (status) { - case MQ_CONSUMER_STATUS__READY: + case MQ_CONSUMER_STATUS_READY: return "ready"; - case MQ_CONSUMER_STATUS__LOST: - case MQ_CONSUMER_STATUS__LOST_REBD: + case MQ_CONSUMER_STATUS_LOST: return "lost"; case MQ_CONSUMER_STATUS_REBALANCE: return "rebalancing"; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 23b2b9d7c6ff7ca94ecbf28789598aa653b58d66..1bd629e56f59c4b49bd1027a04ce81ccea166445 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -382,6 +382,40 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { return terrno; } +static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pCfg) { + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1; + if (pCfg->pages < TSDB_MIN_PAGES_PER_VNODE || pCfg->pages > TSDB_MAX_PAGES_PER_VNODE) return -1; + if (pCfg->pageSize < TSDB_MIN_PAGESIZE_PER_VNODE || pCfg->pageSize > TSDB_MAX_PAGESIZE_PER_VNODE) return -1; + if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1; + if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1; + if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1; + if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) return -1; + if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1; + if (pCfg->daysToKeep0 > pCfg->daysToKeep1) return -1; + if (pCfg->daysToKeep1 > pCfg->daysToKeep2) return -1; + if (pCfg->walFsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->walFsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1; + if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1; + if (pCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return -1; + if (pCfg->cacheLastSize < TSDB_MIN_DB_CACHE_SIZE || pCfg->cacheLastSize > TSDB_MAX_DB_CACHE_SIZE) return -1; + if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1; + if (pCfg->replications != 1 && pCfg->replications != 3) return -1; + if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1; + if (pCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1; + if (pCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1; + if (pCfg->minRows > pCfg->maxRows) return -1; + if (pCfg->walRetentionPeriod < TSDB_DB_MIN_WAL_RETENTION_PERIOD) return -1; + if (pCfg->walRetentionSize < TSDB_DB_MIN_WAL_RETENTION_SIZE) return -1; + if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; + if (pCfg->replications > mndGetDnodeSize(pMnode)) { + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; + } + + terrno = 0; + return terrno; +} + static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->numOfVgroups < 0) pCfg->numOfVgroups = TSDB_DEFAULT_VN_PER_DB; if (pCfg->numOfStables < 0) pCfg->numOfStables = TSDB_DEFAULT_DB_SINGLE_STABLE; @@ -414,6 +448,13 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; } +static int32_t mndSetPrepareNewVgActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) { + if (mndAddPrepareNewVgAction(pMnode, pTrans, (pVgroups + v)) != 0) return -1; + } + return 0; +} + static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { SSdbRaw *pDbRaw = mndDbActionEncode(pDb); if (pDbRaw == NULL) return -1; @@ -424,7 +465,7 @@ static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); if (pVgRaw == NULL) return -1; if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; - if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_UPDATE) != 0) return -1; } return 0; @@ -446,7 +487,8 @@ static int32_t mndSetCreateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD return 0; } -static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { +static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups, + SUserObj *pUserDuped) { SSdbRaw *pDbRaw = mndDbActionEncode(pDb); if (pDbRaw == NULL) return -1; if (mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1; @@ -459,6 +501,13 @@ static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj * if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; } + if (pUserDuped) { + SSdbRaw *pUserRaw = mndUserActionEncode(pUserDuped); + if (pUserRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pUserRaw) != 0) return -1; + if (sdbSetRawStatus(pUserRaw, SDB_STATUS_READY) != 0) return -1; + } + return 0; } @@ -565,6 +614,15 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, return -1; } + // add database privileges for user + SUserObj newUserObj = {0}, *pNewUserDuped = NULL; + if (!pUser->superUser) { + if (mndUserDupObj(pUser, &newUserObj) != 0) goto _OVER; + taosHashPut(newUserObj.readDbs, dbObj.name, strlen(dbObj.name) + 1, dbObj.name, TSDB_FILENAME_LEN); + taosHashPut(newUserObj.writeDbs, dbObj.name, strlen(dbObj.name) + 1, dbObj.name, TSDB_FILENAME_LEN); + pNewUserDuped = &newUserObj; + } + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "create-db"); if (pTrans == NULL) goto _OVER; @@ -572,12 +630,13 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, mInfo("trans:%d, used to create db:%s", pTrans->id, pCreate->db); mndTransSetDbName(pTrans, dbObj.name, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; mndTransSetOper(pTrans, MND_OPER_CREATE_DB); + if (mndSetPrepareNewVgActions(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; - if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; + if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups, pNewUserDuped) != 0) goto _OVER; if (mndSetCreateDbRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -586,6 +645,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, _OVER: taosMemoryFree(pVgroups); + mndUserFreeObj(&newUserObj); mndTransDrop(pTrans); return code; } @@ -814,7 +874,7 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p int32_t code = -1; mndTransSetDbName(pTrans, pOld->name, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; @@ -871,7 +931,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); if (code != 0) goto _OVER; - code = mndCheckDbCfg(pMnode, &dbObj.cfg); + code = mndCheckInChangeDbCfg(pMnode, &dbObj.cfg); if (code != 0) goto _OVER; dbObj.cfgVersion++; @@ -898,6 +958,42 @@ _OVER: return code; } +static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) { + strcpy(cfgRsp->db, pDb->name); + cfgRsp->dbId = pDb->uid; + cfgRsp->cfgVersion = pDb->cfgVersion; + cfgRsp->numOfVgroups = pDb->cfg.numOfVgroups; + cfgRsp->numOfStables = pDb->cfg.numOfStables; + cfgRsp->buffer = pDb->cfg.buffer; + cfgRsp->cacheSize = pDb->cfg.cacheLastSize; + cfgRsp->pageSize = pDb->cfg.pageSize; + cfgRsp->pages = pDb->cfg.pages; + cfgRsp->daysPerFile = pDb->cfg.daysPerFile; + cfgRsp->daysToKeep0 = pDb->cfg.daysToKeep0; + cfgRsp->daysToKeep1 = pDb->cfg.daysToKeep1; + cfgRsp->daysToKeep2 = pDb->cfg.daysToKeep2; + cfgRsp->minRows = pDb->cfg.minRows; + cfgRsp->maxRows = pDb->cfg.maxRows; + cfgRsp->walFsyncPeriod = pDb->cfg.walFsyncPeriod; + cfgRsp->hashPrefix = pDb->cfg.hashPrefix; + cfgRsp->hashSuffix = pDb->cfg.hashSuffix; + cfgRsp->walLevel = pDb->cfg.walLevel; + cfgRsp->precision = pDb->cfg.precision; + cfgRsp->compression = pDb->cfg.compression; + cfgRsp->replications = pDb->cfg.replications; + cfgRsp->strict = pDb->cfg.strict; + cfgRsp->cacheLast = pDb->cfg.cacheLast; + cfgRsp->tsdbPageSize = pDb->cfg.tsdbPageSize; + cfgRsp->walRetentionPeriod = pDb->cfg.walRetentionPeriod; + cfgRsp->walRollPeriod = pDb->cfg.walRollPeriod; + cfgRsp->walRetentionSize = pDb->cfg.walRetentionSize; + cfgRsp->walSegmentSize = pDb->cfg.walSegmentSize; + cfgRsp->numOfRetensions = pDb->cfg.numOfRetensions; + cfgRsp->pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); + cfgRsp->schemaless = pDb->cfg.schemaless; + cfgRsp->sstTrigger = pDb->cfg.sstTrigger; +} + static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -910,41 +1006,15 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { goto _OVER; } - pDb = mndAcquireDb(pMnode, cfgReq.db); - if (pDb == NULL) { - goto _OVER; - } + if (strcasecmp(cfgReq.db, TSDB_INFORMATION_SCHEMA_DB) && strcasecmp(cfgReq.db, TSDB_PERFORMANCE_SCHEMA_DB)) { + pDb = mndAcquireDb(pMnode, cfgReq.db); + if (pDb == NULL) { + goto _OVER; + } - cfgRsp.numOfVgroups = pDb->cfg.numOfVgroups; - cfgRsp.numOfStables = pDb->cfg.numOfStables; - cfgRsp.buffer = pDb->cfg.buffer; - cfgRsp.cacheSize = pDb->cfg.cacheLastSize; - cfgRsp.pageSize = pDb->cfg.pageSize; - cfgRsp.pages = pDb->cfg.pages; - cfgRsp.daysPerFile = pDb->cfg.daysPerFile; - cfgRsp.daysToKeep0 = pDb->cfg.daysToKeep0; - cfgRsp.daysToKeep1 = pDb->cfg.daysToKeep1; - cfgRsp.daysToKeep2 = pDb->cfg.daysToKeep2; - cfgRsp.minRows = pDb->cfg.minRows; - cfgRsp.maxRows = pDb->cfg.maxRows; - cfgRsp.walFsyncPeriod = pDb->cfg.walFsyncPeriod; - cfgRsp.hashPrefix = pDb->cfg.hashPrefix; - cfgRsp.hashSuffix = pDb->cfg.hashSuffix; - cfgRsp.walLevel = pDb->cfg.walLevel; - cfgRsp.precision = pDb->cfg.precision; - cfgRsp.compression = pDb->cfg.compression; - cfgRsp.replications = pDb->cfg.replications; - cfgRsp.strict = pDb->cfg.strict; - cfgRsp.cacheLast = pDb->cfg.cacheLast; - cfgRsp.tsdbPageSize = pDb->cfg.tsdbPageSize; - cfgRsp.walRetentionPeriod = pDb->cfg.walRetentionPeriod; - cfgRsp.walRollPeriod = pDb->cfg.walRollPeriod; - cfgRsp.walRetentionSize = pDb->cfg.walRetentionSize; - cfgRsp.walSegmentSize = pDb->cfg.walSegmentSize; - cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions; - cfgRsp.pRetensions = pDb->cfg.pRetensions; - cfgRsp.schemaless = pDb->cfg.schemaless; - cfgRsp.sstTrigger = pDb->cfg.sstTrigger; + mndDumpDbCfgInfo(&cfgRsp, pDb); + } + int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &cfgRsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { @@ -962,6 +1032,8 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { _OVER: + tFreeSDbCfgRsp(&cfgRsp); + if (code != 0) { mError("db:%s, failed to get cfg since %s", cfgReq.db, terrstr()); } @@ -1099,7 +1171,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { mInfo("trans:%d start to drop db:%s", pTrans->id, pDb->name); mndTransSetDbName(pTrans, pDb->name, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { goto _OVER; } @@ -1231,11 +1303,10 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { sdbRelease(pSdb, pVgroup); if (pDb && (vindex >= pDb->cfg.numOfVgroups)) { + sdbCancelFetch(pSdb, pIter); break; } } - - sdbCancelFetch(pSdb, pIter); } int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUseDbReq *pReq) { @@ -1341,103 +1412,119 @@ _OVER: return code; } -int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen) { - SUseDbBatchRsp batchUseRsp = {0}; - batchUseRsp.pArray = taosArrayInit(numOfDbs, sizeof(SUseDbRsp)); - if (batchUseRsp.pArray == NULL) { +int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen) { + SDbHbBatchRsp batchRsp = {0}; + batchRsp.pArray = taosArrayInit(numOfDbs, sizeof(SDbHbRsp)); + if (batchRsp.pArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } for (int32_t i = 0; i < numOfDbs; ++i) { - SDbVgVersion *pDbVgVersion = &pDbs[i]; - pDbVgVersion->dbId = be64toh(pDbVgVersion->dbId); - pDbVgVersion->vgVersion = htonl(pDbVgVersion->vgVersion); - pDbVgVersion->numOfTable = htonl(pDbVgVersion->numOfTable); - pDbVgVersion->stateTs = be64toh(pDbVgVersion->stateTs); + SDbCacheInfo *pDbCacheInfo = &pDbs[i]; + pDbCacheInfo->dbId = be64toh(pDbCacheInfo->dbId); + pDbCacheInfo->vgVersion = htonl(pDbCacheInfo->vgVersion); + pDbCacheInfo->cfgVersion = htonl(pDbCacheInfo->cfgVersion); + pDbCacheInfo->numOfTable = htonl(pDbCacheInfo->numOfTable); + pDbCacheInfo->stateTs = be64toh(pDbCacheInfo->stateTs); - SUseDbRsp usedbRsp = {0}; + SDbHbRsp rsp = {0}; - if ((0 == strcasecmp(pDbVgVersion->dbFName, TSDB_INFORMATION_SCHEMA_DB) || - (0 == strcasecmp(pDbVgVersion->dbFName, TSDB_PERFORMANCE_SCHEMA_DB)))) { - memcpy(usedbRsp.db, pDbVgVersion->dbFName, TSDB_DB_FNAME_LEN); + if ((0 == strcasecmp(pDbCacheInfo->dbFName, TSDB_INFORMATION_SCHEMA_DB) || + (0 == strcasecmp(pDbCacheInfo->dbFName, TSDB_PERFORMANCE_SCHEMA_DB)))) { int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode); - if (pDbVgVersion->vgVersion < vgVersion) { - usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); - - mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); - usedbRsp.vgVersion = vgVersion++; - } else { - usedbRsp.vgVersion = pDbVgVersion->vgVersion; + if (pDbCacheInfo->vgVersion >= vgVersion) { + continue; } - usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + + rsp.useDbRsp = taosMemoryCalloc(1, sizeof(SUseDbRsp)); + memcpy(rsp.useDbRsp->db, pDbCacheInfo->dbFName, TSDB_DB_FNAME_LEN); + rsp.useDbRsp->pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); + + mndBuildDBVgroupInfo(NULL, pMnode, rsp.useDbRsp->pVgroupInfos); + rsp.useDbRsp->vgVersion = vgVersion++; + + rsp.useDbRsp->vgNum = taosArrayGetSize(rsp.useDbRsp->pVgroupInfos); - taosArrayPush(batchUseRsp.pArray, &usedbRsp); + taosArrayPush(batchRsp.pArray, &rsp); continue; } - SDbObj *pDb = mndAcquireDb(pMnode, pDbVgVersion->dbFName); + SDbObj *pDb = mndAcquireDb(pMnode, pDbCacheInfo->dbFName); if (pDb == NULL) { - mTrace("db:%s, no exist", pDbVgVersion->dbFName); - memcpy(usedbRsp.db, pDbVgVersion->dbFName, TSDB_DB_FNAME_LEN); - usedbRsp.uid = pDbVgVersion->dbId; - usedbRsp.vgVersion = -1; - taosArrayPush(batchUseRsp.pArray, &usedbRsp); + mTrace("db:%s, no exist", pDbCacheInfo->dbFName); + rsp.useDbRsp = taosMemoryCalloc(1, sizeof(SUseDbRsp)); + memcpy(rsp.useDbRsp->db, pDbCacheInfo->dbFName, TSDB_DB_FNAME_LEN); + rsp.useDbRsp->uid = pDbCacheInfo->dbId; + rsp.useDbRsp->vgVersion = -1; + taosArrayPush(batchRsp.pArray, &rsp); continue; } int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); - if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable && - pDbVgVersion->stateTs == pDb->stateTs) { - mTrace("db:%s, valid dbinfo, vgVersion:%d stateTs:%" PRId64 - " numOfTables:%d, not changed vgVersion:%d stateTs:%" PRId64 " numOfTables:%d", - pDbVgVersion->dbFName, pDbVgVersion->vgVersion, pDbVgVersion->stateTs, pDbVgVersion->numOfTable, - pDb->vgVersion, pDb->stateTs, numOfTable); + if (pDbCacheInfo->vgVersion >= pDb->vgVersion && + pDbCacheInfo->cfgVersion >= pDb->cfgVersion && + numOfTable == pDbCacheInfo->numOfTable && + pDbCacheInfo->stateTs == pDb->stateTs) { + mTrace("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64 + " numOfTables:%d, not changed vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d", + pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs, pDbCacheInfo->numOfTable, + pDb->vgVersion, pDb->cfgVersion, pDb->stateTs, numOfTable); mndReleaseDb(pMnode, pDb); continue; } else { - mInfo("db:%s, valid dbinfo, vgVersion:%d stateTs:%" PRId64 - " numOfTables:%d, changed to vgVersion:%d stateTs:%" PRId64 " numOfTables:%d", - pDbVgVersion->dbFName, pDbVgVersion->vgVersion, pDbVgVersion->stateTs, pDbVgVersion->numOfTable, - pDb->vgVersion, pDb->stateTs, numOfTable); + mInfo("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64 + " numOfTables:%d, changed to vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d", + pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs, pDbCacheInfo->numOfTable, + pDb->vgVersion, pDb->cfgVersion, pDb->stateTs, numOfTable); } - usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); - if (usedbRsp.pVgroupInfos == NULL) { - mndReleaseDb(pMnode, pDb); - mError("db:%s, failed to malloc usedb response", pDb->name); - continue; + if (pDbCacheInfo->cfgVersion < pDb->cfgVersion) { + rsp.cfgRsp = taosMemoryCalloc(1, sizeof(SDbCfgRsp)); + mndDumpDbCfgInfo(rsp.cfgRsp, pDb); + } + + if (pDbCacheInfo->vgVersion < pDb->vgVersion || + numOfTable != pDbCacheInfo->numOfTable || + pDbCacheInfo->stateTs != pDb->stateTs) { + rsp.useDbRsp = taosMemoryCalloc(1, sizeof(SUseDbRsp)); + rsp.useDbRsp->pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); + if (rsp.useDbRsp->pVgroupInfos == NULL) { + mndReleaseDb(pMnode, pDb); + mError("db:%s, failed to malloc usedb response", pDb->name); + continue; + } + + mndBuildDBVgroupInfo(pDb, pMnode, rsp.useDbRsp->pVgroupInfos); + memcpy(rsp.useDbRsp->db, pDb->name, TSDB_DB_FNAME_LEN); + rsp.useDbRsp->uid = pDb->uid; + rsp.useDbRsp->vgVersion = pDb->vgVersion; + rsp.useDbRsp->stateTs = pDb->stateTs; + rsp.useDbRsp->vgNum = (int32_t)taosArrayGetSize(rsp.useDbRsp->pVgroupInfos); + rsp.useDbRsp->hashMethod = pDb->cfg.hashMethod; + rsp.useDbRsp->hashPrefix = pDb->cfg.hashPrefix; + rsp.useDbRsp->hashSuffix = pDb->cfg.hashSuffix; } - mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); - memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); - usedbRsp.uid = pDb->uid; - usedbRsp.vgVersion = pDb->vgVersion; - usedbRsp.stateTs = pDb->stateTs; - usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos); - usedbRsp.hashMethod = pDb->cfg.hashMethod; - usedbRsp.hashPrefix = pDb->cfg.hashPrefix; - usedbRsp.hashSuffix = pDb->cfg.hashSuffix; - - taosArrayPush(batchUseRsp.pArray, &usedbRsp); + taosArrayPush(batchRsp.pArray, &rsp); mndReleaseDb(pMnode, pDb); } - int32_t rspLen = tSerializeSUseDbBatchRsp(NULL, 0, &batchUseRsp); + int32_t rspLen = tSerializeSDbHbBatchRsp(NULL, 0, &batchRsp); void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tFreeSUseDbBatchRsp(&batchUseRsp); + tFreeSDbHbBatchRsp(&batchRsp); return -1; } - tSerializeSUseDbBatchRsp(pRsp, rspLen, &batchUseRsp); + tSerializeSDbHbBatchRsp(pRsp, rspLen, &batchRsp); *ppRsp = pRsp; *pRspLen = rspLen; - tFreeSUseDbBatchRsp(&batchUseRsp); + tFreeSDbHbBatchRsp(&batchRsp); return 0; } @@ -1517,6 +1604,13 @@ const char *mndGetDbStr(const char *src) { return pos; } +const char *mndGetStableStr(const char *src) { + char *pos = strstr(src, TS_PATH_DELIMITER); + if (pos != NULL) ++pos; + if (pos == NULL) return src; + return mndGetDbStr(pos); +} + static int64_t getValOfDiffPrecision(int8_t unit, int64_t val) { int64_t v = 0; switch (unit) { @@ -1779,12 +1873,6 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walRetentionSize, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walRollPeriod, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walSegmentSize, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.sstTrigger, false); diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 6dab0182362112d0ed5798f5829a1ad780a58768..3dab144eefcf92787b016062285990862fa3ebb5 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -30,11 +30,11 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->igExpired) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->fillHistory) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->triggerParam) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->watermark) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->conf.igExpired) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->conf.trigger) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->conf.fillHistory) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->conf.triggerParam) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->conf.watermark) < 0) return -1; if (tEncodeI64(pEncoder, pObj->sourceDbUid) < 0) return -1; if (tEncodeI64(pEncoder, pObj->targetDbUid) < 0) return -1; @@ -70,6 +70,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI32(pEncoder, innerSz) < 0) return -1; for (int32_t j = 0; j < innerSz; j++) { SStreamTask *pTask = taosArrayGetP(pArray, j); + pTask->ver = SSTREAM_TASK_VER; if (tEncodeStreamTask(pEncoder, pTask) < 0) return -1; } } @@ -97,11 +98,11 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->igExpired) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->fillHistory) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->triggerParam) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->watermark) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->conf.igExpired) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->conf.trigger) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->conf.fillHistory) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->conf.triggerParam) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->conf.watermark) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->sourceDbUid) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->targetDbUid) < 0) return -1; @@ -154,18 +155,10 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { return 0; } -void tFreeStreamObj(SStreamObj *pStream) { - taosMemoryFree(pStream->sql); - taosMemoryFree(pStream->ast); - taosMemoryFree(pStream->physicalPlan); - - if (pStream->outputSchema.nCols) { - taosMemoryFree(pStream->outputSchema.pSchema); - } - - int32_t sz = taosArrayGetSize(pStream->tasks); - for (int32_t i = 0; i < sz; i++) { - SArray *pLevel = taosArrayGetP(pStream->tasks, i); +static void *freeStreamTasks(SArray *pTaskLevel) { + int32_t numOfLevel = taosArrayGetSize(pTaskLevel); + for (int32_t i = 0; i < numOfLevel; i++) { + SArray *pLevel = taosArrayGetP(pTaskLevel, i); int32_t taskSz = taosArrayGetSize(pLevel); for (int32_t j = 0; j < taskSz; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); @@ -175,7 +168,20 @@ void tFreeStreamObj(SStreamObj *pStream) { taosArrayDestroy(pLevel); } - taosArrayDestroy(pStream->tasks); + return taosArrayDestroy(pTaskLevel); +} + +void tFreeStreamObj(SStreamObj *pStream) { + taosMemoryFree(pStream->sql); + taosMemoryFree(pStream->ast); + taosMemoryFree(pStream->physicalPlan); + + if (pStream->outputSchema.nCols || pStream->outputSchema.pSchema) { + taosMemoryFree(pStream->outputSchema.pSchema); + } + + pStream->tasks = freeStreamTasks(pStream->tasks); + pStream->pHTasksList = freeStreamTasks(pStream->pHTasksList); // tagSchema.pSchema if (pStream->tagSchema.nCols > 0) { @@ -187,14 +193,14 @@ SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); if (pVgEpNew == NULL) return NULL; pVgEpNew->vgId = pVgEp->vgId; - pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg); + // pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg); pVgEpNew->epSet = pVgEp->epSet; return pVgEpNew; } void tDeleteSMqVgEp(SMqVgEp *pVgEp) { if (pVgEp) { - taosMemoryFreeClear(pVgEp->qmsg); + // taosMemoryFreeClear(pVgEp->qmsg); taosMemoryFree(pVgEp); } } @@ -202,19 +208,23 @@ void tDeleteSMqVgEp(SMqVgEp *pVgEp) { int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgEp->vgId); - tlen += taosEncodeString(buf, pVgEp->qmsg); + // tlen += taosEncodeString(buf, pVgEp->qmsg); tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); return tlen; } -void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) { +void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp, int8_t sver) { buf = taosDecodeFixedI32(buf, &pVgEp->vgId); - buf = taosDecodeString(buf, &pVgEp->qmsg); + if (sver == 1) { + uint64_t size = 0; + buf = taosDecodeVariantU64(buf, &size); + buf = POINTER_SHIFT(buf, size); + } buf = taosDecodeSEpSet(buf, &pVgEp->epSet); return (void *)buf; } -SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]) { +SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup) { SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); if (pConsumer == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -245,16 +255,20 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_L return NULL; } - pConsumer->upTime = taosGetTimestampMs(); + pConsumer->createTime = taosGetTimestampMs(); return pConsumer; } -void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer) { +void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer, bool delete) { + if (pConsumer == NULL) return; taosArrayDestroyP(pConsumer->currentTopics, (FDelete)taosMemoryFree); taosArrayDestroyP(pConsumer->rebNewTopics, (FDelete)taosMemoryFree); taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree); taosArrayDestroyP(pConsumer->assignedTopics, (FDelete)taosMemoryFree); + if (delete) { + taosMemoryFree(pConsumer); + } } int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { @@ -269,7 +283,7 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { tlen += taosEncodeFixedI32(buf, pConsumer->pid); tlen += taosEncodeSEpSet(buf, &pConsumer->ep); - tlen += taosEncodeFixedI64(buf, pConsumer->upTime); + tlen += taosEncodeFixedI64(buf, pConsumer->createTime); tlen += taosEncodeFixedI64(buf, pConsumer->subscribeTime); tlen += taosEncodeFixedI64(buf, pConsumer->rebalanceTime); @@ -321,10 +335,14 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { tlen += taosEncodeFixedI32(buf, 0); } + tlen += taosEncodeFixedI8(buf, pConsumer->withTbName); + tlen += taosEncodeFixedI8(buf, pConsumer->autoCommit); + tlen += taosEncodeFixedI32(buf, pConsumer->autoCommitInterval); + tlen += taosEncodeFixedI32(buf, pConsumer->resetOffsetCfg); return tlen; } -void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) { +void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer, int8_t sver) { int32_t sz; buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); buf = taosDecodeStringTo(buf, pConsumer->clientId); @@ -335,7 +353,7 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) { buf = taosDecodeFixedI32(buf, &pConsumer->pid); buf = taosDecodeSEpSet(buf, &pConsumer->ep); - buf = taosDecodeFixedI64(buf, &pConsumer->upTime); + buf = taosDecodeFixedI64(buf, &pConsumer->createTime); buf = taosDecodeFixedI64(buf, &pConsumer->subscribeTime); buf = taosDecodeFixedI64(buf, &pConsumer->rebalanceTime); @@ -375,55 +393,99 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) { taosArrayPush(pConsumer->assignedTopics, &topic); } + if (sver > 1) { + buf = taosDecodeFixedI8(buf, &pConsumer->withTbName); + buf = taosDecodeFixedI8(buf, &pConsumer->autoCommit); + buf = taosDecodeFixedI32(buf, &pConsumer->autoCommitInterval); + buf = taosDecodeFixedI32(buf, &pConsumer->resetOffsetCfg); + } return (void *)buf; } -SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) { - SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp)); - if (pConsumerEpNew == NULL) return NULL; - pConsumerEpNew->consumerId = pConsumerEpOld->consumerId; - pConsumerEpNew->vgs = taosArrayDup(pConsumerEpOld->vgs, (__array_item_dup_fn_t)tCloneSMqVgEp); - return pConsumerEpNew; -} - -void tDeleteSMqConsumerEp(void *data) { - SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)data; - taosArrayDestroyP(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp); -} +// SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) { +// SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp)); +// if (pConsumerEpNew == NULL) return NULL; +// pConsumerEpNew->consumerId = pConsumerEpOld->consumerId; +// pConsumerEpNew->vgs = taosArrayDup(pConsumerEpOld->vgs, NULL); +// return pConsumerEpNew; +// } +// +// void tDeleteSMqConsumerEp(void *data) { +// SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)data; +// taosArrayDestroy(pConsumerEp->vgs); +// } int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp); -#if 0 - int32_t sz = taosArrayGetSize(pConsumerEp->vgs); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); - tlen += tEncodeSMqVgEp(buf, pVgEp); + int32_t szVgs = taosArrayGetSize(pConsumerEp->offsetRows); + tlen += taosEncodeFixedI32(buf, szVgs); + for (int32_t j = 0; j < szVgs; ++j) { + OffsetRows *offRows = taosArrayGet(pConsumerEp->offsetRows, j); + tlen += taosEncodeFixedI32(buf, offRows->vgId); + tlen += taosEncodeFixedI64(buf, offRows->rows); + tlen += taosEncodeFixedI8(buf, offRows->offset.type); + if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) { + tlen += taosEncodeFixedI64(buf, offRows->offset.uid); + tlen += taosEncodeFixedI64(buf, offRows->offset.ts); + } else if (offRows->offset.type == TMQ_OFFSET__LOG) { + tlen += taosEncodeFixedI64(buf, offRows->offset.version); + } else { + // do nothing + } } -#endif + // #if 0 + // int32_t sz = taosArrayGetSize(pConsumerEp->vgs); + // tlen += taosEncodeFixedI32(buf, sz); + // for (int32_t i = 0; i < sz; i++) { + // SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); + // tlen += tEncodeSMqVgEp(buf, pVgEp); + // } + // #endif return tlen; } -void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp) { +void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t sver) { buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); - buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp)); -#if 0 - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pConsumerEp->vgs = taosArrayInit(sz, sizeof(void *)); - for (int32_t i = 0; i < sz; i++) { - SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); - buf = tDecodeSMqVgEp(buf, pVgEp); - taosArrayPush(pConsumerEp->vgs, &pVgEp); + buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp), sver); + if (sver > 1) { + int32_t szVgs = 0; + buf = taosDecodeFixedI32(buf, &szVgs); + if (szVgs > 0) { + pConsumerEp->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows)); + if (NULL == pConsumerEp->offsetRows) return NULL; + for (int32_t j = 0; j < szVgs; ++j) { + OffsetRows *offRows = taosArrayReserve(pConsumerEp->offsetRows, 1); + buf = taosDecodeFixedI32(buf, &offRows->vgId); + buf = taosDecodeFixedI64(buf, &offRows->rows); + buf = taosDecodeFixedI8(buf, &offRows->offset.type); + if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) { + buf = taosDecodeFixedI64(buf, &offRows->offset.uid); + buf = taosDecodeFixedI64(buf, &offRows->offset.ts); + } else if (offRows->offset.type == TMQ_OFFSET__LOG) { + buf = taosDecodeFixedI64(buf, &offRows->offset.version); + } else { + // do nothing + } + } + } } -#endif + // #if 0 + // int32_t sz; + // buf = taosDecodeFixedI32(buf, &sz); + // pConsumerEp->vgs = taosArrayInit(sz, sizeof(void *)); + // for (int32_t i = 0; i < sz; i++) { + // SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); + // buf = tDecodeSMqVgEp(buf, pVgEp); + // taosArrayPush(pConsumerEp->vgs, &pVgEp); + // } + // #endif return (void *)buf; } -SMqSubscribeObj *tNewSubscribeObj(const char* key) { +SMqSubscribeObj *tNewSubscribeObj(const char *key) { SMqSubscribeObj *pSubObj = taosMemoryCalloc(1, sizeof(SMqSubscribeObj)); if (pSubObj == NULL) { return NULL; @@ -468,7 +530,9 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEp)); } pSubNew->unassignedVgs = taosArrayDup(pSub->unassignedVgs, (__array_item_dup_fn_t)tCloneSMqVgEp); + pSubNew->offsetRows = taosArrayDup(pSub->offsetRows, NULL); memcpy(pSubNew->dbName, pSub->dbName, TSDB_DB_FNAME_LEN); + pSubNew->qmsg = taosStrdup(pSub->qmsg); return pSubNew; } @@ -479,9 +543,12 @@ void tDeleteSubscribeObj(SMqSubscribeObj *pSub) { if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; taosArrayDestroyP(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp); + taosArrayDestroy(pConsumerEp->offsetRows); } taosHashCleanup(pSub->consumerHash); taosArrayDestroyP(pSub->unassignedVgs, (FDelete)tDeleteSMqVgEp); + taosMemoryFreeClear(pSub->qmsg); + taosArrayDestroy(pSub->offsetRows); } int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { @@ -508,10 +575,28 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { if (cnt != sz) return -1; tlen += taosEncodeArray(buf, pSub->unassignedVgs, (FEncode)tEncodeSMqVgEp); tlen += taosEncodeString(buf, pSub->dbName); + + int32_t szVgs = taosArrayGetSize(pSub->offsetRows); + tlen += taosEncodeFixedI32(buf, szVgs); + for (int32_t j = 0; j < szVgs; ++j) { + OffsetRows *offRows = taosArrayGet(pSub->offsetRows, j); + tlen += taosEncodeFixedI32(buf, offRows->vgId); + tlen += taosEncodeFixedI64(buf, offRows->rows); + tlen += taosEncodeFixedI8(buf, offRows->offset.type); + if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) { + tlen += taosEncodeFixedI64(buf, offRows->offset.uid); + tlen += taosEncodeFixedI64(buf, offRows->offset.ts); + } else if (offRows->offset.type == TMQ_OFFSET__LOG) { + tlen += taosEncodeFixedI64(buf, offRows->offset.version); + } else { + // do nothing + } + } + tlen += taosEncodeString(buf, pSub->qmsg); return tlen; } -void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) { +void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub, int8_t sver) { // buf = taosDecodeStringTo(buf, pSub->key); buf = taosDecodeFixedI64(buf, &pSub->dbUid); @@ -526,74 +611,100 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) { pSub->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); for (int32_t i = 0; i < sz; i++) { SMqConsumerEp consumerEp = {0}; - buf = tDecodeSMqConsumerEp(buf, &consumerEp); + buf = tDecodeSMqConsumerEp(buf, &consumerEp, sver); taosHashPut(pSub->consumerHash, &consumerEp.consumerId, sizeof(int64_t), &consumerEp, sizeof(SMqConsumerEp)); } - buf = taosDecodeArray(buf, &pSub->unassignedVgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp)); + buf = taosDecodeArray(buf, &pSub->unassignedVgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp), sver); buf = taosDecodeStringTo(buf, pSub->dbName); - return (void *)buf; -} - -SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { - SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry)); - if (pEntryNew == NULL) return NULL; - pEntryNew->epoch = pEntry->epoch; - pEntryNew->consumers = taosArrayDup(pEntry->consumers, (__array_item_dup_fn_t)tCloneSMqConsumerEp); - return pEntryNew; -} - -void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { - taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEp); -} - -int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) { - int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pEntry->epoch); - tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry); - return tlen; -} - -void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) { - buf = taosDecodeFixedI32(buf, &pEntry->epoch); - buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); - return (void *)buf; -} - -SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) { - SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj)); - if (pLogNew == NULL) return pLogNew; - memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN); - pLogNew->logs = taosArrayDup(pLog->logs, (__array_item_dup_fn_t)tCloneSMqConsumerEp); - return pLogNew; -} - -void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) { - taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEp); -} -int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pLog->key); - tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry); - return tlen; -} - -void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) { - buf = taosDecodeStringTo(buf, pLog->key); - buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); + if (sver > 1) { + int32_t szVgs = 0; + buf = taosDecodeFixedI32(buf, &szVgs); + if (szVgs > 0) { + pSub->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows)); + if (NULL == pSub->offsetRows) return NULL; + for (int32_t j = 0; j < szVgs; ++j) { + OffsetRows *offRows = taosArrayReserve(pSub->offsetRows, 1); + buf = taosDecodeFixedI32(buf, &offRows->vgId); + buf = taosDecodeFixedI64(buf, &offRows->rows); + buf = taosDecodeFixedI8(buf, &offRows->offset.type); + if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) { + buf = taosDecodeFixedI64(buf, &offRows->offset.uid); + buf = taosDecodeFixedI64(buf, &offRows->offset.ts); + } else if (offRows->offset.type == TMQ_OFFSET__LOG) { + buf = taosDecodeFixedI64(buf, &offRows->offset.version); + } else { + // do nothing + } + } + } + buf = taosDecodeString(buf, &pSub->qmsg); + } else { + pSub->qmsg = taosStrdup(""); + } return (void *)buf; } -int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pOffset->key); - tlen += taosEncodeFixedI64(buf, pOffset->offset); - return tlen; -} - -void *tDecodeSMqOffsetObj(void *buf, SMqOffsetObj *pOffset) { - buf = taosDecodeStringTo(buf, pOffset->key); - buf = taosDecodeFixedI64(buf, &pOffset->offset); - return buf; -} +// SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { +// SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry)); +// if (pEntryNew == NULL) return NULL; +// pEntryNew->epoch = pEntry->epoch; +// pEntryNew->consumers = taosArrayDup(pEntry->consumers, (__array_item_dup_fn_t)tCloneSMqConsumerEp); +// return pEntryNew; +// } +// +// void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { +// taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEp); +// } + +// int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) { +// int32_t tlen = 0; +// tlen += taosEncodeFixedI32(buf, pEntry->epoch); +// tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry); +// return tlen; +// } +// +// void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) { +// buf = taosDecodeFixedI32(buf, &pEntry->epoch); +// buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); +// return (void *)buf; +// } + +// SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) { +// SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj)); +// if (pLogNew == NULL) return pLogNew; +// memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN); +// pLogNew->logs = taosArrayDup(pLog->logs, (__array_item_dup_fn_t)tCloneSMqConsumerEp); +// return pLogNew; +// } +// +// void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) { +// taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEp); +// } + +// int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) { +// int32_t tlen = 0; +// tlen += taosEncodeString(buf, pLog->key); +// tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry); +// return tlen; +// } +// +// void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) { +// buf = taosDecodeStringTo(buf, pLog->key); +// buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); +// return (void *)buf; +// } +// +// int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) { +// int32_t tlen = 0; +// tlen += taosEncodeString(buf, pOffset->key); +// tlen += taosEncodeFixedI64(buf, pOffset->offset); +// return tlen; +// } +// +// void *tDecodeSMqOffsetObj(void *buf, SMqOffsetObj *pOffset) { +// buf = taosDecodeStringTo(buf, pOffset->key); +// buf = taosDecodeFixedI64(buf, &pOffset->offset); +// return buf; +// } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d4cbcaaacda393b52df4d9fc27d10de55be7ae03..234e81a670e44f0606bdd9163326220278e9ebd3 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -41,9 +41,15 @@ static const char *offlineReason[] = { "timezone not match", "locale not match", "charset not match", + "ttlChangeOnWrite not match", "unknown", }; +enum { + DND_ACTIVE_CODE, + DND_CONN_ACTIVE_CODE, +}; + static int32_t mndCreateDefaultDnode(SMnode *pMnode); static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode); static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw); @@ -65,6 +71,8 @@ static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); +static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t opLen, int32_t *pOutValue); + int32_t mndInitDnode(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_DNODE, @@ -227,6 +235,14 @@ static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode) { static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew) { mTrace("dnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); pOld->updateTime = pNew->updateTime; +#ifdef TD_ENTERPRISE + if (strncmp(pOld->active, pNew->active, TSDB_ACTIVE_KEY_LEN) != 0) { + strncpy(pOld->active, pNew->active, TSDB_ACTIVE_KEY_LEN); + } + if (strncmp(pOld->connActive, pNew->connActive, TSDB_CONN_ACTIVE_KEY_LEN) != 0) { + strncpy(pOld->connActive, pNew->connActive, TSDB_CONN_ACTIVE_KEY_LEN); + } +#endif return 0; } @@ -399,6 +415,12 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const S return DND_REASON_CHARSET_NOT_MATCH; } + if (pCfg->ttlChangeOnWrite != tsTtlChangeOnWrite) { + mError("dnode:%d, ttlChangeOnWrite:%d inconsistent with cluster:%d", pDnode->id, pCfg->ttlChangeOnWrite, + tsTtlChangeOnWrite); + return DND_REASON_TTL_CHANGE_ON_WRITE_NOT_MATCH; + } + return 0; } @@ -509,13 +531,23 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SMnodeObj *pObj = mndAcquireMnode(pMnode, pDnode->id); if (pObj != NULL) { - if (pObj->syncState != statusReq.mload.syncState || pObj->syncRestore != statusReq.mload.syncRestore) { - mInfo("dnode:%d, mnode syncState from %s to %s, restoreState from %d to %d", pObj->id, syncStr(pObj->syncState), - syncStr(statusReq.mload.syncState), pObj->syncRestore, statusReq.mload.syncRestore); + bool roleChanged = pObj->syncState != statusReq.mload.syncState || + (statusReq.mload.syncTerm != -1 && pObj->syncTerm != statusReq.mload.syncTerm); + bool restoreChanged = pObj->syncRestore != statusReq.mload.syncRestore; + if (roleChanged || restoreChanged) { + mInfo("dnode:%d, mnode syncState from %s to %s, restoreState from %d to %d, syncTerm from %" PRId64 + " to %" PRId64, + pObj->id, syncStr(pObj->syncState), syncStr(statusReq.mload.syncState), pObj->syncRestore, + statusReq.mload.syncRestore, pObj->syncTerm, statusReq.mload.syncTerm); pObj->syncState = statusReq.mload.syncState; pObj->syncRestore = statusReq.mload.syncRestore; - pObj->stateStartTime = taosGetTimestampMs(); + pObj->syncTerm = statusReq.mload.syncTerm; + } + + if (roleChanged) { + pObj->roleTimeMs = (statusReq.mload.roleTimeMs != 0) ? statusReq.mload.roleTimeMs : taosGetTimestampMs(); } + mndReleaseMnode(pMnode, pObj); } @@ -619,7 +651,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq, "create-dnode"); if (pTrans == NULL) goto _OVER; mInfo("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; pRaw = mndDnodeActionEncode(&dnodeObj); if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; @@ -635,6 +667,72 @@ _OVER: return code; } +static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfgReq, int8_t action) { + SSdbRaw *pRaw = NULL; + STrans *pTrans = NULL; + SDnodeObj *pDnode = NULL; + bool cfgAll = pCfgReq->dnodeId == -1; + int32_t iter = 0; + + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + while (1) { + if (cfgAll) { + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); + if (pIter == NULL) break; + ++iter; + } else if (!(pDnode = mndAcquireDnode(pMnode, pCfgReq->dnodeId))) { + goto _OVER; + } + + if (!pTrans) { + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "config-dnode"); + if (!pTrans) goto _OVER; + if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + } + + SDnodeObj tmpDnode = *pDnode; + if (action == DND_ACTIVE_CODE) { + strncpy(tmpDnode.active, pCfgReq->value, TSDB_ACTIVE_KEY_LEN); + } else if (action == DND_CONN_ACTIVE_CODE) { + strncpy(tmpDnode.connActive, pCfgReq->value, TSDB_CONN_ACTIVE_KEY_LEN); + } else { + terrno = TSDB_CODE_INVALID_CFG; + goto _OVER; + } + + pRaw = mndDnodeActionEncode(&tmpDnode); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + + mInfo("dnode:%d, config dnode, cfg:%d, app:%p config:%s value:%s", pDnode->id, pCfgReq->dnodeId, pReq->info.ahandle, + pCfgReq->config, pCfgReq->value); + + if (cfgAll) { + sdbRelease(pSdb, pDnode); + pDnode = NULL; + } else { + break; + } + } + + if (pTrans && mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + tsGrantHBInterval = TMIN(TMAX(5, iter / 2), 30); + terrno = 0; + +_OVER: + if (cfgAll) { + sdbRelease(pSdb, pDnode); + } else { + mndReleaseDnode(pMnode, pDnode); + } + sdbCancelFetch(pSdb, pIter); + mndTransDrop(pTrans); + sdbFreeRaw(pRaw); + return terrno; +} + static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -707,18 +805,22 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { strcpy(info.name, "statusInterval"); snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsStatusInterval); + strcpy(info.scope, "server"); taosArrayPush(rsp.variables, &info); strcpy(info.name, "timezone"); snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsTimezoneStr); + strcpy(info.scope, "both"); taosArrayPush(rsp.variables, &info); strcpy(info.name, "locale"); snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsLocale); + strcpy(info.scope, "both"); taosArrayPush(rsp.variables, &info); strcpy(info.name, "charset"); snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsCharset); + strcpy(info.scope, "both"); taosArrayPush(rsp.variables, &info); int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp); @@ -781,7 +883,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { code = mndCreateDnode(pMnode, pReq, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; - + tsGrantHBInterval = 5; _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, terrstr()); @@ -804,7 +906,7 @@ int32_t mndProcessRestoreDnodeReqImpl(SRpcMsg *pReq){ #endif static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMnodeObj *pMObj, SQnodeObj *pQObj, - SSnodeObj *pSObj, int32_t numOfVnodes, bool force) { + SSnodeObj *pSObj, int32_t numOfVnodes, bool force, bool unsafe) { int32_t code = -1; SSdbRaw *pRaw = NULL; STrans *pTrans = NULL; @@ -813,7 +915,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to drop dnode:%d, force:%d", pTrans->id, pDnode->id, force); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; pRaw = mndDnodeActionEncode(pDnode); if (pRaw == NULL) goto _OVER; @@ -844,7 +946,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM if (numOfVnodes > 0) { mInfo("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id); - if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id, force) != 0) goto _OVER; + if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id, force, unsafe) != 0) goto _OVER; } if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -871,11 +973,18 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { goto _OVER; } - mInfo("dnode:%d, start to drop, ep:%s:%d", dropReq.dnodeId, dropReq.fqdn, dropReq.port); + mInfo("dnode:%d, start to drop, ep:%s:%d, force:%s, unsafe:%s", + dropReq.dnodeId, dropReq.fqdn, dropReq.port, dropReq.force?"true":"false", dropReq.unsafe?"true":"false"); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { goto _OVER; } + bool force = dropReq.force; + if(dropReq.unsafe) + { + force = true; + } + pDnode = mndAcquireDnode(pMnode, dropReq.dnodeId); if (pDnode == NULL) { int32_t err = terrno; @@ -903,16 +1012,23 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); - if ((numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) && !dropReq.force) { - if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { - terrno = TSDB_CODE_DNODE_OFFLINE; - mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), - numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); - goto _OVER; - } + bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs()); + + if (isonline && force) { + terrno = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE; + mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), + numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); + goto _OVER; } - code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes, dropReq.force); + if (!isonline && !force) { + terrno = TSDB_CODE_DNODE_OFFLINE; + mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), + numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); + goto _OVER; + } + + code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes, force, dropReq.unsafe); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: @@ -970,6 +1086,48 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { strcpy(dcfgReq.config, "monitor"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); + } else if (strncasecmp(cfgReq.config, "keeptimeoffset", 14) == 0) { + int32_t optLen = strlen("keeptimeoffset"); + int32_t flag = -1; + int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); + if (code < 0) return code; + + if (flag < 0 || flag > 23) { + mError("dnode:%d, failed to config keepTimeOffset since value:%d. Valid range: [0, 23]", cfgReq.dnodeId, flag); + terrno = TSDB_CODE_INVALID_CFG; + return -1; + } + + strcpy(dcfgReq.config, "keeptimeoffset"); + snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); +#ifdef TD_ENTERPRISE + } else if (strncasecmp(cfgReq.config, "activeCode", 10) == 0 || strncasecmp(cfgReq.config, "cActiveCode", 11) == 0) { + int8_t opt = strncasecmp(cfgReq.config, "a", 1) == 0 ? DND_ACTIVE_CODE : DND_CONN_ACTIVE_CODE; + int8_t index = opt == DND_ACTIVE_CODE ? 10 : 11; + if (' ' != cfgReq.config[index] && 0 != cfgReq.config[index]) { + mError("dnode:%d, failed to config activeCode since invalid conf:%s", cfgReq.dnodeId, cfgReq.config); + terrno = TSDB_CODE_INVALID_CFG; + return -1; + } + int32_t vlen = strlen(cfgReq.value); + if (vlen > 0 && ((opt == DND_ACTIVE_CODE && vlen != (TSDB_ACTIVE_KEY_LEN - 1)) || + (opt == DND_CONN_ACTIVE_CODE && + (vlen > (TSDB_CONN_ACTIVE_KEY_LEN - 1) || vlen < (TSDB_ACTIVE_KEY_LEN - 1))))) { + mError("dnode:%d, failed to config activeCode since invalid vlen:%d. conf:%s, val:%s", cfgReq.dnodeId, vlen, + cfgReq.config, cfgReq.value); + terrno = TSDB_CODE_INVALID_OPTION; + return -1; + } + + strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? "activeCode" : "cActiveCode"); + snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%s", cfgReq.value); + + if (mndConfigDnode(pMnode, pReq, &cfgReq, opt) != 0) { + mError("dnode:%d, failed to config activeCode since %s", cfgReq.dnodeId, terrstr()); + return -1; + } + return 0; +#endif } else { bool findOpt = false; for (int32_t d = 0; d < optionSize; ++d) { @@ -1023,7 +1181,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq); mInfo("dnode:%d, send config req to dnode, app:%p config:%s value:%s", cfgReq.dnodeId, pReq->info.ahandle, dcfgReq.config, dcfgReq.value); - SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen, .info = pReq->info}; + SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; tmsgSendReq(&epSet, &rpcMsg); code = 0; } @@ -1174,3 +1332,28 @@ static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +// get int32_t value from 'SMCfgDnodeReq' +static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t opLen, int32_t *pOutValue) { + terrno = 0; + if (' ' != pMCfgReq->config[opLen] && 0 != pMCfgReq->config[opLen]) { + goto _err; + } + + if (' ' == pMCfgReq->config[opLen]) { + // 'key value' + if (strlen(pMCfgReq->value) != 0) goto _err; + *pOutValue = atoi(pMCfgReq->config + opLen + 1); + } else { + // 'key' 'value' + if (strlen(pMCfgReq->value) == 0) goto _err; + *pOutValue = atoi(pMCfgReq->value); + } + + return 0; + +_err: + mError("dnode:%d, failed to config keeptimeoffset since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); + terrno = TSDB_CODE_INVALID_CFG; + return -1; +} diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index a991bddda8a997c6074ee21e811f3829e0370fe8..62b5cb00e6abd1ff8fd0a567a7679f825f6782c1 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -285,6 +285,7 @@ void dumpTopic(SSdb *pSdb, SJson *json) { tjsonAddStringToObject(item, "subType", i642str(pObj->subType)); tjsonAddStringToObject(item, "withMeta", i642str(pObj->withMeta)); tjsonAddStringToObject(item, "stbUid", i642str(pObj->stbUid)); + tjsonAddStringToObject(item, "stbName", mndGetStableStr(pObj->stbName)); tjsonAddStringToObject(item, "sqlLen", i642str(pObj->sqlLen)); tjsonAddStringToObject(item, "astLen", i642str(pObj->astLen)); tjsonAddStringToObject(item, "sqlLen", i642str(pObj->sqlLen)); @@ -366,10 +367,10 @@ void dumpStream(SSdb *pSdb, SJson *json) { tjsonAddStringToObject(item, "smaId", i642str(pObj->smaId)); tjsonAddStringToObject(item, "uid", i642str(pObj->uid)); tjsonAddStringToObject(item, "status", i642str(pObj->status)); - tjsonAddStringToObject(item, "igExpired", i642str(pObj->igExpired)); - tjsonAddStringToObject(item, "trigger", i642str(pObj->trigger)); - tjsonAddStringToObject(item, "triggerParam", i642str(pObj->triggerParam)); - tjsonAddStringToObject(item, "watermark", i642str(pObj->watermark)); + tjsonAddStringToObject(item, "igExpired", i642str(pObj->conf.igExpired)); + tjsonAddStringToObject(item, "trigger", i642str(pObj->conf.trigger)); + tjsonAddStringToObject(item, "triggerParam", i642str(pObj->conf.triggerParam)); + tjsonAddStringToObject(item, "watermark", i642str(pObj->conf.watermark)); tjsonAddStringToObject(item, "sourceDbUid", i642str(pObj->sourceDbUid)); tjsonAddStringToObject(item, "targetDbUid", i642str(pObj->targetDbUid)); tjsonAddStringToObject(item, "sourceDb", mndGetDbStr(pObj->sourceDb)); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index af9bedd220e3ad3769e9219f2686d2da9884c9c7..4ffc7a20c26890451576a68e1f50572bd5a6db15 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -562,7 +562,7 @@ static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int32_t le return msg; } - if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_GEOMETRY) { int32_t bytes = len > 0 ? (int32_t)(len - VARSTR_HEADER_SIZE) : len; snprintf(buf, buflen - 1, "%s(%d)", tDataTypes[type].name, type == TSDB_DATA_TYPE_NCHAR ? bytes / 4 : bytes); diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index 83172acf641608427a045623b835b9300901ad2c..2157804559bb54e552e1508a27f37737696953c7 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -515,7 +515,6 @@ int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, i if (pDb == NULL) return 0; } SSmaAndTagIter *pIter = pShow->pIter; - int invalid = -1; while (numOfRows < rows) { pIter->pIdxIter = sdbFetch(pSdb, SDB_IDX, pIter->pIdxIter, (void **)&pIdx); if (pIter->pIdxIter == NULL) break; @@ -542,32 +541,32 @@ int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, i STR_TO_VARSTR(n3, (char *)tNameGetTableName(&stbName)); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)n1, false); + colDataSetVal(pColInfo, numOfRows, (const char *)n1, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)n2, false); + colDataSetVal(pColInfo, numOfRows, (const char *)n2, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)n3, false); + colDataSetVal(pColInfo, numOfRows, (const char *)n3, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&invalid, false); + colDataSetVal(pColInfo, numOfRows, NULL, true); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pIdx->createdTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pIdx->createdTime, false); char col[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(col, (char *)pIdx->colName); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)col, false); + colDataSetVal(pColInfo, numOfRows, (const char *)col, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); char tag[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(tag, (char *)"tag_index"); - colDataAppend(pColInfo, numOfRows, (const char *)tag, false); + colDataSetVal(pColInfo, numOfRows, (const char *)tag, false); numOfRows++; sdbRelease(pSdb, pIdx); @@ -645,7 +644,7 @@ int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pSt // mInfo("trans:%d, used to add index to stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; mndTransSetSerial(pTrans); @@ -721,7 +720,7 @@ static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *p mInfo("trans:%d, used to drop idx:%s", pTrans->id, pIdx->name); mndTransSetDbName(pTrans, pDb->name, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; mndTransSetSerial(pTrans); if (mndSetDropIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; @@ -831,6 +830,7 @@ int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxOb if (pIdx->stbUid == pStb->uid && strcasecmp(pIdx->colName, tagName) == 0) { memcpy((char *)idx, (char *)pIdx, sizeof(SIdxObj)); sdbRelease(pSdb, pIdx); + sdbCancelFetch(pSdb, pIter); return 0; } @@ -851,7 +851,7 @@ int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { if (pIdx->dbUid == pDb->uid) { if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) { sdbRelease(pSdb, pIdx); - sdbCancelFetch(pSdb, pIdx); + sdbCancelFetch(pSdb, pIter); return -1; } } @@ -860,4 +860,4 @@ int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { } return 0; -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 381b1e64ed97080a38b3e45e53fe74c18ea3dc15..1071a6cf6ebbe34cf6cd1873f1180ede6c113219 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -804,7 +804,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr if (pObj->id == pMnode->selfDnodeId) { pClusterInfo->first_ep_dnode_id = pObj->id; tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep)); - pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode); + pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f; // pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f); tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role)); } else { @@ -890,7 +890,10 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { SSyncState state = syncGetState(pMnode->syncMgmt.sync); pLoad->syncState = state.state; pLoad->syncRestore = state.restored; - mTrace("mnode current syncState is %s, syncRestore:%d", syncStr(pLoad->syncState), pLoad->syncRestore); + pLoad->syncTerm = state.term; + pLoad->roleTimeMs = state.roleTimeMs; + mTrace("mnode current syncState is %s, syncRestore:%d, syncTerm:%" PRId64 " ,roleTimeMs:%" PRId64, + syncStr(pLoad->syncState), pLoad->syncRestore, pLoad->syncTerm, pLoad->roleTimeMs); return 0; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 19c3d59167fd86cc98f1a66159ee97ede21313f8..4ee2bc159b4697564a0614d8ff72d8235e071e98 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -319,7 +319,7 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p return 0; } -static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, +static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeTypeReq *pAlterMnodeTypeReq, SEpSet *pAlterMnodeTypeEpSet) { int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq); void *pReq = taosMemoryMalloc(contLen); @@ -578,7 +578,7 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; SMnodeObj mnodeObj = {0}; mnodeObj.id = pDnode->id; @@ -695,7 +695,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode if (totalMnodes == 2) { if (force) { mError("cant't force drop dnode, since a mnode on it and replica is 2"); - terrno = TSDB_CODE_DNODE_OFFLINE; + terrno = TSDB_CODE_MNODE_ONLY_TWO_MNODE; return -1; } mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes); @@ -732,7 +732,7 @@ static int32_t mndDropMnode(SMnode *pMnode, SRpcMsg *pReq, SMnodeObj *pObj) { if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -803,9 +803,17 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB int32_t numOfRows = 0; int32_t cols = 0; SMnodeObj *pObj = NULL; + SMnodeObj *pSelfObj = NULL; ESdbStatus objStatus = 0; char *pWrite; int64_t curMs = taosGetTimestampMs(); + int64_t dummyTimeMs = 0; + + pSelfObj = sdbAcquire(pSdb, SDB_MNODE, &pMnode->selfDnodeId); + if (pSelfObj == NULL) { + mError("mnode:%d, failed to acquire self %s", pMnode->selfDnodeId, terrstr()); + goto _out; + } while (numOfRows < rows) { pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus, true); @@ -825,7 +833,8 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB if (pObj->id == pMnode->selfDnodeId) { snprintf(role, sizeof(role), "%s%s", syncStr(TAOS_SYNC_STATE_LEADER), pMnode->restored ? "" : "*"); } - if (mndIsDnodeOnline(pObj->pDnode, curMs)) { + bool isDnodeOnline = mndIsDnodeOnline(pObj->pDnode, curMs); + if (isDnodeOnline) { tstrncpy(role, syncStr(pObj->syncState), sizeof(role)); if (pObj->syncState == TAOS_SYNC_STATE_LEADER && pObj->id != pMnode->selfDnodeId) { tstrncpy(role, syncStr(TAOS_SYNC_STATE_ERROR), sizeof(role)); @@ -840,7 +849,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB const char *status = "ready"; if (objStatus == SDB_STATUS_CREATING) status = "creating"; if (objStatus == SDB_STATUS_DROPPING) status = "dropping"; - if (!mndIsDnodeOnline(pObj->pDnode, curMs)) status = "offline"; + if (!isDnodeOnline) status = "offline"; char b3[9 + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(b3, status, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -850,7 +859,15 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->stateStartTime, false); + if (pObj->syncTerm != pSelfObj->syncTerm || !isDnodeOnline) { + // state of old term / no status report => use dummyTimeMs + if (pObj->syncTerm > pSelfObj->syncTerm) { + mError("mnode:%d has a newer term:%" PRId64 " than me:%" PRId64, pObj->id, pObj->syncTerm, pSelfObj->syncTerm); + } + colDataSetVal(pColInfo, numOfRows, (const char *)&dummyTimeMs, false); + } else { + colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->roleTimeMs, false); + } numOfRows++; sdbRelease(pSdb, pObj); @@ -858,6 +875,8 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pShow->numOfRows += numOfRows; +_out: + sdbRelease(pSdb, pSelfObj); return numOfRows; } @@ -999,12 +1018,12 @@ static void mndReloadSyncConfig(SMnode *pMnode) { } if (pMnode->syncMgmt.sync > 0) { - mInfo("vgId:1, mnode sync reconfig, totalReplica:%d replica:%d myIndex:%d", + mInfo("vgId:1, mnode sync reconfig, totalReplica:%d replica:%d myIndex:%d", cfg.totalReplicaNum, cfg.replicaNum, cfg.myIndex); for (int32_t i = 0; i < cfg.totalReplicaNum; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; - mInfo("vgId:1, index:%d, ep:%s:%u dnode:%d cluster:%" PRId64 " role:%d", i, pNode->nodeFqdn, pNode->nodePort, + mInfo("vgId:1, index:%d, ep:%s:%u dnode:%d cluster:%" PRId64 " role:%d", i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId, pNode->nodeRole); } diff --git a/source/dnode/mnode/impl/src/mndPrivilege.c b/source/dnode/mnode/impl/src/mndPrivilege.c index de0374c6e89c506ce1b9cad966e2a4dd8905b7b7..bec516b1ee4a5370c672ae392e964222b4634a1a 100644 --- a/source/dnode/mnode/impl/src/mndPrivilege.c +++ b/source/dnode/mnode/impl/src/mndPrivilege.c @@ -36,7 +36,9 @@ int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); pRsp->superAuth = 1; pRsp->enable = pUser->enable; + pRsp->sysInfo = pUser->sysInfo; pRsp->version = pUser->authVersion; + pRsp->passVer = pUser->passVersion; return 0; } #endif \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index d0f88940a95ad87ba3528c9472811c83cd8f6eb1..524ea1a06bdec94d6f77b9fcb8dcb524b9cdc0a9 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -227,12 +227,12 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { } if ((code = taosCheckVersionCompatibleFromStr(connReq.sVer, version, 3)) != 0) { + mGError("version not compatible. client version: %s, server version: %s", connReq.sVer, version); terrno = code; goto _OVER; } code = -1; - taosIp2String(pReq->info.conn.clientIp, ip); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) { mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr()); @@ -245,7 +245,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto _OVER; } - if (strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1) != 0) { + if (strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1) != 0 && !tsMndSkipGrant) { mGError("user:%s, failed to login from %s since invalid pass, input:%s", pReq->info.conn.user, ip, connReq.passwd); code = TSDB_CODE_MND_AUTH_FAILURE; goto _OVER; @@ -270,6 +270,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { } } +_CONNECT: pConn = mndCreateConn(pMnode, pReq->info.conn.user, connReq.connType, pReq->info.conn.clientIp, pReq->info.conn.clientPort, connReq.pid, connReq.app, connReq.startTime); if (pConn == NULL) { @@ -287,6 +288,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.dnodeNum = mndGetDnodeSize(pMnode); connectRsp.svrTimestamp = taosGetTimestampSec(); connectRsp.passVer = pUser->passVersion; + connectRsp.authVer = pUser->authVersion; strcpy(connectRsp.sVer, version); snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, @@ -534,7 +536,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb case HEARTBEAT_KEY_DBINFO: { void *rspMsg = NULL; int32_t rspLen = 0; - mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen); + mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbCacheInfo), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { SKv kv1 = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv1); @@ -551,16 +553,6 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } break; } - case HEARTBEAT_KEY_USER_PASSINFO: { - void *rspMsg = NULL; - int32_t rspLen = 0; - mndValidateUserPassInfo(pMnode, kv->value, kv->valueLen / sizeof(SUserPassVersion), &rspMsg, &rspLen); - if (rspMsg && rspLen > 0) { - SKv kv1 = {.key = HEARTBEAT_KEY_USER_PASSINFO, .valueLen = rspLen, .value = rspMsg}; - taosArrayPush(hbRsp.info, &kv1); - } - break; - } default: mError("invalid kv key:%d", kv->key); hbRsp.status = TSDB_CODE_APP_ERROR; @@ -764,104 +756,132 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl return numOfRows; } -static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - int32_t cols = 0; - SConnObj *pConn = NULL; - - if (pShow->pIter == NULL) { - SProfileMgmt *pMgmt = &pMnode->profileMgmt; - pShow->pIter = taosCacheCreateIter(pMgmt->connCache); +/** + * @param pConn the conn queries pack from + * @param[out] pBlock the block data packed into + * @param offset skip [offset] queries in pConn + * @param rowsToPack at most rows to pack + * @return rows packed +*/ +static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBlock* pBlock, uint32_t offset, uint32_t rowsToPack) { + int32_t cols = 0; + taosRLockLatch(&pConn->queryLock); + int32_t numOfQueries = taosArrayGetSize(pConn->pQueries); + if (NULL == pConn->pQueries || numOfQueries <= offset) { + taosRUnLockLatch(&pConn->queryLock); + return 0; } - while (numOfRows < rows) { - pConn = mndGetNextConn(pMnode, pShow->pIter); - if (pConn == NULL) { - pShow->pIter = NULL; - break; - } + int32_t i = offset; + for (; i < numOfQueries && (i - offset) < rowsToPack; ++i) { + int32_t curRowIndex = pBlock->info.rows; + SQueryDesc *pQuery = taosArrayGet(pConn->pQueries, i); + cols = 0; - taosRLockLatch(&pConn->queryLock); - if (NULL == pConn->pQueries || taosArrayGetSize(pConn->pQueries) <= 0) { - taosRUnLockLatch(&pConn->queryLock); - continue; - } + char queryId[26 + VARSTR_HEADER_SIZE] = {0}; + sprintf(&queryId[VARSTR_HEADER_SIZE], "%x:%" PRIx64, pConn->id, pQuery->reqRid); + varDataLen(queryId) = strlen(&queryId[VARSTR_HEADER_SIZE]); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, curRowIndex, (const char *)queryId, false); - int32_t numOfQueries = taosArrayGetSize(pConn->pQueries); - for (int32_t i = 0; i < numOfQueries && numOfRows < rows; ++i) { - SQueryDesc *pQuery = taosArrayGet(pConn->pQueries, i); - cols = 0; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->queryId, false); - char queryId[26 + VARSTR_HEADER_SIZE] = {0}; - sprintf(&queryId[VARSTR_HEADER_SIZE], "%x:%" PRIx64, pConn->id, pQuery->reqRid); - varDataLen(queryId) = strlen(&queryId[VARSTR_HEADER_SIZE]); - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)queryId, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, curRowIndex, (const char *)&pConn->id, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->queryId, false); + char app[TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE]; + STR_TO_VARSTR(app, pConn->app); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, curRowIndex, (const char *)app, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->id, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, curRowIndex, (const char *)&pConn->pid, false); - char app[TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE]; - STR_TO_VARSTR(app, pConn->app); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)app, false); + char user[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(user, pConn->user); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, curRowIndex, (const char *)user, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->pid, false); + char endpoint[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; + sprintf(&endpoint[VARSTR_HEADER_SIZE], "%s:%d", taosIpStr(pConn->ip), pConn->port); + varDataLen(endpoint) = strlen(&endpoint[VARSTR_HEADER_SIZE]); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, curRowIndex, (const char *)endpoint, false); - char user[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(user, pConn->user); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)user, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->stime, false); - char endpoint[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; - sprintf(&endpoint[VARSTR_HEADER_SIZE], "%s:%d", taosIpStr(pConn->ip), pConn->port); - varDataLen(endpoint) = strlen(&endpoint[VARSTR_HEADER_SIZE]); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)endpoint, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->useconds, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->stime, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->stableQuery, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->useconds, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->isSubQuery, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->stableQuery, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->subPlanNum, false); + + char subStatus[TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t strSize = sizeof(subStatus); + int32_t offset = VARSTR_HEADER_SIZE; + for (int32_t i = 0; i < pQuery->subPlanNum && offset < strSize; ++i) { + if (i) { + offset += snprintf(subStatus + offset, strSize - offset - 1, ","); + } + SQuerySubDesc *pDesc = taosArrayGet(pQuery->subDesc, i); + offset += snprintf(subStatus + offset, strSize - offset - 1, "%" PRIu64 ":%s", pDesc->tid, pDesc->status); + } + varDataLen(subStatus) = strlen(&subStatus[VARSTR_HEADER_SIZE]); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, curRowIndex, subStatus, (varDataLen(subStatus) == 0) ? true : false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->subPlanNum, false); + char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(sql, pQuery->sql); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, curRowIndex, (const char *)sql, false); - char subStatus[TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE] = {0}; - int32_t strSize = sizeof(subStatus); - int32_t offset = VARSTR_HEADER_SIZE; - for (int32_t i = 0; i < pQuery->subPlanNum && offset < strSize; ++i) { - if (i) { - offset += snprintf(subStatus + offset, strSize - offset - 1, ","); - } - SQuerySubDesc *pDesc = taosArrayGet(pQuery->subDesc, i); - offset += snprintf(subStatus + offset, strSize - offset - 1, "%" PRIu64 ":%s", pDesc->tid, pDesc->status); - } - varDataLen(subStatus) = strlen(&subStatus[VARSTR_HEADER_SIZE]); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, subStatus, false); + pBlock->info.rows++; + } - char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(sql, pQuery->sql); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)sql, false); + taosRUnLockLatch(&pConn->queryLock); + return i - offset; +} - numOfRows++; - } +static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { + SMnode * pMnode = pReq->info.node; + SSdb * pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SConnObj *pConn = NULL; - taosRUnLockLatch(&pConn->queryLock); + if (pShow->pIter == NULL) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + pShow->pIter = taosCacheCreateIter(pMgmt->connCache); + } + + // means fetched some data last time for this conn + if (pShow->curIterPackedRows > 0) { + size_t len = 0; + pConn = taosCacheIterGetData(pShow->pIter, &len); + if (pConn && (taosArrayGetSize(pConn->pQueries) > pShow->curIterPackedRows)) { + numOfRows = packQueriesIntoBlock(pShow, pConn, pBlock, pShow->curIterPackedRows, rows); + pShow->curIterPackedRows += numOfRows; + } } + while (numOfRows < rows) { + pConn = mndGetNextConn(pMnode, pShow->pIter); + if (pConn == NULL) { + pShow->pIter = NULL; + break; + } + + int32_t packedRows = packQueriesIntoBlock(pShow, pConn, pBlock, 0, rows - numOfRows); + pShow->curIterPackedRows = packedRows; + numOfRows += packedRows; + } pShow->numOfRows += numOfRows; return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index b5c9ce1f650176f9b6a7c116cd4c1f2616ae8817..5ec81440bbb0fa90c0e858b1b2381e0fcd6b4487 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -454,6 +454,7 @@ int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) { sdbRelease(pSdb, pObj); if (limit > 0 && numOfRows >= limit) { + sdbCancelFetch(pSdb, pIter); break; } } diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 5278fc776107c31e43375b16b708c0c8ad34901d..8e95fa3d6dd26ec9f6b99afe6dc2f49ce8fea389 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -33,6 +33,7 @@ void mndPostProcessQueryMsg(SRpcMsg *pMsg) { int32_t mndProcessQueryMsg(SRpcMsg *pMsg) { int32_t code = -1; SMnode *pMnode = pMsg->info.node; + SReadHandle handle = {.mnd = pMnode, .pMsgCb = &pMnode->msgCb}; mTrace("msg:%p, in query queue is processing", pMsg); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 734f624be0e19c942c10244f28263570d6ea4504..36771147a9b5c312cb225ccae27bb426db39b3f5 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -14,32 +14,21 @@ */ #include "mndScheduler.h" -#include "mndConsumer.h" #include "mndDb.h" -#include "mndDnode.h" -#include "mndMnode.h" -#include "mndShow.h" #include "mndSnode.h" -#include "mndStb.h" -#include "mndStream.h" -#include "mndSubscribe.h" -#include "mndTopic.h" -#include "mndTrans.h" -#include "mndUser.h" #include "mndVgroup.h" #include "parser.h" #include "tcompare.h" #include "tname.h" #include "tuuid.h" +#define SINK_NODE_LEVEL (0) extern bool tsDeployOnSnode; -static int32_t mndAddTaskToTaskSet(SArray* pArray, SStreamTask* pTask) { - int32_t childId = taosArrayGetSize(pArray); - pTask->selfChildId = childId; - taosArrayPush(pArray, &pTask); - return 0; -} +static int32_t setTaskUpstreamEpInfo(const SStreamTask* pTask, SStreamTask* pDownstream); +static int32_t mndAddSinkTaskToStream(SStreamObj* pStream, SArray* pTaskList, SMnode* pMnode, int32_t vgId, + SVgObj* pVgroup, int32_t fillHistory); +static void setFixedDownstreamEpInfo(SStreamTask* pDstTask, const SStreamTask* pTask); int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, int64_t watermark, int64_t deleteMark) { @@ -97,28 +86,33 @@ END: return terrno; } -int32_t mndAddSinkToTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask) { +int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) { if (pStream->smaId != 0) { - pTask->outputType = TASK_OUTPUT__SMA; + pTask->outputInfo.type = TASK_OUTPUT__SMA; pTask->smaSink.smaId = pStream->smaId; } else { - pTask->outputType = TASK_OUTPUT__TABLE; + pTask->outputInfo.type = TASK_OUTPUT__TABLE; pTask->tbSink.stbUid = pStream->targetStbUid; memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); + if (pTask->tbSink.pSchemaWrapper == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } + return 0; } -int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask) { +int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SArray* pSinkNodeList, + SStreamTask* pTask) { bool isShuffle = false; if (pStream->fixedSinkVgId == 0) { SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb); if (pDb != NULL && pDb->cfg.numOfVgroups > 1) { isShuffle = true; - pTask->outputType = TASK_OUTPUT__SHUFFLE_DISPATCH; - pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; + pTask->outputInfo.type = TASK_OUTPUT__SHUFFLE_DISPATCH; + pTask->msgInfo.msgType = TDMT_STREAM_TASK_DISPATCH; if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) { return -1; } @@ -127,47 +121,45 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream sdbRelease(pMnode->pSdb, pDb); } + int32_t numOfSinkNodes = taosArrayGetSize(pSinkNodeList); + if (isShuffle) { memcpy(pTask->shuffleDispatcher.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t sz = taosArrayGetSize(pVgs); - SArray* sinkLv = taosArrayGetP(pStream->tasks, 0); - int32_t sinkLvSize = taosArrayGetSize(sinkLv); - for (int32_t i = 0; i < sz; i++) { + + int32_t numOfVgroups = taosArrayGetSize(pVgs); + for (int32_t i = 0; i < numOfVgroups; i++) { SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i); - for (int32_t j = 0; j < sinkLvSize; j++) { - SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j); - if (pLastLevelTask->nodeId == pVgInfo->vgId) { - pVgInfo->taskId = pLastLevelTask->id.taskId; + + for (int32_t j = 0; j < numOfSinkNodes; j++) { + SStreamTask* pSinkTask = taosArrayGetP(pSinkNodeList, j); + if (pSinkTask->info.nodeId == pVgInfo->vgId) { + pVgInfo->taskId = pSinkTask->id.taskId; break; } } } } else { - pTask->outputType = TASK_OUTPUT__FIXED_DISPATCH; - pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; - SArray* pArray = taosArrayGetP(pStream->tasks, 0); - // one sink only - SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0); - pTask->fixedEpDispatcher.taskId = lastLevelTask->id.taskId; - pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; - pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet; + SStreamTask* pOneSinkTask = taosArrayGetP(pSinkNodeList, 0); + setFixedDownstreamEpInfo(pTask, pOneSinkTask); } + return 0; } -int32_t mndAssignTaskToVg(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { +int32_t mndAssignStreamTaskToVgroup(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { int32_t msgLen; - pTask->nodeId = pVgroup->vgId; - pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); - plan->execNode.nodeId = pVgroup->vgId; - plan->execNode.epSet = pTask->epSet; + pTask->info.nodeId = pVgroup->vgId; + pTask->info.epSet = mndGetVgroupEpset(pMnode, pVgroup); + plan->execNode.nodeId = pTask->info.nodeId; + plan->execNode.epSet = pTask->info.epSet; if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } + return 0; } @@ -176,17 +168,19 @@ SSnodeObj* mndSchedFetchOneSnode(SMnode* pMnode) { void* pIter = NULL; // TODO random fetch pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void**)&pObj); + sdbCancelFetch(pMnode->pSdb, pIter); return pObj; } -int32_t mndAssignTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SSnodeObj* pSnode) { +int32_t mndAssignStreamTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SSnodeObj* pSnode) { int32_t msgLen; - pTask->nodeId = SNODE_HANDLE; - pTask->epSet = mndAcquireEpFromSnode(pMnode, pSnode); + pTask->info.nodeId = SNODE_HANDLE; + pTask->info.epSet = mndAcquireEpFromSnode(pMnode, pSnode); plan->execNode.nodeId = SNODE_HANDLE; - plan->execNode.epSet = pTask->epSet; + plan->execNode.epSet = pTask->info.epSet; + mDebug("s-task:0x%x set the agg task to snode:%d", pTask->id.taskId, SNODE_HANDLE); if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { terrno = TSDB_CODE_QRY_INVALID_INPUT; @@ -195,6 +189,7 @@ int32_t mndAssignTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, return 0; } +// todo random choose a node to do compute SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) { void* pIter = NULL; SVgObj* pVgroup = NULL; @@ -205,327 +200,485 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) { sdbRelease(pMnode->pSdb, pVgroup); continue; } + sdbCancelFetch(pMnode->pSdb, pIter); return pVgroup; } return pVgroup; } -int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { - SSdb* pSdb = pMnode->pSdb; - void* pIter = NULL; - SArray* tasks = taosArrayGetP(pStream->tasks, 0); +// create sink node for each vgroup. +int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SArray* pTaskList, SStreamObj* pStream, int32_t fillHistory) { + SSdb* pSdb = pMnode->pSdb; + void* pIter = NULL; while (1) { SVgObj* pVgroup = NULL; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); - if (pIter == NULL) break; - if (!mndVgroupInDb(pVgroup, pStream->targetDbUid)) { - sdbRelease(pSdb, pVgroup); - continue; + if (pIter == NULL) { + break; } - SStreamTask* pTask = tNewStreamTask(pStream->uid); - if (pTask == NULL) { + if (!mndVgroupInDb(pVgroup, pStream->targetDbUid)) { sdbRelease(pSdb, pVgroup); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + continue; } - pTask->fillHistory = pStream->fillHistory; - mndAddTaskToTaskSet(tasks, pTask); - - pTask->nodeId = pVgroup->vgId; - pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); - // type - pTask->taskLevel = TASK_LEVEL__SINK; - - // sink - if (pStream->smaId != 0) { - pTask->outputType = TASK_OUTPUT__SMA; - pTask->smaSink.smaId = pStream->smaId; - } else { - pTask->outputType = TASK_OUTPUT__TABLE; - pTask->tbSink.stbUid = pStream->targetStbUid; - memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); - pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); - if (pTask->tbSink.pSchemaWrapper == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - } + mndAddSinkTaskToStream(pStream, pTaskList, pMnode, pVgroup->vgId, pVgroup, fillHistory); sdbRelease(pSdb, pVgroup); } + return 0; } -int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { - SArray* tasks = taosArrayGetP(pStream->tasks, 0); - SStreamTask* pTask = tNewStreamTask(pStream->uid); +int32_t mndAddSinkTaskToStream(SStreamObj* pStream, SArray* pTaskList, SMnode* pMnode, int32_t vgId, SVgObj* pVgroup, + int32_t fillHistory) { + int64_t uid = (fillHistory == 0)? pStream->uid:pStream->hTaskUid; + SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SINK, fillHistory, 0, pTaskList); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pTask->fillHistory = pStream->fillHistory; - mndAddTaskToTaskSet(tasks, pTask); - pTask->nodeId = pStream->fixedSinkVgId; -#if 0 - SVgObj* pVgroup = mndAcquireVgroup(pMnode, pStream->fixedSinkVgId); - if (pVgroup == NULL) { - return -1; + pTask->info.nodeId = vgId; + pTask->info.epSet = mndGetVgroupEpset(pMnode, pVgroup); + mndSetSinkTaskInfo(pStream, pTask); + return 0; +} + +static int32_t addSourceStreamTask(SMnode* pMnode, SVgObj* pVgroup, SArray* pTaskList, SArray* pSinkTaskList, + SStreamObj* pStream, SSubplan* plan, uint64_t uid, int8_t fillHistory, + bool hasExtraSink, int64_t firstWindowSkey) { + SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SOURCE, fillHistory, pStream->conf.triggerParam, pTaskList); + if (pTask == NULL) { + return terrno; } - pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); -#endif - pTask->epSet = mndGetVgroupEpset(pMnode, &pStream->fixedSinkVg); - pTask->taskLevel = TASK_LEVEL__SINK; + // todo set the correct ts, which should be last key of queried table. + STimeWindow* pWindow = &pTask->dataRange.window; - // sink - if (pStream->smaId != 0) { - pTask->outputType = TASK_OUTPUT__SMA; - pTask->smaSink.smaId = pStream->smaId; + pWindow->skey = INT64_MIN; + pWindow->ekey = firstWindowSkey - 1; + mDebug("add source task 0x%x window:%" PRId64 " - %" PRId64, pTask->id.taskId, pWindow->skey, pWindow->ekey); + + // sink or dispatch + if (hasExtraSink) { + mndAddDispatcherForInternalTask(pMnode, pStream, pSinkTaskList, pTask); } else { - pTask->outputType = TASK_OUTPUT__TABLE; - pTask->tbSink.stbUid = pStream->targetStbUid; - memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); - pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); + mndSetSinkTaskInfo(pStream, pTask); } - return 0; + if (mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup) < 0) { + return terrno; + } + + for(int32_t i = 0; i < taosArrayGetSize(pSinkTaskList); ++i) { + SStreamTask* pSinkTask = taosArrayGetP(pSinkTaskList, i); + setTaskUpstreamEpInfo(pTask, pSinkTask); + } + + return TSDB_CODE_SUCCESS; } -int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { - SSdb* pSdb = pMnode->pSdb; - SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); - if (pPlan == NULL) { +static SStreamChildEpInfo* createStreamTaskEpInfo(const SStreamTask* pTask) { + SStreamChildEpInfo* pEpInfo = taosMemoryMalloc(sizeof(SStreamChildEpInfo)); + if (pEpInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pEpInfo->childId = pTask->info.selfChildId; + pEpInfo->epSet = pTask->info.epSet; + pEpInfo->nodeId = pTask->info.nodeId; + pEpInfo->taskId = pTask->id.taskId; + + return pEpInfo; +} + +void setFixedDownstreamEpInfo(SStreamTask* pDstTask, const SStreamTask* pTask) { + STaskDispatcherFixedEp* pDispatcher = &pDstTask->fixedEpDispatcher; + pDispatcher->taskId = pTask->id.taskId; + pDispatcher->nodeId = pTask->info.nodeId; + pDispatcher->epSet = pTask->info.epSet; + + pDstTask->outputInfo.type = TASK_OUTPUT__FIXED_DISPATCH; + pDstTask->msgInfo.msgType = TDMT_STREAM_TASK_DISPATCH; +} + +int32_t setTaskUpstreamEpInfo(const SStreamTask* pTask, SStreamTask* pDownstream) { + SStreamChildEpInfo* pEpInfo = createStreamTaskEpInfo(pTask); + if (pEpInfo == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (pDownstream->pUpstreamEpInfoList == NULL) { + pDownstream->pUpstreamEpInfoList = taosArrayInit(4, POINTER_BYTES); + } + + taosArrayPush(pDownstream->pUpstreamEpInfoList, &pEpInfo); + return TSDB_CODE_SUCCESS; +} + +static SArray* addNewTaskList(SArray* pTasksList) { + SArray* pTaskList = taosArrayInit(0, POINTER_BYTES); + taosArrayPush(pTasksList, &pTaskList); + return pTaskList; +} + +// set the history task id +static void setHTasksId(SArray* pTaskList, const SArray* pHTaskList) { + for (int32_t i = 0; i < taosArrayGetSize(pTaskList); ++i) { + SStreamTask** pStreamTask = taosArrayGet(pTaskList, i); + SStreamTask** pHTask = taosArrayGet(pHTaskList, i); + + (*pStreamTask)->historyTaskId.taskId = (*pHTask)->id.taskId; + (*pStreamTask)->historyTaskId.streamId = (*pHTask)->id.streamId; + + (*pHTask)->streamTaskId.taskId = (*pStreamTask)->id.taskId; + (*pHTask)->streamTaskId.streamId = (*pStreamTask)->id.streamId; + + mDebug("s-task:0x%" PRIx64 "-0x%x related history task:0x%" PRIx64 "-0x%x, level:%d", (*pStreamTask)->id.streamId, + (*pStreamTask)->id.taskId, (*pHTask)->id.streamId, (*pHTask)->id.taskId, (*pHTask)->info.taskLevel); + } +} + +static int32_t addSourceTasksForOneLevelStream(SMnode* pMnode, const SQueryPlan* pPlan, SStreamObj* pStream, + bool hasExtraSink, int64_t nextWindowSkey) { + // create exec stream task, since only one level, the exec task is also the source task + SArray* pTaskList = addNewTaskList(pStream->tasks); + SSdb* pSdb = pMnode->pSdb; + + SArray* pHTaskList = NULL; + if (pStream->conf.fillHistory) { + pHTaskList = addNewTaskList(pStream->pHTasksList); + } + + SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); + if (LIST_LENGTH(inner->pNodeList) != 1) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - int32_t planTotLevel = LIST_LENGTH(pPlan->pSubplans); - - pStream->tasks = taosArrayInit(planTotLevel, sizeof(void*)); - bool hasExtraSink = false; - bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; - SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb); - if (pDbObj == NULL) { + SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); + if (plan->subplanType != SUBPLAN_TYPE_SCAN) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - bool multiTarget = pDbObj->cfg.numOfVgroups > 1; - sdbRelease(pSdb, pDbObj); + void* pIter = NULL; + while (1) { + SVgObj* pVgroup; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); + if (pIter == NULL) { + break; + } - if (planTotLevel == 2 || externalTargetDB || multiTarget || pStream->fixedSinkVgId) { - /*if (true) {*/ - SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); - taosArrayPush(pStream->tasks, &taskOneLevel); - // add extra sink - hasExtraSink = true; - if (pStream->fixedSinkVgId == 0) { - if (mndAddShuffleSinkTasksToStream(pMnode, pStream) < 0) { - // TODO free - return -1; - } - } else { - if (mndAddFixedSinkTaskToStream(pMnode, pStream) < 0) { - // TODO free - return -1; - } + if (!mndVgroupInDb(pVgroup, pStream->sourceDbUid)) { + sdbRelease(pSdb, pVgroup); + continue; + } + + // new stream task + SArray** pSinkTaskList = taosArrayGet(pStream->tasks, SINK_NODE_LEVEL); + int32_t code = addSourceStreamTask(pMnode, pVgroup, pTaskList, *pSinkTaskList, pStream, plan, pStream->uid, 0, + hasExtraSink, nextWindowSkey); + if (code != TSDB_CODE_SUCCESS) { + sdbRelease(pSdb, pVgroup); + return -1; + } + + if (pStream->conf.fillHistory) { + SArray** pHSinkTaskList = taosArrayGet(pStream->pHTasksList, SINK_NODE_LEVEL); + code = addSourceStreamTask(pMnode, pVgroup, pHTaskList, *pHSinkTaskList, pStream, plan, pStream->hTaskUid, + 1, hasExtraSink, nextWindowSkey); + } + + sdbRelease(pSdb, pVgroup); + if (code != TSDB_CODE_SUCCESS) { + return -1; } } - pStream->totalLevel = planTotLevel + hasExtraSink; - if (planTotLevel > 1) { - SStreamTask* pInnerTask; - // inner level - { - SArray* taskInnerLevel = taosArrayInit(0, sizeof(void*)); - taosArrayPush(pStream->tasks, &taskInnerLevel); + if (pStream->conf.fillHistory) { + setHTasksId(pTaskList, pHTaskList); + } - SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); - SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); - if (plan->subplanType != SUBPLAN_TYPE_MERGE) { - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; - } + return TSDB_CODE_SUCCESS; +} - pInnerTask = tNewStreamTask(pStream->uid); - if (pInnerTask == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - qDestroyQueryPlan(pPlan); - return -1; - } +static int32_t doAddSourceTask(SArray* pTaskList, int8_t fillHistory, int64_t uid, SStreamTask* pDownstreamTask, + SMnode* pMnode, SSubplan* pPlan, SVgObj* pVgroup, int64_t nextWindowSkey) { + SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SOURCE, fillHistory, 0, pTaskList); + if (pTask == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } - pInnerTask->fillHistory = pStream->fillHistory; - mndAddTaskToTaskSet(taskInnerLevel, pInnerTask); + // todo set the correct ts, which should be last key of queried table. + STimeWindow* pWindow = &pTask->dataRange.window; + pWindow->skey = INT64_MIN; + pWindow->ekey = nextWindowSkey - 1; - pInnerTask->childEpInfo = taosArrayInit(0, sizeof(void*)); + mDebug("s-task:0x%x level:%d set time window:%" PRId64 " - %" PRId64, pTask->id.taskId, pTask->info.taskLevel, + pWindow->skey, pWindow->ekey); - pInnerTask->taskLevel = TASK_LEVEL__AGG; + // all the source tasks dispatch result to a single agg node. + setFixedDownstreamEpInfo(pTask, pDownstreamTask); + if (mndAssignStreamTaskToVgroup(pMnode, pTask, pPlan, pVgroup) < 0) { + return -1; + } - // trigger - pInnerTask->triggerParam = pStream->triggerParam; + return setTaskUpstreamEpInfo(pTask, pDownstreamTask); +} - // dispatch - if (mndAddDispatcherToInnerTask(pMnode, pStream, pInnerTask) < 0) { - qDestroyQueryPlan(pPlan); - return -1; - } +static int32_t doAddAggTask(uint64_t uid, SArray* pTaskList, SArray* pSinkNodeList, SMnode* pMnode, SStreamObj* pStream, + int32_t fillHistory, SStreamTask** pAggTask) { + *pAggTask = tNewStreamTask(uid, TASK_LEVEL__AGG, fillHistory, pStream->conf.triggerParam, pTaskList); + if (*pAggTask == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } - if (tsDeployOnSnode) { - SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode); - if (pSnode == NULL) { - SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); - if (mndAssignTaskToVg(pMnode, pInnerTask, plan, pVgroup) < 0) { - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - return -1; - } - sdbRelease(pSdb, pVgroup); - } else { - if (mndAssignTaskToSnode(pMnode, pInnerTask, plan, pSnode) < 0) { - sdbRelease(pSdb, pSnode); - qDestroyQueryPlan(pPlan); - return -1; - } - } + // dispatch + if (mndAddDispatcherForInternalTask(pMnode, pStream, pSinkNodeList, *pAggTask) < 0) { + return -1; + } + + return 0; +} + +static int32_t addAggTask(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan, SStreamTask** pAggTask, + SStreamTask** pHAggTask) { + SArray* pAggTaskList = addNewTaskList(pStream->tasks); + SSdb* pSdb = pMnode->pSdb; + + SNodeListNode* pInnerNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); + SSubplan* plan = (SSubplan*)nodesListGetNode(pInnerNode->pNodeList, 0); + if (plan->subplanType != SUBPLAN_TYPE_MERGE) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + + *pAggTask = NULL; + SArray* pSinkNodeList = taosArrayGetP(pStream->tasks, SINK_NODE_LEVEL); + + int32_t code = doAddAggTask(pStream->uid, pAggTaskList, pSinkNodeList, pMnode, pStream, 0, pAggTask); + if (code != TSDB_CODE_SUCCESS) { + return -1; + } + + SVgObj* pVgroup = NULL; + SSnodeObj* pSnode = NULL; + + if (tsDeployOnSnode) { + pSnode = mndSchedFetchOneSnode(pMnode); + if (pSnode == NULL) { + pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); + } + } else { + pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); + } + + if (pSnode != NULL) { + code = mndAssignStreamTaskToSnode(pMnode, *pAggTask, plan, pSnode); + } else { + code = mndAssignStreamTaskToVgroup(pMnode, *pAggTask, plan, pVgroup); + } + + if (pStream->conf.fillHistory) { + SArray* pHAggTaskList = addNewTaskList(pStream->pHTasksList); + SArray* pHSinkNodeList = taosArrayGetP(pStream->pHTasksList, SINK_NODE_LEVEL); + + *pHAggTask = NULL; + code = doAddAggTask(pStream->hTaskUid, pHAggTaskList, pHSinkNodeList, pMnode, pStream, pStream->conf.fillHistory, + pHAggTask); + if (code != TSDB_CODE_SUCCESS) { + if (pSnode != NULL) { + sdbRelease(pSdb, pSnode); } else { - SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); - if (mndAssignTaskToVg(pMnode, pInnerTask, plan, pVgroup) < 0) { - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - return -1; - } sdbRelease(pSdb, pVgroup); } + return code; } - // source level - SArray* taskSourceLevel = taosArrayInit(0, sizeof(void*)); - taosArrayPush(pStream->tasks, &taskSourceLevel); - - SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 1); - SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); - if (plan->subplanType != SUBPLAN_TYPE_SCAN) { - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; + if (pSnode != NULL) { + code = mndAssignStreamTaskToSnode(pMnode, *pHAggTask, plan, pSnode); + } else { + code = mndAssignStreamTaskToVgroup(pMnode, *pHAggTask, plan, pVgroup); } - void* pIter = NULL; - while (1) { - SVgObj* pVgroup; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); - if (pIter == NULL) break; - if (!mndVgroupInDb(pVgroup, pStream->sourceDbUid)) { - sdbRelease(pSdb, pVgroup); - continue; - } + setHTasksId(pAggTaskList, pHAggTaskList); + } - SStreamTask* pTask = tNewStreamTask(pStream->uid); - if (pTask == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - return -1; - } - pTask->fillHistory = pStream->fillHistory; - mndAddTaskToTaskSet(taskSourceLevel, pTask); + if (pSnode != NULL) { + sdbRelease(pSdb, pSnode); + } else { + sdbRelease(pSdb, pVgroup); + } - pTask->triggerParam = 0; + return code; +} - // source - pTask->taskLevel = TASK_LEVEL__SOURCE; +static int32_t addSourceTasksForMultiLevelStream(SMnode* pMnode, SQueryPlan* pPlan, SStreamObj* pStream, + SStreamTask* pDownstreamTask, SStreamTask* pHDownstreamTask, int64_t nextWindowSkey) { + SArray* pSourceTaskList = addNewTaskList(pStream->tasks); - // add fixed vg dispatch - pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; - pTask->outputType = TASK_OUTPUT__FIXED_DISPATCH; + SArray* pHSourceTaskList = NULL; + if (pStream->conf.fillHistory) { + pHSourceTaskList = addNewTaskList(pStream->pHTasksList); + } - pTask->fixedEpDispatcher.taskId = pInnerTask->id.taskId; - pTask->fixedEpDispatcher.nodeId = pInnerTask->nodeId; - pTask->fixedEpDispatcher.epSet = pInnerTask->epSet; + SSdb* pSdb = pMnode->pSdb; + SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 1); + SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); + if (plan->subplanType != SUBPLAN_TYPE_SCAN) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } - if (mndAssignTaskToVg(pMnode, pTask, plan, pVgroup) < 0) { - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - return -1; - } + void* pIter = NULL; + while (1) { + SVgObj* pVgroup; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); + if (pIter == NULL) { + break; + } + + if (!mndVgroupInDb(pVgroup, pStream->sourceDbUid)) { + sdbRelease(pSdb, pVgroup); + continue; + } - SStreamChildEpInfo* pEpInfo = taosMemoryMalloc(sizeof(SStreamChildEpInfo)); - if (pEpInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = + doAddSourceTask(pSourceTaskList, 0, pStream->uid, pDownstreamTask, pMnode, plan, pVgroup, nextWindowSkey); + if (code != TSDB_CODE_SUCCESS) { + sdbRelease(pSdb, pVgroup); + terrno = code; + return -1; + } + + if (pStream->conf.fillHistory) { + code = doAddSourceTask(pHSourceTaskList, 1, pStream->hTaskUid, pHDownstreamTask, pMnode, plan, pVgroup, + nextWindowSkey); + if (code != TSDB_CODE_SUCCESS) { sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - return -1; + return code; } - pEpInfo->childId = pTask->selfChildId; - pEpInfo->epSet = pTask->epSet; - pEpInfo->nodeId = pTask->nodeId; - pEpInfo->taskId = pTask->id.taskId; - taosArrayPush(pInnerTask->childEpInfo, &pEpInfo); - sdbRelease(pSdb, pVgroup); } + + sdbRelease(pSdb, pVgroup); + } + + if (pStream->conf.fillHistory) { + setHTasksId(pSourceTaskList, pHSourceTaskList); } - if (planTotLevel == 1) { - SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); - taosArrayPush(pStream->tasks, &taskOneLevel); + return TSDB_CODE_SUCCESS; +} - SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); - if (LIST_LENGTH(inner->pNodeList) != 1) { - terrno = TSDB_CODE_QRY_INVALID_INPUT; +static int32_t addSinkTasks(SArray* pTasksList, SMnode* pMnode, SStreamObj* pStream, SArray** pCreatedTaskList, + int32_t fillHistory) { + SArray* pSinkTaskList = addNewTaskList(pTasksList); + if (pStream->fixedSinkVgId == 0) { + if (mndAddShuffleSinkTasksToStream(pMnode, pSinkTaskList, pStream, fillHistory) < 0) { + // TODO free return -1; } - SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); - if (plan->subplanType != SUBPLAN_TYPE_SCAN) { - terrno = TSDB_CODE_QRY_INVALID_INPUT; + } else { + if (mndAddSinkTaskToStream(pStream, pSinkTaskList, pMnode, pStream->fixedSinkVgId, &pStream->fixedSinkVg, + fillHistory) < 0) { + // TODO free return -1; } + } - void* pIter = NULL; - while (1) { - SVgObj* pVgroup; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); - if (pIter == NULL) break; - if (!mndVgroupInDb(pVgroup, pStream->sourceDbUid)) { - sdbRelease(pSdb, pVgroup); - continue; - } + *pCreatedTaskList = pSinkTaskList; + return TSDB_CODE_SUCCESS; +} - SStreamTask* pTask = tNewStreamTask(pStream->uid); - if (pTask == NULL) { - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - return -1; - } - pTask->fillHistory = pStream->fillHistory; - mndAddTaskToTaskSet(taskOneLevel, pTask); +static void setSinkTaskUpstreamInfo(SArray* pTasksList, const SStreamTask* pUpstreamTask) { + SArray* pSinkTaskList = taosArrayGetP(pTasksList, SINK_NODE_LEVEL); + for(int32_t i = 0; i < taosArrayGetSize(pSinkTaskList); ++i) { + SStreamTask* pSinkTask = taosArrayGetP(pSinkTaskList, i); + setTaskUpstreamEpInfo(pUpstreamTask, pSinkTask); + } +} - // source - pTask->taskLevel = TASK_LEVEL__SOURCE; +static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan, int64_t nextWindowSkey) { + SSdb* pSdb = pMnode->pSdb; + int32_t numOfPlanLevel = LIST_LENGTH(pPlan->pSubplans); - // trigger - pTask->triggerParam = pStream->triggerParam; + bool hasExtraSink = false; + bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; + SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb); + if (pDbObj == NULL) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } - // sink or dispatch - if (hasExtraSink) { - mndAddDispatcherToInnerTask(pMnode, pStream, pTask); - } else { - mndAddSinkToTask(pMnode, pStream, pTask); - } + bool multiTarget = (pDbObj->cfg.numOfVgroups > 1); + sdbRelease(pSdb, pDbObj); - if (mndAssignTaskToVg(pMnode, pTask, plan, pVgroup) < 0) { - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - return -1; + pStream->tasks = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES); + pStream->pHTasksList = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES); + + if (numOfPlanLevel == 2 || externalTargetDB || multiTarget || pStream->fixedSinkVgId) { + // add extra sink + hasExtraSink = true; + + SArray* pSinkTaskList = NULL; + int32_t code = addSinkTasks(pStream->tasks, pMnode, pStream, &pSinkTaskList, 0); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // check for fill history + if (pStream->conf.fillHistory) { + SArray* pHSinkTaskList = NULL; + code = addSinkTasks(pStream->pHTasksList, pMnode, pStream, &pHSinkTaskList, 1); + if (code != TSDB_CODE_SUCCESS) { + return code; } - sdbRelease(pSdb, pVgroup); + + setHTasksId(pSinkTaskList, pHSinkTaskList); } } - qDestroyQueryPlan(pPlan); + + pStream->totalLevel = numOfPlanLevel + hasExtraSink; + + if (numOfPlanLevel > 1) { + SStreamTask* pAggTask = NULL; + SStreamTask* pHAggTask = NULL; + + int32_t code = addAggTask(pStream, pMnode, pPlan, &pAggTask, &pHAggTask); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + setSinkTaskUpstreamInfo(pStream->tasks, pAggTask); + setSinkTaskUpstreamInfo(pStream->pHTasksList, pHAggTask); + + // source level + return addSourceTasksForMultiLevelStream(pMnode, pPlan, pStream, pAggTask, pHAggTask, nextWindowSkey); + } else if (numOfPlanLevel == 1) { + return addSourceTasksForOneLevelStream(pMnode, pPlan, pStream, hasExtraSink, nextWindowSkey); + } + return 0; } +int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t nextWindowSkey) { + SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); + if (pPlan == NULL) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + + int32_t code = doScheduleStream(pStream, pMnode, pPlan, nextWindowSkey); + qDestroyQueryPlan(pPlan); + + return code; +} + int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) { SSdb* pSdb = pMnode->pSdb; SVgObj* pVgroup = NULL; @@ -538,7 +691,23 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } + } else if (pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL) { + SNode* pAst = NULL; + if (nodesStringToNode(pTopic->ast, &pAst) != 0) { + mError("topic:%s, failed to create since %s", pTopic->name, terrstr()); + return -1; + } + SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; + if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) { + mError("failed to create topic:%s since %s", pTopic->name, terrstr()); + nodesDestroyNode(pAst); + return -1; + } + nodesDestroyNode(pAst); + } + + if (pPlan) { int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); if (levelNum != 1) { qDestroyQueryPlan(pPlan); @@ -579,23 +748,19 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib mDebug("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId); - if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) { - int32_t msgLen; + sdbRelease(pSdb, pVgroup); + } - pSubplan->execNode.epSet = pVgEp->epSet; - pSubplan->execNode.nodeId = pVgEp->vgId; + if (pSubplan) { + int32_t msgLen; - if (qSubPlanToString(pSubplan, &pVgEp->qmsg, &msgLen) < 0) { - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; - } - } else { - pVgEp->qmsg = taosStrdup(""); + if (qSubPlanToString(pSubplan, &pSub->qmsg, &msgLen) < 0) { + qDestroyQueryPlan(pPlan); + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; } - - sdbRelease(pSdb, pVgroup); + } else { + pSub->qmsg = taosStrdup(""); } qDestroyQueryPlan(pPlan); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index c50b205f37f569fced5a690f329ae3ba26c1ac6a..44f47517006c46b5f3cdc69360757402b2e6f352 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -20,6 +20,7 @@ #define SHOW_STEP_SIZE 100 #define SHOW_COLS_STEP_SIZE 4096 +#define SHOW_PRIVILEGES_STEP_SIZE 2048 static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq); static void mndFreeShowObj(SShowObj *pShow); @@ -234,6 +235,8 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { if(pShow->type == TSDB_MGMT_TABLE_COL){ // expend capacity for ins_columns rowsToRead = SHOW_COLS_STEP_SIZE; + } else if (pShow->type == TSDB_MGMT_TABLE_PRIVILEGES) { + rowsToRead = SHOW_PRIVILEGES_STEP_SIZE; } ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type]; if (retrieveFp == NULL) { diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 3d654a23d87b082aa441012375d0691089e391d8..b84297f6bfd77f33ca2e27b04fd5b2b172a1286c 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -388,7 +388,7 @@ static int32_t mndSetCreateSmaVgroupRedoLogs(SMnode *pMnode, STrans *pTrans, SVg SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); if (pVgRaw == NULL) return -1; if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; - if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_UPDATE) != 0) return -1; return 0; } @@ -504,6 +504,11 @@ static void mndDestroySmaObj(SSmaObj *pSmaObj) { static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCreate, SDbObj *pDb, SStbObj *pStb, const char *streamName) { + if (pDb->cfg.replications > 1) { + terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + mError("sma:%s, failed to create since not support multiple replicas", pCreate->name); + return -1; + } SSmaObj smaObj = {0}; memcpy(smaObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); memcpy(smaObj.stb, pStb->name, TSDB_TABLE_FNAME_LEN); @@ -555,20 +560,20 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea streamObj.version = 1; streamObj.sql = taosStrdup(pCreate->sql); streamObj.smaId = smaObj.uid; - streamObj.watermark = pCreate->watermark; + streamObj.conf.watermark = pCreate->watermark; streamObj.deleteMark = pCreate->deleteMark; - streamObj.fillHistory = STREAM_FILL_HISTORY_ON; - streamObj.trigger = STREAM_TRIGGER_WINDOW_CLOSE; - streamObj.triggerParam = pCreate->maxDelay; + streamObj.conf.fillHistory = STREAM_FILL_HISTORY_ON; + streamObj.conf.trigger = STREAM_TRIGGER_WINDOW_CLOSE; + streamObj.conf.triggerParam = pCreate->maxDelay; streamObj.ast = taosStrdup(smaObj.ast); // check the maxDelay - if (streamObj.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) { + if (streamObj.conf.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) { int64_t msInterval = convertTimeFromPrecisionToUnit(pCreate->interval, pDb->cfg.precision, TIME_UNIT_MILLISECOND); - streamObj.triggerParam = msInterval > TSDB_MIN_ROLLUP_MAX_DELAY ? msInterval : TSDB_MIN_ROLLUP_MAX_DELAY; + streamObj.conf.triggerParam = msInterval > TSDB_MIN_ROLLUP_MAX_DELAY ? msInterval : TSDB_MIN_ROLLUP_MAX_DELAY; } - if (streamObj.triggerParam > TSDB_MAX_ROLLUP_MAX_DELAY) { - streamObj.triggerParam = TSDB_MAX_ROLLUP_MAX_DELAY; + if (streamObj.conf.triggerParam > TSDB_MAX_ROLLUP_MAX_DELAY) { + streamObj.conf.triggerParam = TSDB_MAX_ROLLUP_MAX_DELAY; } if (mndAllocSmaVgroup(pMnode, pDb, &streamObj.fixedSinkVg) != 0) { @@ -597,8 +602,8 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea .pAstRoot = pAst, .topicQuery = false, .streamQuery = true, - .triggerType = streamObj.trigger, - .watermark = streamObj.watermark, + .triggerType = streamObj.conf.trigger, + .watermark = streamObj.conf.watermark, .deleteMark = streamObj.deleteMark, }; @@ -622,18 +627,18 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "create-sma"); if (pTrans == NULL) goto _OVER; mndTransSetDbName(pTrans, pDb->name, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to create sma:%s stream:%s", pTrans->id, pCreate->name, streamObj.name); - + if (mndAddPrepareNewVgAction(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupCommitLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg, &smaObj) != 0) goto _OVER; - if (mndScheduleStream(pMnode, &streamObj) != 0) goto _OVER; + if (mndScheduleStream(pMnode, &streamObj, 1685959190000) != 0) goto _OVER; if (mndPersistStream(pMnode, pTrans, &streamObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -845,7 +850,7 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p mInfo("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); mndTransSetDbName(pTrans, pDb->name, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; mndTransSetSerial(pTrans); @@ -889,11 +894,11 @@ _OVER: } int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { - SSdb *pSdb = pMnode->pSdb; - SSmaObj *pSma = NULL; - void *pIter = NULL; - SVgObj *pVgroup = NULL; - int32_t code = -1; + SSdb *pSdb = pMnode->pSdb; + SSmaObj *pSma = NULL; + void *pIter = NULL; + SVgObj *pVgroup = NULL; + int32_t code = -1; while (1) { pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); @@ -911,12 +916,18 @@ int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p if (pStream != NULL && pStream->smaId == pSma->uid) { if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { mError("stream:%s, failed to drop task since %s", pStream->name, terrstr()); + mndReleaseStream(pMnode, pStream); goto _OVER; } + if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { + mndReleaseStream(pMnode, pStream); goto _OVER; } + + mndReleaseStream(pMnode, pStream); } + if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER; if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; @@ -1114,6 +1125,7 @@ int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool return code; } + rsp->indexSize += sizeof(info) + pSma->exprLen + 1; *exist = true; sdbRelease(pSdb, pSma); @@ -1277,13 +1289,13 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc STR_TO_VARSTR(col, (char *)""); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)col, false); + colDataSetVal(pColInfo, numOfRows, (const char *)col, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); char tag[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(tag, (char *)"sma_index"); - colDataAppend(pColInfo, numOfRows, (const char *)tag, false); + colDataSetVal(pColInfo, numOfRows, (const char *)tag, false); numOfRows++; sdbRelease(pSdb, pSma); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8b708c3e0f6f06c1e784f3a7e3807371041716b8..70fd74afc0a80b327265e17d2cb5f1a8d0069769 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -687,6 +687,31 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } +int32_t mndSetForceDropCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + int32_t contLen; + + void *pReq = mndBuildVCreateStbReq(pMnode, pVgroup, pStb, &contLen, NULL, 0); + if (pReq == NULL) { + return -1; + } + + STransAction action = {0}; + action.mTraceId = pTrans->mTraceId; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_CREATE_STB; + action.acceptableCode = TSDB_CODE_TDB_STB_ALREADY_EXIST; + action.retryCode = TSDB_CODE_TDB_STB_NOT_EXIST; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; @@ -849,7 +874,7 @@ _OVER: int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) return -1; + if (mndTransCheckConflict(pMnode, pTrans) != 0) return -1; if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; @@ -863,7 +888,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; - SVDropTtlTableReq ttlReq = {.timestamp = taosGetTimestampSec()}; + SVDropTtlTableReq ttlReq = {.timestampSec = taosGetTimestampSec()}; int32_t reqLen = tSerializeSVDropTtlTableReq(NULL, 0, &ttlReq); int32_t contLen = reqLen + sizeof(SMsgHead); @@ -875,7 +900,6 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { SMsgHead *pHead = rpcMallocCont(contLen); if (pHead == NULL) { - sdbCancelFetch(pSdb, pVgroup); sdbRelease(pSdb, pVgroup); continue; } @@ -889,7 +913,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { if (code != 0) { mError("vgId:%d, failed to send drop ttl table request to vnode since 0x%x", pVgroup->vgId, code); } else { - mInfo("vgId:%d, send drop ttl table request to vnode, time:%d", pVgroup->vgId, ttlReq.timestamp); + mInfo("vgId:%d, send drop ttl table request to vnode, time:%" PRId32, pVgroup->vgId, ttlReq.timestampSec); } sdbRelease(pSdb, pVgroup); } @@ -1163,7 +1187,7 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } - + if(pNew->nextColId < 0 || pNew->nextColId >= 0x7fff - ntags){ terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW; return -1; @@ -1205,7 +1229,7 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, mInfo("topic:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d, subType:%d sql:%s", pTopic->name, stbFullName, suid, colId, pTopic->subType, pTopic->sql); - if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { + if (pTopic->ast == NULL) { sdbRelease(pSdb, pTopic); continue; } @@ -1215,6 +1239,7 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC; mError("topic:%s, create ast error", pTopic->name); sdbRelease(pSdb, pTopic); + sdbCancelFetch(pSdb, pIter); return -1; } @@ -1235,6 +1260,7 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, mError("topic:%s, check colId:%d conflicted", pTopic->name, pCol->colId); nodesDestroyNode(pAst); nodesDestroyList(pNodeList); + sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pTopic); return -1; } @@ -1262,6 +1288,7 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION; mError("stream:%s, create ast error", pStream->name); sdbRelease(pSdb, pStream); + sdbCancelFetch(pSdb, pIter); return -1; } @@ -1281,6 +1308,7 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName nodesDestroyNode(pAst); nodesDestroyList(pNodeList); sdbRelease(pSdb, pStream); + sdbCancelFetch(pSdb, pIter); return -1; } mInfo("stream:%s, check colId:%d passed", pStream->name, pCol->colId); @@ -1310,6 +1338,7 @@ static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName, terrno = TSDB_CODE_SDB_INVALID_DATA_CONTENT; mError("tsma:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d failed since parse AST err", pSma->name, stbFullName, suid, colId); + sdbCancelFetch(pSdb, pIter); return -1; } @@ -1330,6 +1359,7 @@ static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName, nodesDestroyNode(pAst); nodesDestroyList(pNodeList); sdbRelease(pSdb, pSma); + sdbCancelFetch(pSdb, pIter); return -1; } mInfo("tsma:%s, check colId:%d passed", pSma->name, pCol->colId); @@ -1448,7 +1478,7 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj SSchema *pTag = pNew->pTags + tag; - if (!(pTag->type == TSDB_DATA_TYPE_BINARY || pTag->type == TSDB_DATA_TYPE_NCHAR)) { + if (!(pTag->type == TSDB_DATA_TYPE_BINARY || pTag->type == TSDB_DATA_TYPE_NCHAR || pTag->type == TSDB_DATA_TYPE_GEOMETRY)) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -1568,7 +1598,7 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO } SSchema *pCol = pNew->pColumns + col; - if (!(pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR)) { + if (!(pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR || pCol->type == TSDB_DATA_TYPE_GEOMETRY)) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -1713,6 +1743,7 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa SSchema *pSrcSchema = &pStb->pColumns[i]; memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); pSchema->type = pSrcSchema->type; + pSchema->flags = pSrcSchema->flags; pSchema->colId = pSrcSchema->colId; pSchema->bytes = pSrcSchema->bytes; } @@ -1763,6 +1794,7 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, SSchema *pSrcSchema = &pStb->pColumns[i]; memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); pSchema->type = pSrcSchema->type; + pSchema->flags = pSrcSchema->flags; pSchema->colId = pSrcSchema->colId; pSchema->bytes = pSrcSchema->bytes; } @@ -1772,6 +1804,7 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, SSchema *pSrcSchema = &pStb->pTags[i]; memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); pSchema->type = pSrcSchema->type; + pSchema->flags = pSrcSchema->flags; pSchema->colId = pSrcSchema->colId; pSchema->bytes = pSrcSchema->bytes; } @@ -1943,7 +1976,7 @@ static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbOb mInfo("trans:%d, used to alter stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; if (needRsp) { void *pCont = NULL; @@ -1973,7 +2006,7 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO mInfo("trans:%d, used to alter stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; if (needRsp) { void *pCont = NULL; @@ -2217,7 +2250,7 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p mInfo("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; @@ -2243,11 +2276,12 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) { if (pTopic->stbUid == suid) { sdbRelease(pSdb, pTopic); + sdbCancelFetch(pSdb, pIter); return -1; } } - if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { + if (pTopic->ast == NULL) { sdbRelease(pSdb, pTopic); continue; } @@ -2257,6 +2291,7 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; mError("topic:%s, create ast error", pTopic->name); sdbRelease(pSdb, pTopic); + sdbCancelFetch(pSdb, pIter); return -1; } @@ -2270,6 +2305,7 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, sdbRelease(pSdb, pTopic); nodesDestroyNode(pAst); nodesDestroyList(pNodeList); + sdbCancelFetch(pSdb, pIter); return -1; } else { goto NEXT; @@ -2297,6 +2333,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, } if (pStream->targetStbUid == suid) { + sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pStream); return -1; } @@ -2305,6 +2342,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, if (nodesStringToNode(pStream->ast, &pAst) != 0) { terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION; mError("stream:%s, create ast error", pStream->name); + sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pStream); return -1; } @@ -2316,6 +2354,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, SColumnNode *pCol = (SColumnNode *)pNode; if (pCol->tableId == suid) { + sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pStream); nodesDestroyNode(pAst); nodesDestroyList(pNodeList); @@ -2470,12 +2509,14 @@ static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) { goto _OVER; } - if (0 == strcmp(cfgReq.dbFName, TSDB_INFORMATION_SCHEMA_DB)) { + char dbName[TSDB_DB_NAME_LEN] = {0}; + mndExtractShortDbNameFromDbFullName(cfgReq.dbFName, dbName); + if (0 == strcmp(dbName, TSDB_INFORMATION_SCHEMA_DB)) { mInfo("information_schema table:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); if (mndBuildInsTableCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp) != 0) { goto _OVER; } - } else if (0 == strcmp(cfgReq.dbFName, TSDB_PERFORMANCE_SCHEMA_DB)) { + } else if (0 == strcmp(dbName, TSDB_PERFORMANCE_SCHEMA_DB)) { mInfo("performance_schema table:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); if (mndBuildPerfsTableCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp) != 0) { goto _OVER; @@ -2644,6 +2685,13 @@ void mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst) { tNameGetDbName(&name, dst); } +void mndExtractShortDbNameFromDbFullName(const char *stbFullName, char *dst) { + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&name, dst); +} + void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize) { int32_t pos = -1; int32_t num = 0; @@ -2742,7 +2790,7 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t // varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE])); // // SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)stbName, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)stbName, false); // // char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; // tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB); @@ -2750,29 +2798,29 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t // varDataSetLen(db, strlen(varDataVal(db))); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)db, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)db, false); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->createdTime, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->createdTime, false); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->numOfColumns, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfColumns, false); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->numOfTags, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfTags, false); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->updateTime, false); // number of tables +// colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->updateTime, false); // number of tables // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); // if (pStb->commentLen > 0) { // char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; // STR_TO_VARSTR(comment, pStb->comment); -// colDataAppend(pColInfo, numOfRows, comment, false); +// colDataSetVal(pColInfo, numOfRows, comment, false); // } else if (pStb->commentLen == 0) { // char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; // STR_TO_VARSTR(comment, ""); -// colDataAppend(pColInfo, numOfRows, comment, false); +// colDataSetVal(pColInfo, numOfRows, comment, false); // } else { // colDataSetNULL(pColInfo, numOfRows); // } @@ -2782,14 +2830,14 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t // varDataSetLen(watermark, strlen(varDataVal(watermark))); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)watermark, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)watermark, false); // // char maxDelay[64 + VARSTR_HEADER_SIZE] = {0}; // sprintf(varDataVal(maxDelay), "%" PRId64 "a,%" PRId64 "a", pStb->maxdelay[0], pStb->maxdelay[1]); // varDataSetLen(maxDelay, strlen(varDataVal(maxDelay))); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)maxDelay, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)maxDelay, false); // // char rollup[160 + VARSTR_HEADER_SIZE] = {0}; // int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); @@ -2808,7 +2856,7 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t // varDataSetLen(rollup, strlen(varDataVal(rollup))); // // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, numOfRows, (const char *)rollup, false); +// colDataSetVal(pColInfo, numOfRows, (const char *)rollup, false); // // numOfRows++; // sdbRelease(pSdb, pStb); @@ -3067,20 +3115,20 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *p for (int32_t j = 0; j < pm->colNum; j++) { // table name SColumnInfoData *pColInfoData = taosArrayGet(p->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, tName, false); + colDataSetVal(pColInfoData, numOfRows, tName, false); // database name pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dName, false); + colDataSetVal(pColInfoData, numOfRows, dName, false); pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, typeName, false); + colDataSetVal(pColInfoData, numOfRows, typeName, false); // col name char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(colName, pm->schema[j].name); pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, colName, false); + colDataSetVal(pColInfoData, numOfRows, colName, false); // col type int8_t colType = pm->schema[j].type; @@ -3095,10 +3143,10 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *p (int32_t)((pm->schema[j].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(colTypeStr, colTypeLen); - colDataAppend(pColInfoData, numOfRows, (char *)colTypeStr, false); + colDataSetVal(pColInfoData, numOfRows, (char *)colTypeStr, false); pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (const char *)&pm->schema[j].bytes, false); + colDataSetVal(pColInfoData, numOfRows, (const char *)&pm->schema[j].bytes, false); for (int32_t k = 6; k <= 8; ++k) { pColInfoData = taosArrayGet(p->pDataBlock, k); colDataSetNULL(pColInfoData, numOfRows); @@ -3134,8 +3182,14 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB SSdb *pSdb = pMnode->pSdb; SStbObj *pStb = NULL; - int32_t numOfRows = buildSysDbColsInfo(pBlock, pShow->db, pShow->filterTb); - mDebug("mndRetrieveStbCol get system table cols, rows:%d, db:%s", numOfRows, pShow->db); + + int32_t numOfRows = 0; + if (!pShow->sysDbRsp) { + numOfRows = buildSysDbColsInfo(pBlock, pShow->db, pShow->filterTb); + mDebug("mndRetrieveStbCol get system table cols, rows:%d, db:%s", numOfRows, pShow->db); + pShow->sysDbRsp = true; + } + SDbObj *pDb = NULL; if (strlen(pShow->db) > 0) { pDb = mndAcquireDb(pMnode, pShow->db); @@ -3192,19 +3246,19 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB for (int i = 0; i < pStb->numOfColumns; i++) { int32_t cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)stbName, false); + colDataSetVal(pColInfo, numOfRows, (const char *)stbName, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)db, false); + colDataSetVal(pColInfo, numOfRows, (const char *)db, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, typeName, false); + colDataSetVal(pColInfo, numOfRows, typeName, false); // col name char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(colName, pStb->pColumns[i].name); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, colName, false); + colDataSetVal(pColInfo, numOfRows, colName, false); // col type int8_t colType = pStb->pColumns[i].type; @@ -3219,10 +3273,10 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB (int32_t)((pStb->pColumns[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(colTypeStr, colTypeLen); - colDataAppend(pColInfo, numOfRows, (char *)colTypeStr, false); + colDataSetVal(pColInfo, numOfRows, (char *)colTypeStr, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStb->pColumns[i].bytes, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->pColumns[i].bytes, false); while (cols < pShow->numOfColumns) { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetNULL(pColInfo, numOfRows); @@ -3273,7 +3327,7 @@ static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) { mInfo("trans:%d, used to add index to stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 68b8dd72019df1ed623170249272e3cda3c5ad1a..a0d53ec780e26b969d1d543761582261b6251474 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -28,7 +28,7 @@ #include "parser.h" #include "tname.h" -#define MND_STREAM_VER_NUMBER 2 +#define MND_STREAM_VER_NUMBER 3 #define MND_STREAM_RESERVE_SIZE 64 #define MND_STREAM_MAX_NUM 60 @@ -140,10 +140,12 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { void *buf = NULL; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto STREAM_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) { + goto STREAM_DECODE_OVER; + } - if (sver != 1 && sver != 2) { - terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + if (sver != MND_STREAM_VER_NUMBER) { + terrno = 0; goto STREAM_DECODE_OVER; } @@ -198,12 +200,13 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream) { static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream) { mTrace("stream:%s, perform update action", pOldStream->name); - atomic_exchange_64(&pOldStream->updateTime, pNewStream->updateTime); + atomic_exchange_32(&pOldStream->version, pNewStream->version); taosWLockLatch(&pOldStream->lock); pOldStream->status = pNewStream->status; + pOldStream->updateTime = pNewStream->updateTime; taosWUnLockLatch(&pOldStream->lock); return 0; @@ -239,7 +242,7 @@ static void mndShowStreamStatus(char *dst, SStreamObj *pStream) { } static void mndShowStreamTrigger(char *dst, SStreamObj *pStream) { - int8_t trigger = pStream->trigger; + int8_t trigger = pStream->conf.trigger; if (trigger == STREAM_TRIGGER_AT_ONCE) { strcpy(dst, "at once"); } else if (trigger == STREAM_TRIGGER_WINDOW_CLOSE) { @@ -299,13 +302,18 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, pObj->smaId = 0; pObj->uid = mndGenerateUid(pObj->name, strlen(pObj->name)); + + char p[TSDB_STREAM_FNAME_LEN + 32] = {0}; + snprintf(p, tListLen(p), "%s_%s", pObj->name, "fillhistory"); + + pObj->hTaskUid = mndGenerateUid(pObj->name, strlen(pObj->name)); pObj->status = 0; - pObj->igExpired = pCreate->igExpired; - pObj->trigger = pCreate->triggerType; - pObj->triggerParam = pCreate->maxDelay; - pObj->watermark = pCreate->watermark; - pObj->fillHistory = pCreate->fillHistory; + pObj->conf.igExpired = pCreate->igExpired; + pObj->conf.trigger = pCreate->triggerType; + pObj->conf.triggerParam = pCreate->maxDelay; + pObj->conf.watermark = pCreate->watermark; + pObj->conf.fillHistory = pCreate->fillHistory; pObj->deleteMark = pCreate->deleteMark; pObj->igCheckUpdate = pCreate->igUpdate; @@ -387,9 +395,9 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, .pAstRoot = pAst, .topicQuery = false, .streamQuery = true, - .triggerType = pObj->trigger == STREAM_TRIGGER_MAX_DELAY ? STREAM_TRIGGER_WINDOW_CLOSE : pObj->trigger, - .watermark = pObj->watermark, - .igExpired = pObj->igExpired, + .triggerType = pObj->conf.trigger == STREAM_TRIGGER_MAX_DELAY ? STREAM_TRIGGER_WINDOW_CLOSE : pObj->conf.trigger, + .watermark = pObj->conf.watermark, + .igExpired = pObj->conf.igExpired, .deleteMark = pObj->deleteMark, .igCheckUpdate = pObj->igCheckUpdate, }; @@ -424,34 +432,43 @@ FAIL: return 0; } -int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { +int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) { SEncoder encoder; tEncoderInit(&encoder, NULL, 0); + + pTask->ver = SSTREAM_TASK_VER; tEncodeStreamTask(&encoder, pTask); + int32_t size = encoder.pos; int32_t tlen = sizeof(SMsgHead) + size; tEncoderClear(&encoder); + void *buf = taosMemoryCalloc(1, tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - ((SMsgHead *)buf)->vgId = htonl(pTask->nodeId); + + ((SMsgHead *)buf)->vgId = htonl(pTask->info.nodeId); + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncoderInit(&encoder, abuf, size); + tEncodeStreamTask(&encoder, pTask); tEncoderClear(&encoder); STransAction action = {0}; action.mTraceId = pTrans->mTraceId; - memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); + memcpy(&action.epSet, &pTask->info.epSet, sizeof(SEpSet)); action.pCont = buf; action.contLen = tlen; action.msgType = TDMT_STREAM_TASK_DEPLOY; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(buf); return -1; } + return 0; } @@ -459,14 +476,33 @@ int32_t mndPersistStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStrea int32_t level = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < level; i++) { SArray *pLevel = taosArrayGetP(pStream->tasks, i); - int32_t sz = taosArrayGetSize(pLevel); - for (int32_t j = 0; j < sz; j++) { + + int32_t numOfTasks = taosArrayGetSize(pLevel); + for (int32_t j = 0; j < numOfTasks; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); if (mndPersistTaskDeployReq(pTrans, pTask) < 0) { return -1; } } } + + // persistent stream task for already stored ts data + if (pStream->conf.fillHistory) { + level = taosArrayGetSize(pStream->pHTasksList); + + for (int32_t i = 0; i < level; i++) { + SArray *pLevel = taosArrayGetP(pStream->pHTasksList, i); + + int32_t numOfTasks = taosArrayGetSize(pLevel); + for (int32_t j = 0; j < numOfTasks; j++) { + SStreamTask *pTask = taosArrayGetP(pLevel, j); + if (mndPersistTaskDeployReq(pTrans, pTask) < 0) { + return -1; + } + } + } + } + return 0; } @@ -474,11 +510,13 @@ int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { if (mndPersistStreamTasks(pMnode, pTrans, pStream) < 0) { return -1; } + SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); return -1; } + (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); return 0; } @@ -487,9 +525,9 @@ int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); return -1; } + (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); return 0; } @@ -503,7 +541,6 @@ static int32_t mndSetStreamRecover(SMnode *pMnode, STrans *pTrans, const SStream if (pCommitRaw == NULL) return -1; if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); return -1; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); @@ -603,16 +640,19 @@ _OVER: static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { // vnode - /*if (pTask->nodeId > 0) {*/ + /*if (pTask->info.nodeId > 0) {*/ SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->head.vgId = htonl(pTask->nodeId); + + pReq->head.vgId = htonl(pTask->info.nodeId); pReq->taskId = pTask->id.taskId; + pReq->streamId = pTask->id.streamId; + STransAction action = {0}; - memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); + memcpy(&action.epSet, &pTask->info.epSet, sizeof(SEpSet)); action.pCont = pReq; action.contLen = sizeof(SVDropStreamTaskReq); action.msgType = TDMT_STREAM_TASK_DROP; @@ -700,16 +740,19 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { if (pStream->sourceDbUid == streamObj.sourceDbUid) { ++numOfStream; } + sdbRelease(pMnode->pSdb, pStream); if (numOfStream > MND_STREAM_MAX_NUM) { mError("too many streams, no more than %d for each database", MND_STREAM_MAX_NUM); terrno = TSDB_CODE_MND_TOO_MANY_STREAMS; + sdbCancelFetch(pMnode->pSdb, pIter); goto _OVER; } if (pStream->targetStbUid == streamObj.targetStbUid) { mError("Cannot write the same stable as other stream:%s", pStream->name); terrno = TSDB_CODE_MND_INVALID_TARGET_TABLE; + sdbCancelFetch(pMnode->pSdb, pIter); goto _OVER; } } @@ -723,6 +766,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { pDb = NULL; goto _OVER; } + mndReleaseDb(pMnode, pDb); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-stream"); @@ -730,10 +774,11 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); goto _OVER; } + mInfo("trans:%d, used to create stream:%s", pTrans->id, createStreamReq.name); mndTransSetDbName(pTrans, createStreamReq.sourceDB, streamObj.targetDb); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { mndTransDrop(pTrans); goto _OVER; } @@ -746,7 +791,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } // schedule stream task for stream obj - if (mndScheduleStream(pMnode, &streamObj) < 0) { + if (mndScheduleStream(pMnode, &streamObj, createStreamReq.lastTs) < 0) { mError("stream:%s, failed to schedule since %s", createStreamReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; @@ -832,7 +877,7 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, con SMStreamDoCheckpointMsg *pMsg) { SStreamCheckpointSourceReq req = {0}; req.checkpointId = pMsg->checkpointId; - req.nodeId = pTask->nodeId; + req.nodeId = pTask->info.nodeId; req.expireTime = -1; req.streamId = pTask->streamId; req.taskId = pTask->taskId; @@ -861,7 +906,7 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, con SMsgHead *pMsgHead = (SMsgHead *)buf; pMsgHead->contLen = htonl(tlen); - pMsgHead->vgId = htonl(pTask->nodeId); + pMsgHead->vgId = htonl(pTask->info.nodeId); tEncoderClear(&encoder); @@ -888,7 +933,7 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "stream-checkpoint"); if (pTrans == NULL) return -1; mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { mndReleaseStream(pMnode, pStream); mndTransDrop(pTrans); return -1; @@ -900,12 +945,12 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { for (int32_t i = 0; i < totLevel; i++) { SArray *pLevel = taosArrayGetP(pStream->tasks, i); SStreamTask *pTask = taosArrayGetP(pLevel, 0); - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { int32_t sz = taosArrayGetSize(pLevel); for (int32_t j = 0; j < sz; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); - /*A(pTask->nodeId > 0);*/ - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->nodeId); + /*A(pTask->info.nodeId > 0);*/ + SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); if (pVgObj == NULL) { taosRUnLockLatch(&pStream->lock); mndReleaseStream(pMnode, pStream); @@ -963,8 +1008,6 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; - /*SDbObj *pDb = NULL;*/ - /*SUserObj *pUser = NULL;*/ SMDropStreamReq dropReq = {0}; if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) { @@ -999,7 +1042,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { mInfo("trans:%d, used to drop stream:%s", pTrans->id, dropReq.name); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); return -1; @@ -1073,7 +1116,7 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return 0; } -static int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) { +int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) { SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); if (pDb == NULL) { @@ -1155,7 +1198,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pStream->watermark, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pStream->conf.watermark, false); char trigger[20 + VARSTR_HEADER_SIZE] = {0}; char trigger2[20] = {0}; @@ -1185,12 +1228,16 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock while (numOfRows < rowsCapacity) { pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream); - if (pShow->pIter == NULL) break; + if (pShow->pIter == NULL) { + break; + } // lock taosRLockLatch(&pStream->lock); + // count task num int32_t sz = taosArrayGetSize(pStream->tasks); + int32_t count = 0; for (int32_t i = 0; i < sz; i++) { SArray *pLevel = taosArrayGetP(pStream->tasks, i); @@ -1200,10 +1247,12 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock if (numOfRows + count > rowsCapacity) { blockDataEnsureCapacity(pBlock, numOfRows + count); } + // add row for each task for (int32_t i = 0; i < sz; i++) { SArray *pLevel = taosArrayGetP(pStream->tasks, i); int32_t levelCnt = taosArrayGetSize(pLevel); + for (int32_t j = 0; j < levelCnt; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); @@ -1213,18 +1262,25 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock // stream name char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName)); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)streamName, false); // task id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pTask->id.taskId, false); + + char idstr[128] = {0}; + int32_t len = tintToHex(pTask->id.taskId, &idstr[4]); + idstr[2] = '0'; + idstr[3] = 'x'; + varDataSetLen(idstr, len + 2); + colDataSetVal(pColInfo, numOfRows, idstr, false); // node type char nodeType[20 + VARSTR_HEADER_SIZE] = {0}; varDataSetLen(nodeType, 5); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - if (pTask->nodeId > 0) { + if (pTask->info.nodeId > 0) { memcpy(varDataVal(nodeType), "vnode", 5); } else { memcpy(varDataVal(nodeType), "snode", 5); @@ -1233,30 +1289,50 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock // node id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - int64_t nodeId = TMAX(pTask->nodeId, 0); + int64_t nodeId = TMAX(pTask->info.nodeId, 0); colDataSetVal(pColInfo, numOfRows, (const char *)&nodeId, false); // level char level[20 + VARSTR_HEADER_SIZE] = {0}; - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { memcpy(varDataVal(level), "source", 6); varDataSetLen(level, 6); - } else if (pTask->taskLevel == TASK_LEVEL__AGG) { + } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) { memcpy(varDataVal(level), "agg", 3); varDataSetLen(level, 3); - } else if (pTask->taskLevel == TASK_LEVEL__SINK) { + } else if (pTask->info.taskLevel == TASK_LEVEL__SINK) { memcpy(varDataVal(level), "sink", 4); varDataSetLen(level, 4); - } else if (pTask->taskLevel == TASK_LEVEL__SINK) { } + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&level, false); // status - char status[20 + VARSTR_HEADER_SIZE] = {0}; - char status2[20] = {0}; - strcpy(status, "normal"); - STR_WITH_MAXSIZE_TO_VARSTR(status, status2, sizeof(status)); + char status[20 + VARSTR_HEADER_SIZE] = {0}; + int8_t taskStatus = atomic_load_8(&pTask->status.taskStatus); + if (taskStatus == TASK_STATUS__NORMAL) { + memcpy(varDataVal(status), "normal", 6); + varDataSetLen(status, 6); + } else if (taskStatus == TASK_STATUS__DROPPING) { + memcpy(varDataVal(status), "dropping", 8); + varDataSetLen(status, 8); + } else if (taskStatus == TASK_STATUS__FAIL) { + memcpy(varDataVal(status), "fail", 4); + varDataSetLen(status, 4); + } else if (taskStatus == TASK_STATUS__STOP) { + memcpy(varDataVal(status), "stop", 4); + varDataSetLen(status, 4); + } else if (taskStatus == TASK_STATUS__SCAN_HISTORY) { + memcpy(varDataVal(status), "history", 7); + varDataSetLen(status, 7); + } else if (taskStatus == TASK_STATUS__HALT) { + memcpy(varDataVal(status), "halt", 4); + varDataSetLen(status, 4); + } else if (taskStatus == TASK_STATUS__PAUSE) { + memcpy(varDataVal(status), "pause", 5); + varDataSetLen(status, 5); + } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&status, false); @@ -1285,10 +1361,12 @@ static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->head.vgId = htonl(pTask->nodeId); + pReq->head.vgId = htonl(pTask->info.nodeId); pReq->taskId = pTask->id.taskId; + pReq->streamId = pTask->id.streamId; + STransAction action = {0}; - memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); + memcpy(&action.epSet, &pTask->info.epSet, sizeof(SEpSet)); action.pCont = pReq; action.contLen = sizeof(SVPauseStreamTaskReq); action.msgType = TDMT_STREAM_TASK_PAUSE; @@ -1299,21 +1377,36 @@ static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) { return 0; } -int32_t mndPauseAllStreamTasks(STrans *pTrans, SStreamObj *pStream) { - int32_t size = taosArrayGetSize(pStream->tasks); +int32_t mndPauseAllStreamTaskImpl(STrans *pTrans, SArray *tasks) { + int32_t size = taosArrayGetSize(tasks); for (int32_t i = 0; i < size; i++) { - SArray *pTasks = taosArrayGetP(pStream->tasks, i); + SArray *pTasks = taosArrayGetP(tasks, i); int32_t sz = taosArrayGetSize(pTasks); for (int32_t j = 0; j < sz; j++) { SStreamTask *pTask = taosArrayGetP(pTasks, j); - if (pTask->taskLevel == TASK_LEVEL__SOURCE && mndPauseStreamTask(pTrans, pTask) < 0) { + if (pTask->info.taskLevel != TASK_LEVEL__SINK && mndPauseStreamTask(pTrans, pTask) < 0) { return -1; } + + if (atomic_load_8(&pTask->status.taskStatus) != TASK_STATUS__PAUSE) { + atomic_store_8(&pTask->status.keepTaskStatus, pTask->status.taskStatus); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__PAUSE); + } } } return 0; } +int32_t mndPauseAllStreamTasks(STrans *pTrans, SStreamObj *pStream) { + int32_t code = mndPauseAllStreamTaskImpl(pTrans, pStream->tasks); + if (code != 0) { + return code; + } + // pStream->pHTasksList is null + // code = mndPauseAllStreamTaskImpl(pTrans, pStream->pHTasksList); + return code; +} + static int32_t mndPersistStreamLog(STrans *pTrans, const SStreamObj *pStream, int8_t status) { SStreamObj streamObj = {0}; memcpy(streamObj.name, pStream->name, TSDB_STREAM_FNAME_LEN); @@ -1323,7 +1416,6 @@ static int32_t mndPersistStreamLog(STrans *pTrans, const SStreamObj *pStream, in if (pCommitRaw == NULL) return -1; if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); return -1; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); @@ -1345,7 +1437,6 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { if (pStream == NULL) { if (pauseReq.igNotExists) { mInfo("stream:%s, not exist, if exist is set", pauseReq.name); - sdbRelease(pMnode->pSdb, pStream); return 0; } else { terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; @@ -1353,6 +1444,11 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { } } + if (pStream->status == STREAM_STATUS__PAUSE) { + sdbRelease(pMnode->pSdb, pStream); + return 0; + } + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { sdbRelease(pMnode->pSdb, pStream); return -1; @@ -1367,7 +1463,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { mInfo("trans:%d, used to pause stream:%s", pTrans->id, pauseReq.name); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); return -1; @@ -1401,18 +1497,19 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } - static int32_t mndResumeStreamTask(STrans *pTrans, SStreamTask *pTask, int8_t igUntreated) { SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq)); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->head.vgId = htonl(pTask->nodeId); + pReq->head.vgId = htonl(pTask->info.nodeId); pReq->taskId = pTask->id.taskId; + pReq->streamId = pTask->id.streamId; pReq->igUntreated = igUntreated; + STransAction action = {0}; - memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); + memcpy(&action.epSet, &pTask->info.epSet, sizeof(SEpSet)); action.pCont = pReq; action.contLen = sizeof(SVResumeStreamTaskReq); action.msgType = TDMT_STREAM_TASK_RESUME; @@ -1430,11 +1527,16 @@ int32_t mndResumeAllStreamTasks(STrans *pTrans, SStreamObj *pStream, int8_t igUn int32_t sz = taosArrayGetSize(pTasks); for (int32_t j = 0; j < sz; j++) { SStreamTask *pTask = taosArrayGetP(pTasks, j); - if (pTask->taskLevel == TASK_LEVEL__SOURCE && mndResumeStreamTask(pTrans, pTask, igUntreated) < 0) { + if (pTask->info.taskLevel != TASK_LEVEL__SINK && mndResumeStreamTask(pTrans, pTask, igUntreated) < 0) { return -1; } + + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__PAUSE) { + atomic_store_8(&pTask->status.taskStatus, pTask->status.keepTaskStatus); + } } } + // pStream->pHTasksList is null return 0; } @@ -1461,6 +1563,10 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { } } + if (pStream->status != STREAM_STATUS__PAUSE) { + return 0; + } + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { sdbRelease(pMnode->pSdb, pStream); return -1; @@ -1475,7 +1581,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { mInfo("trans:%d, used to pause stream:%s", pTrans->id, pauseReq.name); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); return -1; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e62102fa77254c37e0d8244ae437a9570f870dfb..53f22f6e60fae17d51ac55406b24efb4558e43f6 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -24,7 +24,7 @@ #include "tcompare.h" #include "tname.h" -#define MND_SUBSCRIBE_VER_NUMBER 1 +#define MND_SUBSCRIBE_VER_NUMBER 2 #define MND_SUBSCRIBE_RESERVE_SIZE 64 #define MND_SUBSCRIBE_REBALANCE_CNT 3 @@ -99,22 +99,41 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj return pSub; } -static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscribeObj *pSub, - const SMqRebOutputVg *pRebVg) { +static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, + const SMqRebOutputVg *pRebVg, SSubplan* pPlan) { SMqRebVgReq req = {0}; req.oldConsumerId = pRebVg->oldConsumerId; req.newConsumerId = pRebVg->newConsumerId; req.vgId = pRebVg->pVgEp->vgId; - req.qmsg = pRebVg->pVgEp->qmsg; + if(pPlan){ + pPlan->execNode.epSet = pRebVg->pVgEp->epSet; + pPlan->execNode.nodeId = pRebVg->pVgEp->vgId; + int32_t msgLen; + if (qSubPlanToString(pPlan, &req.qmsg, &msgLen) < 0) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + }else{ + req.qmsg = taosStrdup(""); + } req.subType = pSub->subType; req.withMeta = pSub->withMeta; req.suid = pSub->stbUid; tstrncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); - int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); + int32_t tlen = 0; + int32_t ret = 0; + tEncodeSize(tEncodeSMqRebVgReq, &req, tlen, ret); + if (ret < 0) { + taosMemoryFree(req.qmsg); + return -1; + } + + tlen += sizeof(SMsgHead); void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(req.qmsg); return -1; } @@ -123,24 +142,32 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri pMsgHead->contLen = htonl(tlen); pMsgHead->vgId = htonl(pRebVg->pVgEp->vgId); - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncodeSMqRebVgReq(&abuf, &req); + SEncoder encoder = {0}; + tEncoderInit(&encoder, POINTER_SHIFT(buf, sizeof(SMsgHead)), tlen); + if (tEncodeSMqRebVgReq(&encoder, &req) < 0) { + taosMemoryFreeClear(buf); + tEncoderClear(&encoder); + taosMemoryFree(req.qmsg); + return -1; + } + tEncoderClear(&encoder); *pBuf = buf; *pLen = tlen; + taosMemoryFree(req.qmsg); return 0; } -static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub, - const SMqRebOutputVg *pRebVg) { -// if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { -// terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; -// return -1; -// } +static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub, + const SMqRebOutputVg *pRebVg, SSubplan* pPlan) { + if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { + terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; + return -1; + } void *buf; int32_t tlen; - if (mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg) < 0) { + if (mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg, pPlan) < 0) { return -1; } @@ -148,7 +175,7 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SM SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); if (pVgObj == NULL) { taosMemoryFree(buf); - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST; return -1; } @@ -217,7 +244,7 @@ static void doRemoveLostConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, con SMqRebOutputVg outputVg = {.oldConsumerId = consumerId, .newConsumerId = -1, .pVgEp = pVgEp}; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("sub:%s mq re-balance remove vgId:%d from consumer:%" PRIx64, pSubKey, pVgEp->vgId, consumerId); + mInfo("sub:%s mq re-balance remove vgId:%d from consumer:0x%" PRIx64, pSubKey, pVgEp->vgId, consumerId); } taosArrayDestroy(pConsumerEp->vgs); @@ -242,13 +269,13 @@ static void doAddNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pI for (int32_t i = 0; i < numOfNewConsumers; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); - SMqConsumerEp newConsumerEp; + SMqConsumerEp newConsumerEp = {0}; newConsumerEp.consumerId = consumerId; newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); taosArrayPush(pOutput->newConsumers, &consumerId); - mInfo("sub:%s mq rebalance add new consumer:%" PRIx64, pSubKey, consumerId); + mInfo("sub:%s mq rebalance add new consumer:0x%" PRIx64, pSubKey, consumerId); } } @@ -269,17 +296,17 @@ static void addUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) { } } -static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){ - for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){ - SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i); - SMqRebOutputVg outputVg = { - .oldConsumerId = pConsumerEp->consumerId, - .newConsumerId = pConsumerEp->consumerId, - .pVgEp = pVgEp, - }; - taosArrayPush(pOutput->rebVgs, &outputVg); - } -} +//static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){ +// for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){ +// SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i); +// SMqRebOutputVg outputVg = { +// .oldConsumerId = pConsumerEp->consumerId, +// .newConsumerId = pConsumerEp->consumerId, +// .pVgEp = pVgEp, +// }; +// taosArrayPush(pOutput->rebVgs, &outputVg); +// } +//} static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt, int32_t imbConsumerNum) { @@ -330,7 +357,7 @@ static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHas } } } - putNoTransferToOutput(pOutput, pConsumerEp); +// putNoTransferToOutput(pOutput, pConsumerEp); } } @@ -436,11 +463,58 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR SMqRebOutputVg* pRebOutput = (SMqRebOutputVg *)pRemovedIter; taosArrayPush(pOutput->rebVgs, pRebOutput); - if(taosHashGetSize(pOutput->pSub->consumerHash) == 0){ // if all consumer is removed, put all vg into unassigned - taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); + if(taosHashGetSize(pOutput->pSub->consumerHash) == 0){ // if all consumer is removed + taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); // put all vg into unassigned } } +// if(taosHashGetSize(pOutput->pSub->consumerHash) == 0) { // if all consumer is removed + SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key); // put all offset rows + if (pSub) { + taosRLockLatch(&pSub->lock); + if (pOutput->pSub->offsetRows == NULL) { + pOutput->pSub->offsetRows = taosArrayInit(4, sizeof(OffsetRows)); + } + pIter = NULL; + while (1) { + pIter = taosHashIterate(pSub->consumerHash, pIter); + if (pIter == NULL) break; + SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; + SMqConsumerEp *pConsumerEpNew = taosHashGet(pOutput->pSub->consumerHash, &pConsumerEp->consumerId, sizeof(int64_t)); + + for (int j = 0; j < taosArrayGetSize(pConsumerEp->offsetRows); j++) { + OffsetRows *d1 = taosArrayGet(pConsumerEp->offsetRows, j); + bool jump = false; + for (int i = 0; pConsumerEpNew && i < taosArrayGetSize(pConsumerEpNew->vgs); i++){ + SMqVgEp *pVgEp = taosArrayGetP(pConsumerEpNew->vgs, i); + if(pVgEp->vgId == d1->vgId){ + jump = true; + mInfo("pSub->offsetRows jump, because consumer id:%"PRIx64 " and vgId:%d not change", pConsumerEp->consumerId, pVgEp->vgId); + break; + } + } + if(jump) continue; + bool find = false; + for (int i = 0; i < taosArrayGetSize(pOutput->pSub->offsetRows); i++) { + OffsetRows *d2 = taosArrayGet(pOutput->pSub->offsetRows, i); + if (d1->vgId == d2->vgId) { + d2->rows += d1->rows; + d2->offset = d1->offset; + find = true; + mInfo("pSub->offsetRows add vgId:%d, after:%"PRId64", before:%"PRId64, d2->vgId, d2->rows, d1->rows); + break; + } + } + if(!find){ + taosArrayPush(pOutput->pSub->offsetRows, d1); + } + } + } + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); +// } + } + // 8. generate logs mInfo("sub:%s mq re-balance calculation completed, re-balanced vg", pSubKey); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { @@ -470,14 +544,25 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) { + struct SSubplan* pPlan = NULL; + if(strcmp(pOutput->pSub->qmsg, "") != 0){ + int32_t code = qStringToSubplan(pOutput->pSub->qmsg, &pPlan); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return -1; + } + } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb"); if (pTrans == NULL) { + nodesDestroyNode((SNode*)pPlan); return -1; } mndTransSetDbName(pTrans, pOutput->pSub->dbName, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { mndTransDrop(pTrans); + nodesDestroyNode((SNode*)pPlan); return -1; } @@ -487,11 +572,13 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu int32_t vgNum = taosArrayGetSize(rebVgs); for (int32_t i = 0; i < vgNum; i++) { SMqRebOutputVg *pRebVg = taosArrayGet(rebVgs, i); - if (mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub, pRebVg) < 0) { + if (mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub, pRebVg, pPlan) < 0) { mndTransDrop(pTrans); + nodesDestroyNode((SNode*)pPlan); return -1; } } + nodesDestroyNode((SNode*)pPlan); // 2. redo log: subscribe and vg assignment // subscribe @@ -500,50 +587,44 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu return -1; } + char topic[TSDB_TOPIC_FNAME_LEN] = {0}; + char cgroup[TSDB_CGROUP_LEN] = {0}; + mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true); + // 3. commit log: consumer to update status and epoch // 3.1 set touched consumer int32_t consumerNum = taosArrayGetSize(pOutput->modifyConsumers); for (int32_t i = 0; i < consumerNum; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->modifyConsumers, i); - SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); - SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); - pConsumerNew->updateType = CONSUMER_UPDATE__TOUCH; - mndReleaseConsumer(pMnode, pConsumerOld); + SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); + pConsumerNew->updateType = CONSUMER_UPDATE_REB; if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { - tDeleteSMqConsumerObj(pConsumerNew); - taosMemoryFree(pConsumerNew); + tDeleteSMqConsumerObj(pConsumerNew, true); mndTransDrop(pTrans); return -1; } - tDeleteSMqConsumerObj(pConsumerNew); - taosMemoryFree(pConsumerNew); + tDeleteSMqConsumerObj(pConsumerNew, true); } // 3.2 set new consumer consumerNum = taosArrayGetSize(pOutput->newConsumers); for (int32_t i = 0; i < consumerNum; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->newConsumers, i); + SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); + pConsumerNew->updateType = CONSUMER_ADD_REB; - SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); - SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); - pConsumerNew->updateType = CONSUMER_UPDATE__ADD; - char *topic = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); - char cgroup[TSDB_CGROUP_LEN]; - mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true); - taosArrayPush(pConsumerNew->rebNewTopics, &topic); - mndReleaseConsumer(pMnode, pConsumerOld); + char* topicTmp = taosStrdup(topic); + taosArrayPush(pConsumerNew->rebNewTopics, &topicTmp); if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { - tDeleteSMqConsumerObj(pConsumerNew); - taosMemoryFree(pConsumerNew); + tDeleteSMqConsumerObj(pConsumerNew, true); mndTransDrop(pTrans); return -1; } - tDeleteSMqConsumerObj(pConsumerNew); - taosMemoryFree(pConsumerNew); + tDeleteSMqConsumerObj(pConsumerNew, true); } // 3.3 set removed consumer @@ -551,24 +632,19 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu for (int32_t i = 0; i < consumerNum; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->removedConsumers, i); - SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); - SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); - pConsumerNew->updateType = CONSUMER_UPDATE__REMOVE; - char *topic = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); - char cgroup[TSDB_CGROUP_LEN]; - mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true); - taosArrayPush(pConsumerNew->rebRemovedTopics, &topic); - mndReleaseConsumer(pMnode, pConsumerOld); + SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); + pConsumerNew->updateType = CONSUMER_REMOVE_REB; + + char* topicTmp = taosStrdup(topic); + taosArrayPush(pConsumerNew->rebRemovedTopics, &topicTmp); if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { - tDeleteSMqConsumerObj(pConsumerNew); - taosMemoryFree(pConsumerNew); + tDeleteSMqConsumerObj(pConsumerNew, true); mndTransDrop(pTrans); return -1; } - tDeleteSMqConsumerObj(pConsumerNew); - taosMemoryFree(pConsumerNew); + tDeleteSMqConsumerObj(pConsumerNew, true); } // 4. TODO commit log: modification log @@ -616,6 +692,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { taosArrayDestroy(rebOutput.modifyConsumers); taosArrayDestroy(rebOutput.rebVgs); + taosHashCancelIterate(pReq->rebSubHash, pIter); terrno = TSDB_CODE_OUT_OF_MEMORY; mInfo("mq re-balance failed, due to out of memory"); taosHashCleanup(pReq->rebSubHash); @@ -695,8 +772,10 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { } static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; - SMDropCgroupReq dropReq = {0}; + SMnode *pMnode = pMsg->info.node; + SMDropCgroupReq dropReq = {0}; + STrans *pTrans = NULL; + int32_t code = TSDB_CODE_ACTION_IN_PROGRESS; if (tDeserializeSMDropCgroupReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -715,38 +794,54 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { } } + taosWLockLatch(&pSub->lock); if (taosHashGetSize(pSub->consumerHash) != 0) { terrno = TSDB_CODE_MND_CGROUP_USED; mError("cgroup:%s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); - mndReleaseSubscribe(pMnode, pSub); - return -1; + code = -1; + goto end; + } + + void *pIter = NULL; + SMqConsumerObj *pConsumer; + while (1) { + pIter = sdbFetch(pMnode->pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); + if (pIter == NULL) { + break; + } + + if (strcmp(dropReq.cgroup, pConsumer->cgroup) == 0) { + mndDropConsumerFromSdb(pMnode, pConsumer->consumerId); + } + sdbRelease(pMnode->pSdb, pConsumer); } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "drop-cgroup"); + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "drop-cgroup"); if (pTrans == NULL) { mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); - mndReleaseSubscribe(pMnode, pSub); - mndTransDrop(pTrans); - return -1; + code = -1; + goto end; } mInfo("trans:%d, used to drop cgroup:%s on topic %s", pTrans->id, dropReq.cgroup, dropReq.topic); if (mndSetDropSubCommitLogs(pMnode, pTrans, pSub) < 0) { mError("cgroup %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); - mndReleaseSubscribe(pMnode, pSub); - mndTransDrop(pTrans); - return -1; + code = -1; + goto end; } if (mndTransPrepare(pMnode, pTrans) < 0) { - mndReleaseSubscribe(pMnode, pSub); - mndTransDrop(pTrans); - return -1; + code = -1; + goto end; } + +end: + taosWUnLockLatch(&pSub->lock); mndReleaseSubscribe(pMnode, pSub); + mndTransDrop(pTrans); - return TSDB_CODE_ACTION_IN_PROGRESS; + return code; } void mndCleanupSubscribe(SMnode *pMnode) {} @@ -796,7 +891,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto SUB_DECODE_OVER; - if (sver != MND_SUBSCRIBE_VER_NUMBER) { + if (sver > MND_SUBSCRIBE_VER_NUMBER || sver < 1) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto SUB_DECODE_OVER; } @@ -815,7 +910,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER); - if (tDecodeSubscribeObj(buf, pSub) == NULL) { + if (tDecodeSubscribeObj(buf, pSub, sver) == NULL) { goto SUB_DECODE_OVER; } @@ -877,6 +972,10 @@ static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubsc pOldSub->unassignedVgs = pNewSub->unassignedVgs; pNewSub->unassignedVgs = tmp1; + SArray *tmp2 = pOldSub->offsetRows; + pOldSub->offsetRows = pNewSub->offsetRows; + pNewSub->offsetRows = tmp2; + taosWUnLockLatch(&pOldSub->lock); return 0; } @@ -909,6 +1008,32 @@ SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key) { return pSub; } +int32_t mndGetGroupNumByTopic(SMnode *pMnode, const char *topicName) { + int32_t num = 0; + SSdb *pSdb = pMnode->pSdb; + + void *pIter = NULL; + SMqSubscribeObj *pSub = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pSub); + if (pIter == NULL) break; + + + char topic[TSDB_TOPIC_FNAME_LEN]; + char cgroup[TSDB_CGROUP_LEN]; + mndSplitSubscribeKey(pSub->key, topic, cgroup, true); + if (strcmp(topic, topicName) != 0) { + sdbRelease(pSdb, pSub); + continue; + } + + num++; + sdbRelease(pSdb, pSub); + } + + return num; +} + void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) { SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pSub); @@ -980,6 +1105,7 @@ int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName) if (taosHashGetSize(pSub->consumerHash) != 0) { sdbRelease(pSdb, pSub); terrno = TSDB_CODE_MND_IN_REBALANCE; + sdbCancelFetch(pSdb, pIter); return -1; } int32_t sz = taosArrayGetSize(pSub->unassignedVgs); @@ -998,12 +1124,14 @@ int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName) if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); sdbRelease(pSdb, pSub); + sdbCancelFetch(pSdb, pIter); return -1; } } if (mndSetDropSubRedoLogs(pMnode, pTrans, pSub) < 0) { sdbRelease(pSdb, pSub); + sdbCancelFetch(pSdb, pIter); goto END; } @@ -1015,13 +1143,72 @@ END: return code; } +static int32_t buildResult(SSDataBlock *pBlock, int32_t* numOfRows, int64_t consumerId, const char* topic, const char* cgroup, SArray* vgs, SArray *offsetRows){ + int32_t sz = taosArrayGetSize(vgs); + for (int32_t j = 0; j < sz; j++) { + SMqVgEp *pVgEp = taosArrayGetP(vgs, j); + + SColumnInfoData *pColInfo; + int32_t cols = 0; + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)topic, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)cgroup, false); + + // vg id + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)&pVgEp->vgId, false); + + // consumer id + char consumerIdHex[32] = {0}; + sprintf(varDataVal(consumerIdHex), "0x%"PRIx64, consumerId); + varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex))); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)consumerIdHex, consumerId == -1); + + mInfo("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic), + consumerId, varDataVal(cgroup), pVgEp->vgId); + + // offset + OffsetRows *data = NULL; + for(int i = 0; i < taosArrayGetSize(offsetRows); i++){ + OffsetRows *tmp = taosArrayGet(offsetRows, i); + if(tmp->vgId != pVgEp->vgId){ + continue; + } + data = tmp; + } + if(data){ + // vg id + char buf[TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0}; + tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &data->offset); + varDataSetLen(buf, strlen(varDataVal(buf))); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)buf, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)&data->rows, false); + }else{ + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetNULL(pColInfo, *numOfRows); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetNULL(pColInfo, *numOfRows); + mError("mnd show subscriptions: do not find vgId:%d in offsetRows", pVgEp->vgId); + } + (*numOfRows)++; + } + return 0; +} + int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SMqSubscribeObj *pSub = NULL; - mDebug("mnd show subscriptions begin"); + mInfo("mnd show subscriptions begin"); while (numOfRows < rowsCapacity) { pShow->pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pShow->pIter, (void **)&pSub); @@ -1035,6 +1222,13 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock blockDataEnsureCapacity(pBlock, numOfRows + pSub->vgNum); } + // topic and cgroup + char topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; + mndSplitSubscribeKey(pSub->key, varDataVal(topic), varDataVal(cgroup), false); + varDataSetLen(topic, strlen(varDataVal(topic))); + varDataSetLen(cgroup, strlen(varDataVal(cgroup))); + SMqConsumerEp *pConsumerEp = NULL; void *pIter = NULL; while (1) { @@ -1042,97 +1236,11 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock if (pIter == NULL) break; pConsumerEp = (SMqConsumerEp *)pIter; - int32_t sz = taosArrayGetSize(pConsumerEp->vgs); - for (int32_t j = 0; j < sz; j++) { - SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); - - SColumnInfoData *pColInfo; - int32_t cols = 0; - - // topic and cgroup - char topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; - mndSplitSubscribeKey(pSub->key, varDataVal(topic), varDataVal(cgroup), false); - varDataSetLen(topic, strlen(varDataVal(topic))); - varDataSetLen(cgroup, strlen(varDataVal(cgroup))); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)topic, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false); - - // vg id - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pVgEp->vgId, false); - - // consumer id - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumerEp->consumerId, false); - - mDebug("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic), - pConsumerEp->consumerId, varDataVal(cgroup), pVgEp->vgId); - - // offset -#if 0 - // subscribe time - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false); - - // rebalance time - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0); -#endif - - numOfRows++; - } + buildResult(pBlock, &numOfRows, pConsumerEp->consumerId, topic, cgroup, pConsumerEp->vgs, pConsumerEp->offsetRows); } // do not show for cleared subscription - int32_t sz = taosArrayGetSize(pSub->unassignedVgs); - for (int32_t i = 0; i < sz; i++) { - SMqVgEp *pVgEp = taosArrayGetP(pSub->unassignedVgs, i); - - SColumnInfoData *pColInfo; - int32_t cols = 0; - - // topic and cgroup - char topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; - mndSplitSubscribeKey(pSub->key, varDataVal(topic), varDataVal(cgroup), false); - varDataSetLen(topic, strlen(varDataVal(topic))); - varDataSetLen(cgroup, strlen(varDataVal(cgroup))); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)topic, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false); - - // vg id - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pVgEp->vgId, false); - - // consumer id - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, NULL, true); - - mDebug("mnd show subscriptions(unassigned): topic %s, cgroup %s vgid %d", varDataVal(topic), varDataVal(cgroup), - pVgEp->vgId); - - // offset -#if 0 - // subscribe time - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false); - - // rebalance time - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0); -#endif - - numOfRows++; - } + buildResult(pBlock, &numOfRows, -1, topic, cgroup, pSub->unassignedVgs, pSub->offsetRows); pBlock->info.rows = numOfRows; @@ -1140,7 +1248,7 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock sdbRelease(pSdb, pSub); } - mDebug("mnd end show subscriptions"); + mInfo("mnd end show subscriptions"); pShow->numOfRows += numOfRows; return numOfRows; diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 0a6df02f5fbf2636de06358831b577917846b797..68bfe09b5eb1e7aff413be23d44fd5167fd7fd87 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -17,6 +17,7 @@ #include "mndSync.h" #include "mndCluster.h" #include "mndTrans.h" +#include "mndVgroup.h" static int32_t mndSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { if (pMsg == NULL || pMsg->pCont == NULL) { @@ -73,76 +74,200 @@ static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { return code; } -int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { - SMnode *pMnode = pFsm->data; +static int32_t mndTransValidatePrepareAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { + SSdbRow *pRow = NULL; + int32_t code = -1; + + if (pAction->msgType == TDMT_MND_CREATE_VG) { + pRow = mndVgroupActionDecode(pAction->pRaw); + if (pRow == NULL) goto _OUT; + + SVgObj *pVgroup = sdbGetRowObj(pRow); + if (pVgroup == NULL) goto _OUT; + + int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); + if (maxVgId > pVgroup->vgId) { + mError("trans:%d, failed to satisfy vgroup id %d of prepare action. maxVgId:%d", pTrans->id, pVgroup->vgId, + maxVgId); + goto _OUT; + } + } + + code = 0; +_OUT: + taosMemoryFreeClear(pRow); + return code; +} + +static int32_t mndTransValidatePrepareStage(SMnode *pMnode, STrans *pTrans) { + int32_t code = -1; + int32_t action = 0; + + int32_t numOfActions = taosArrayGetSize(pTrans->prepareActions); + if (numOfActions == 0) { + code = 0; + goto _OUT; + } + + mInfo("trans:%d, validate %d prepare actions.", pTrans->id, numOfActions); + + for (action = 0; action < numOfActions; ++action) { + STransAction *pAction = taosArrayGet(pTrans->prepareActions, action); + + if (pAction->actionType != TRANS_ACTION_RAW) { + mError("trans:%d, prepare action:%d of unexpected type:%d", pTrans->id, action, pAction->actionType); + goto _OUT; + } + + code = mndTransValidatePrepareAction(pMnode, pTrans, pAction); + if (code != 0) { + mError("trans:%d, failed to validate prepare action: %d, numOfActions:%d", pTrans->id, action, numOfActions); + goto _OUT; + } + } + + code = 0; +_OUT: + return code; +} + +static int32_t mndTransValidateImp(SMnode *pMnode, STrans *pTrans) { + if (pTrans->stage == TRN_STAGE_PREPARE) { + if (mndTransCheckConflict(pMnode, pTrans) < 0) { + mError("trans:%d, failed to validate trans conflicts.", pTrans->id); + return -1; + } + + return mndTransValidatePrepareStage(pMnode, pTrans); + } + return 0; +} + +static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) { + STrans *pTrans = NULL; + int32_t code = -1; + + SSdbRow *pRow = mndTransDecode(pRaw); + if (pRow == NULL) goto _OUT; + + pTrans = sdbGetRowObj(pRow); + if (pTrans == NULL) goto _OUT; + + code = mndTransValidateImp(pMnode, pTrans); + +_OUT: + if (pTrans) mndTransDropData(pTrans); + if (pRow) taosMemoryFreeClear(pRow); + if (code) terrno = (terrno ? terrno : TSDB_CODE_MND_TRANS_CONFLICT); + return code; +} + +int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { + terrno = TSDB_CODE_SUCCESS; SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSdbRaw *pRaw = pMsg->pCont; - + STrans *pTrans = NULL; + int32_t code = -1; int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw); + + if (transId <= 0) { + mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq); + terrno = TSDB_CODE_INVALID_MSG; + goto _OUT; + } + mInfo("trans:%d, is proposed, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64 " role:%s raw:%p sec:%d seq:%" PRId64, transId, pMgmt->transId, pMeta->code, pMeta->index, pMeta->term, pMeta->lastConfigIndex, syncStr(pMeta->state), pRaw, pMgmt->transSec, pMgmt->transSeq); - if (pMeta->code == 0) { - int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pRaw); - if (code != 0) { - mError("trans:%d, failed to write to sdb since %s", transId, terrstr()); - return 0; - } - sdbSetApplyInfo(pMnode->pSdb, pMeta->index, pMeta->term, pMeta->lastConfigIndex); + code = mndTransValidate(pMnode, pRaw); + if (code != 0) { + mError("trans:%d, failed to validate requested trans since %s", transId, terrstr()); + code = 0; + pMeta->code = terrno; + goto _OUT; + } + + code = sdbWriteWithoutFree(pMnode->pSdb, pRaw); + if (code != 0) { + mError("trans:%d, failed to write to sdb since %s", transId, terrstr()); + code = 0; + pMeta->code = terrno; + goto _OUT; + } + + pTrans = mndAcquireTrans(pMnode, transId); + if (pTrans == NULL) { + mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); + goto _OUT; } + if (pTrans->stage == TRN_STAGE_PREPARE) { + bool continueExec = mndTransPerformPrepareStage(pMnode, pTrans); + if (!continueExec) goto _OUT; + } + + if (pTrans->id != pMgmt->transId) { + mInfo("trans:%d, execute in mnode which not leader or sync timeout, createTime:%" PRId64 " saved trans:%d", + pTrans->id, pTrans->createdTime, pMgmt->transId); + mndTransRefresh(pMnode, pTrans); + } + + sdbSetApplyInfo(pMnode->pSdb, pMeta->index, pMeta->term, pMeta->lastConfigIndex); + sdbWriteFile(pMnode->pSdb, tsMndSdbWriteDelta); + code = 0; + +_OUT: + if (pTrans) mndReleaseTrans(pMnode, pTrans); + return code; +} + +static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) { + SSyncMgmt *pMgmt = &pMnode->syncMgmt; taosThreadMutexLock(&pMgmt->lock); - pMgmt->errCode = pMeta->code; + if (pMgmt->transId == 0) { + goto _OUT; + } - if (transId <= 0) { - taosThreadMutexUnlock(&pMgmt->lock); - mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq); - } else if (transId == pMgmt->transId) { - if (pMgmt->errCode != 0) { - mError("trans:%d, failed to propose since %s, post sem", transId, tstrerror(pMgmt->errCode)); - } else { - mInfo("trans:%d, is proposed and post sem, seq:%" PRId64, transId, pMgmt->transSeq); - } - pMgmt->transId = 0; - pMgmt->transSec = 0; - pMgmt->transSeq = 0; - tsem_post(&pMgmt->syncSem); - taosThreadMutexUnlock(&pMgmt->lock); + pMgmt->transId = 0; + pMgmt->transSec = 0; + pMgmt->transSeq = 0; + pMgmt->errCode = code; + tsem_post(&pMgmt->syncSem); + + if (pMgmt->errCode != 0) { + mError("trans:%d, failed to propose since %s, post sem", pMgmt->transId, tstrerror(pMgmt->errCode)); } else { - taosThreadMutexUnlock(&pMgmt->lock); - STrans *pTrans = mndAcquireTrans(pMnode, transId); - if (pTrans != NULL) { - mInfo("trans:%d, execute in mnode which not leader or sync timeout, createTime:%" PRId64 " saved trans:%d", - transId, pTrans->createdTime, pMgmt->transId); - mndTransExecute(pMnode, pTrans, false); - mndReleaseTrans(pMnode, pTrans); - } else { - mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); - } + mInfo("trans:%d, is proposed and post sem, seq:%" PRId64, pMgmt->transId, pMgmt->transSeq); } - sdbWriteFile(pMnode->pSdb, tsMndSdbWriteDelta); +_OUT: + taosThreadMutexUnlock(&pMgmt->lock); return 0; } -int32_t mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { - int32_t code = 0; +int32_t mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { + SMnode *pMnode = pFsm->data; + int32_t code = pMsg->code; + if (code != 0) { + goto _OUT; + } + pMsg->info.conn.applyIndex = pMeta->index; pMsg->info.conn.applyTerm = pMeta->term; + pMeta->code = 0; - if (pMsg->code == 0) { - SMnode *pMnode = pFsm->data; - atomic_store_64(&pMnode->applied, pMsg->info.conn.applyIndex); - } + atomic_store_64(&pMnode->applied, pMsg->info.conn.applyIndex); if (!syncUtilUserCommit(pMsg->msgType)) { - goto _out; + goto _OUT; } - code = mndProcessWriteMsg(pFsm, pMsg, pMeta); -_out: + code = mndProcessWriteMsg(pMnode, pMsg, pMeta); + +_OUT: + mndPostMgmtCode(pMnode, code ? code : pMeta->code); rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; return code; diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 679fafa28d89e398c1e5900f354925a50c38c06a..ac379a9f9478d579f684263b5a6d73c0f397c922 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -94,7 +94,7 @@ static char* mndBuildTelemetryReport(SMnode* pMnode) { tjsonAddStringToObject(pJson, "instanceId", clusterName); tjsonAddDoubleToObject(pJson, "reportVersion", 1); - if (taosGetOsReleaseName(tmp, sizeof(tmp)) == 0) { + if (taosGetOsReleaseName(tmp, NULL, NULL, sizeof(tmp)) == 0) { tjsonAddStringToObject(pJson, "os", tmp); } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index f6da3709164158acc008708121873da5bb2b78d1..621a80338d95c60d1e271888a404de6a14aa8ad9 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -28,7 +28,7 @@ #include "parser.h" #include "tname.h" -#define MND_TOPIC_VER_NUMBER 2 +#define MND_TOPIC_VER_NUMBER 3 #define MND_TOPIC_RESERVE_SIZE 64 SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic); @@ -109,6 +109,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_INT8(pRaw, dataPos, pTopic->withMeta, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->stbUid, TOPIC_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, pTopic->stbName, TSDB_TABLE_FNAME_LEN, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->astLen, TOPIC_ENCODE_OVER); @@ -169,7 +170,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TOPIC_DECODE_OVER; - if (sver != 1 && sver != 2) { + if (sver < 1 || sver > MND_TOPIC_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto TOPIC_DECODE_OVER; } @@ -196,6 +197,9 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pTopic->withMeta, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->stbUid, TOPIC_DECODE_OVER); + if (sver >= 3) { + SDB_GET_BINARY(pRaw, dataPos, pTopic->stbName, TSDB_TABLE_FNAME_LEN, TOPIC_DECODE_OVER); + } SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char)); if (pTopic->sql == NULL) { @@ -373,6 +377,10 @@ static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) { static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb, const char *userName) { mInfo("start to create topic:%s", pCreate->name); + STrans *pTrans = NULL; + int32_t code = -1; + SNode *pAst = NULL; + SQueryPlan *pPlan = NULL; SMqTopicObj topicObj = {0}; tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN); @@ -397,7 +405,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * if (pCreate->withMeta) { terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto _OUT; } topicObj.ast = taosStrdup(pCreate->ast); @@ -405,30 +413,21 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * qDebugL("topic:%s ast %s", topicObj.name, topicObj.ast); - SNode *pAst = NULL; if (nodesStringToNode(pCreate->ast, &pAst) != 0) { - taosMemoryFree(topicObj.ast); - taosMemoryFree(topicObj.sql); mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + goto _OUT; } - SQueryPlan *pPlan = NULL; - SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) { mError("failed to create topic:%s since %s", pCreate->name, terrstr()); - taosMemoryFree(topicObj.ast); - taosMemoryFree(topicObj.sql); - return -1; + goto _OUT; } topicObj.ntbColIds = taosArrayInit(0, sizeof(int16_t)); if (topicObj.ntbColIds == NULL) { - taosMemoryFree(topicObj.ast); - taosMemoryFree(topicObj.sql); terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + goto _OUT; } extractTopicTbInfo(pAst, &topicObj); @@ -440,28 +439,28 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * if (qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema) != 0) { mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - taosMemoryFree(topicObj.ast); - taosMemoryFree(topicObj.sql); - return -1; + goto _OUT; } if (nodesNodeToString((SNode *)pPlan, false, &topicObj.physicalPlan, NULL) != 0) { mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - taosMemoryFree(topicObj.ast); - taosMemoryFree(topicObj.sql); - return -1; + goto _OUT; } - nodesDestroyNode(pAst); - nodesDestroyNode((SNode *)pPlan); } else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) { SStbObj *pStb = mndAcquireStb(pMnode, pCreate->subStbName); if (pStb == NULL) { terrno = TSDB_CODE_MND_STB_NOT_EXIST; - return -1; + goto _OUT; } + strcpy(topicObj.stbName, pCreate->subStbName); topicObj.stbUid = pStb->uid; mndReleaseStb(pMnode, pStb); + if(pCreate->ast != NULL){ + qDebugL("topic:%s ast %s", topicObj.name, pCreate->ast); + topicObj.ast = taosStrdup(pCreate->ast); + topicObj.astLen = strlen(pCreate->ast) + 1; + } } /*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/ /*topicObj.ast = NULL;*/ @@ -470,23 +469,22 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * /*topicObj.withTbName = 1;*/ /*topicObj.withSchema = 1;*/ - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-topic"); + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-topic"); if (pTrans == NULL) { mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - taosMemoryFreeClear(topicObj.ast); - taosMemoryFreeClear(topicObj.sql); - taosMemoryFreeClear(topicObj.physicalPlan); - return -1; + goto _OUT; } + mndTransSetDbName(pTrans, pDb->name, NULL); + if (mndTransCheckConflict(pMnode, pTrans) != 0) { + goto _OUT; + } mInfo("trans:%d to create topic:%s", pTrans->id, pCreate->name); SSdbRaw *pCommitRaw = mndTopicActionEncode(&topicObj); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - taosMemoryFreeClear(topicObj.physicalPlan); - mndTransDrop(pTrans); - return -1; + goto _OUT; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); @@ -515,17 +513,18 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * tEncodeSize(tEncodeSTqCheckInfo, &info, len, code); if (code < 0) { sdbRelease(pSdb, pVgroup); - mndTransDrop(pTrans); - return -1; + sdbCancelFetch(pSdb, pIter); + goto _OUT; } void *buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); SEncoder encoder; tEncoderInit(&encoder, abuf, len); if (tEncodeSTqCheckInfo(&encoder, &info) < 0) { + taosMemoryFree(buf); sdbRelease(pSdb, pVgroup); - mndTransDrop(pTrans); - return -1; + sdbCancelFetch(pSdb, pIter); + goto _OUT; } tEncoderClear(&encoder); ((SMsgHead *)buf)->vgId = htonl(pVgroup->vgId); @@ -538,32 +537,33 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(buf); sdbRelease(pSdb, pVgroup); - mndTransDrop(pTrans); - return -1; + sdbCancelFetch(pSdb, pIter); + goto _OUT; } - + buf = NULL; sdbRelease(pSdb, pVgroup); } } if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - taosMemoryFreeClear(topicObj.physicalPlan); - mndTransDrop(pTrans); - return -1; + goto _OUT; } + code = TSDB_CODE_ACTION_IN_PROGRESS; + +_OUT: taosMemoryFreeClear(topicObj.physicalPlan); taosMemoryFreeClear(topicObj.sql); taosMemoryFreeClear(topicObj.ast); taosArrayDestroy(topicObj.ntbColIds); - if (topicObj.schema.nCols) { taosMemoryFreeClear(topicObj.schema.pSchema); } - + nodesDestroyNode(pAst); + nodesDestroyNode((SNode *)pPlan); mndTransDrop(pTrans); - return TSDB_CODE_ACTION_IN_PROGRESS; + return code; } static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { @@ -572,6 +572,11 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { SMqTopicObj *pTopic = NULL; SDbObj *pDb = NULL; SCMCreateTopicReq createTopicReq = {0}; + if (sdbGetSize(pMnode->pSdb, SDB_TOPIC) >= tmqMaxTopicNum){ + terrno = TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE; + mError("topic num out of range"); + return code; + } if (tDeserializeSCMCreateTopicReq(pReq->pCont, pReq->contLen, &createTopicReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -645,7 +650,6 @@ static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTo code = 0; _OVER: - mndTransDrop(pTrans); return code; } @@ -684,7 +688,11 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { break; } - if (pConsumer->status == MQ_CONSUMER_STATUS__LOST_REBD) continue; + if (pConsumer->status == MQ_CONSUMER_STATUS_LOST){ + mndDropConsumerFromSdb(pMnode, pConsumer->consumerId); + mndReleaseConsumer(pMnode, pConsumer); + continue; + } int32_t sz = taosArrayGetSize(pConsumer->assignedTopics); for (int32_t i = 0; i < sz; i++) { @@ -692,6 +700,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { if (strcmp(name, pTopic->name) == 0) { mndReleaseConsumer(pMnode, pConsumer); mndReleaseTopic(pMnode, pTopic); + sdbCancelFetch(pSdb, pIter); terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s", dropReq.name, pConsumer->consumerId, pConsumer->cgroup); @@ -705,6 +714,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { if (strcmp(name, pTopic->name) == 0) { mndReleaseConsumer(pMnode, pConsumer); mndReleaseTopic(pMnode, pTopic); + sdbCancelFetch(pSdb, pIter); terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb new)", dropReq.name, pConsumer->consumerId, pConsumer->cgroup); @@ -718,6 +728,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { if (strcmp(name, pTopic->name) == 0) { mndReleaseConsumer(pMnode, pConsumer); mndReleaseTopic(pMnode, pTopic); + sdbCancelFetch(pSdb, pIter); terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb remove)", dropReq.name, pConsumer->consumerId, pConsumer->cgroup); @@ -729,6 +740,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { + mndReleaseTopic(pMnode, pTopic); return -1; } @@ -740,7 +752,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } mndTransSetDbName(pTrans, pTopic->db, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { mndReleaseTopic(pMnode, pTopic); mndTransDrop(pTrans); return -1; @@ -782,14 +794,18 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(buf); sdbRelease(pSdb, pVgroup); + mndReleaseTopic(pMnode, pTopic); + sdbCancelFetch(pSdb, pIter); mndTransDrop(pTrans); return -1; } + sdbRelease(pSdb, pVgroup); } } int32_t code = mndDropTopic(pMnode, pTrans, pReq, pTopic); mndReleaseTopic(pMnode, pTopic); + mndTransDrop(pTrans); if (code != 0) { mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); @@ -830,6 +846,43 @@ int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { return 0; } +static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson){ + char* string = NULL; + cJSON* columns = cJSON_CreateArray(); + if (columns == NULL) { + return; + } + for (int i = 0; i < nCols; i++) { + cJSON* column = cJSON_CreateObject(); + SSchema* s = schema + i; + cJSON* cname = cJSON_CreateString(s->name); + cJSON_AddItemToObject(column, "name", cname); + cJSON* ctype = cJSON_CreateString(tDataTypes[s->type].name); + cJSON_AddItemToObject(column, "type", ctype); + int32_t length = 0; + if (s->type == TSDB_DATA_TYPE_BINARY) { + length = s->bytes - VARSTR_HEADER_SIZE; + } else if (s->type == TSDB_DATA_TYPE_NCHAR || s->type == TSDB_DATA_TYPE_JSON) { + length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + } else{ + length = s->bytes; + } + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(column, "length", cbytes); + cJSON_AddItemToArray(columns, column); + } + string = cJSON_PrintUnformatted(columns); + cJSON_Delete(columns); + + size_t len = strlen(string); + if(string && len <= TSDB_SHOW_SCHEMA_JSON_LEN){ + STR_TO_VARSTR(schemaJson, string); + }else{ + mError("mndRetrieveTopic build schema error json:%p, json len:%zu", string, len); + } + taosMemoryFree(string); +} + static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -862,12 +915,56 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pTopic->createTime, false); - char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; + char *sql = taosMemoryMalloc(strlen(pTopic->sql) + VARSTR_HEADER_SIZE); STR_TO_VARSTR(sql, pTopic->sql); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)sql, false); + taosMemoryFree(sql); + + char *schemaJson = taosMemoryMalloc(TSDB_SHOW_SCHEMA_JSON_LEN + VARSTR_HEADER_SIZE); + if(pTopic->subType == TOPIC_SUB_TYPE__COLUMN){ + schemaToJson(pTopic->schema.pSchema, pTopic->schema.nCols, schemaJson); + }else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE){ + SStbObj *pStb = mndAcquireStb(pMnode, pTopic->stbName); + if (pStb == NULL) { + STR_TO_VARSTR(schemaJson, "NULL"); + mError("mndRetrieveTopic mndAcquireStb null stbName:%s", pTopic->stbName); + }else{ + schemaToJson(pStb->pColumns, pStb->numOfColumns, schemaJson); + mndReleaseStb(pMnode, pStb); + } + }else{ + STR_TO_VARSTR(schemaJson, "NULL"); + } + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)schemaJson, false); + taosMemoryFree(schemaJson); + + char mete[4 + VARSTR_HEADER_SIZE] = {0}; + if(pTopic->withMeta){ + STR_TO_VARSTR(mete, "yes"); + }else{ + STR_TO_VARSTR(mete, "no"); + } + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)mete, false); + + char type[8 + VARSTR_HEADER_SIZE] = {0}; + if(pTopic->subType == TOPIC_SUB_TYPE__COLUMN){ + STR_TO_VARSTR(type, "column"); + }else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE){ + STR_TO_VARSTR(type, "stable"); + }else{ + STR_TO_VARSTR(type, "db"); + } + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)type, false); + numOfRows++; sdbRelease(pSdb, pTopic); } @@ -912,6 +1009,7 @@ bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb) { if (pTopic->dbUid == pDb->uid) { sdbRelease(pSdb, pTopic); + sdbCancelFetch(pSdb, pIter); return true; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index cfb5bef9d0163a8bded942726d7a16715ec2fcc0..7ebaf6dda5c557a7925ea3190e30843cdc4bcb97 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -23,28 +23,25 @@ #include "mndSync.h" #include "mndUser.h" -#define TRANS_VER_NUMBER 1 +#define TRANS_VER1_NUMBER 1 +#define TRANS_VER2_NUMBER 2 #define TRANS_ARRAY_SIZE 8 #define TRANS_RESERVE_SIZE 48 -static SSdbRaw *mndTransActionEncode(STrans *pTrans); -static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld); -static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc); +static int32_t mndTransDelete(SSdb *pSdb, STrans *pTrans, bool callFunc); static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction); static void mndTransDropLogs(SArray *pArray); static void mndTransDropActions(SArray *pArray); -static void mndTransDropData(STrans *pTrans); static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray); static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRedoLogStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformUndoLogStage(SMnode *pMnode, STrans *pTrans); @@ -52,7 +49,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans); +static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans); static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsLeader(pMnode); } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); @@ -67,11 +64,11 @@ int32_t mndInitTrans(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_TRANS, .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndTransActionEncode, - .decodeFp = (SdbDecodeFp)mndTransActionDecode, + .encodeFp = (SdbEncodeFp)mndTransEncode, + .decodeFp = (SdbDecodeFp)mndTransDecode, .insertFp = (SdbInsertFp)mndTransActionInsert, .updateFp = (SdbUpdateFp)mndTransActionUpdate, - .deleteFp = (SdbDeleteFp)mndTransActionDelete, + .deleteFp = (SdbDeleteFp)mndTransDelete, }; mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransTimer); @@ -103,15 +100,55 @@ static int32_t mndTransGetActionsSize(SArray *pArray) { return rawDataLen; } -static SSdbRaw *mndTransActionEncode(STrans *pTrans) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + +static int32_t mndTransEncodeAction(SSdbRaw *pRaw, int32_t *offset, SArray *pActions, int32_t actionsNum) { + int32_t dataPos = *offset; + int8_t unused = 0; + int32_t ret = -1; + + for (int32_t i = 0; i < actionsNum; ++i) { + STransAction *pAction = taosArrayGet(pActions, i); + SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) + SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) + if (pAction->actionType == TRANS_ACTION_RAW) { + int32_t len = sdbGetRawTotalSize(pAction->pRaw); + SDB_SET_INT8(pRaw, dataPos, unused /*pAction->rawWritten*/, _OVER) + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pRaw, len, _OVER) + } else if (pAction->actionType == TRANS_ACTION_MSG) { + SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) + SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) + SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgSent*/, _OVER) + SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgReceived*/, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) + } else { + // nothing + } + } + ret = 0; + +_OVER: + *offset = dataPos; + return ret; +} + +SSdbRaw *mndTransEncode(STrans *pTrans) { + terrno = TSDB_CODE_INVALID_MSG; + int8_t sver = taosArrayGetSize(pTrans->prepareActions) ? TRANS_VER2_NUMBER : TRANS_VER1_NUMBER; int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE + pTrans->paramLen; + rawDataLen += mndTransGetActionsSize(pTrans->prepareActions); rawDataLen += mndTransGetActionsSize(pTrans->redoActions); rawDataLen += mndTransGetActionsSize(pTrans->undoActions); rawDataLen += mndTransGetActionsSize(pTrans->commitActions); - SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TRANS_VER_NUMBER, rawDataLen); + SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, sver, rawDataLen); if (pRaw == NULL) { mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); return NULL; @@ -131,91 +168,22 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_BINARY(pRaw, dataPos, pTrans->stbname, TSDB_TABLE_FNAME_LEN, _OVER) SDB_SET_INT32(pRaw, dataPos, pTrans->redoActionPos, _OVER) + int32_t prepareActionNum = taosArrayGetSize(pTrans->prepareActions); int32_t redoActionNum = taosArrayGetSize(pTrans->redoActions); int32_t undoActionNum = taosArrayGetSize(pTrans->undoActions); int32_t commitActionNum = taosArrayGetSize(pTrans->commitActions); + + if (sver > TRANS_VER1_NUMBER) { + SDB_SET_INT32(pRaw, dataPos, prepareActionNum, _OVER) + } SDB_SET_INT32(pRaw, dataPos, redoActionNum, _OVER) SDB_SET_INT32(pRaw, dataPos, undoActionNum, _OVER) SDB_SET_INT32(pRaw, dataPos, commitActionNum, _OVER) - int8_t unused = 0; - for (int32_t i = 0; i < redoActionNum; ++i) { - STransAction *pAction = taosArrayGet(pTrans->redoActions, i); - SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) - if (pAction->actionType == TRANS_ACTION_RAW) { - int32_t len = sdbGetRawTotalSize(pAction->pRaw); - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->rawWritten*/, _OVER) - SDB_SET_INT32(pRaw, dataPos, len, _OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pRaw, len, _OVER) - } else if (pAction->actionType == TRANS_ACTION_MSG) { - SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) - SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgSent*/, _OVER) - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgReceived*/, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) - SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) - } else { - // nothing - } - } - - for (int32_t i = 0; i < undoActionNum; ++i) { - STransAction *pAction = taosArrayGet(pTrans->undoActions, i); - SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) - if (pAction->actionType == TRANS_ACTION_RAW) { - int32_t len = sdbGetRawTotalSize(pAction->pRaw); - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->rawWritten*/, _OVER) - SDB_SET_INT32(pRaw, dataPos, len, _OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pRaw, len, _OVER) - } else if (pAction->actionType == TRANS_ACTION_MSG) { - SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) - SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgSent*/, _OVER) - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgReceived*/, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) - SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) - } else { - // nothing - } - } - - for (int32_t i = 0; i < commitActionNum; ++i) { - STransAction *pAction = taosArrayGet(pTrans->commitActions, i); - SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER) - if (pAction->actionType == TRANS_ACTION_RAW) { - int32_t len = sdbGetRawTotalSize(pAction->pRaw); - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->rawWritten*/, _OVER) - SDB_SET_INT32(pRaw, dataPos, len, _OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pRaw, len, _OVER) - } else if (pAction->actionType == TRANS_ACTION_MSG) { - SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) - SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgSent*/, _OVER) - SDB_SET_INT8(pRaw, dataPos, unused /*pAction->msgReceived*/, _OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) - SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) - } else { - // nothing - } - } + if (mndTransEncodeAction(pRaw, &dataPos, pTrans->prepareActions, prepareActionNum) < 0) goto _OVER; + if (mndTransEncodeAction(pRaw, &dataPos, pTrans->redoActions, redoActionNum) < 0) goto _OVER; + if (mndTransEncodeAction(pRaw, &dataPos, pTrans->undoActions, undoActionNum) < 0) goto _OVER; + if (mndTransEncodeAction(pRaw, &dataPos, pTrans->commitActions, commitActionNum) < 0) goto _OVER; SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, _OVER) SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, _OVER) @@ -242,23 +210,76 @@ _OVER: return pRaw; } -static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { - terrno = TSDB_CODE_OUT_OF_MEMORY; +static int32_t mndTransDecodeAction(SSdbRaw *pRaw, int32_t *offset, SArray *pActions, int32_t actionNum) { + STransAction action = {0}; + int32_t dataPos = *offset; + int8_t unused = 0; + int8_t stage = 0; + int8_t actionType = 0; + int32_t dataLen = 0; + int32_t ret = -1; + + for (int32_t i = 0; i < actionNum; ++i) { + memset(&action, 0, sizeof(action)); + SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) + SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) + action.actionType = actionType; + SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) + action.stage = stage; + SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) + if (action.actionType == TRANS_ACTION_RAW) { + SDB_GET_INT8(pRaw, dataPos, &unused /*&action.rawWritten*/, _OVER) + SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) + action.pRaw = taosMemoryMalloc(dataLen); + if (action.pRaw == NULL) goto _OVER; + mTrace("raw:%p, is created", action.pRaw); + SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); + if (taosArrayPush(pActions, &action) == NULL) goto _OVER; + action.pRaw = NULL; + } else if (action.actionType == TRANS_ACTION_MSG) { + SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); + tmsgUpdateDnodeEpSet(&action.epSet); + SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) + SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgSent*/, _OVER) + SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgReceived*/, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) + action.pCont = taosMemoryMalloc(action.contLen); + if (action.pCont == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); + if (taosArrayPush(pActions, &action) == NULL) goto _OVER; + action.pCont = NULL; + } else { + if (taosArrayPush(pActions, &action) == NULL) goto _OVER; + } + } + ret = 0; + +_OVER: + *offset = dataPos; + taosMemoryFreeClear(action.pCont); + return ret; +} + +SSdbRow *mndTransDecode(SSdbRaw *pRaw) { + terrno = TSDB_CODE_INVALID_MSG; SSdbRow *pRow = NULL; STrans *pTrans = NULL; char *pData = NULL; int32_t dataLen = 0; int8_t sver = 0; + int32_t prepareActionNum = 0; int32_t redoActionNum = 0; int32_t undoActionNum = 0; int32_t commitActionNum = 0; int32_t dataPos = 0; - STransAction action = {0}; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != TRANS_VER_NUMBER) { + if (sver != TRANS_VER1_NUMBER && sver != TRANS_VER2_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -294,127 +315,28 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_TABLE_FNAME_LEN, _OVER) SDB_GET_BINARY(pRaw, dataPos, pTrans->stbname, TSDB_TABLE_FNAME_LEN, _OVER) SDB_GET_INT32(pRaw, dataPos, &pTrans->redoActionPos, _OVER) + + if (sver > TRANS_VER1_NUMBER) { + SDB_GET_INT32(pRaw, dataPos, &prepareActionNum, _OVER) + } SDB_GET_INT32(pRaw, dataPos, &redoActionNum, _OVER) SDB_GET_INT32(pRaw, dataPos, &undoActionNum, _OVER) SDB_GET_INT32(pRaw, dataPos, &commitActionNum, _OVER) + pTrans->prepareActions = taosArrayInit(prepareActionNum, sizeof(STransAction)); pTrans->redoActions = taosArrayInit(redoActionNum, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(undoActionNum, sizeof(STransAction)); pTrans->commitActions = taosArrayInit(commitActionNum, sizeof(STransAction)); + if (pTrans->prepareActions == NULL) goto _OVER; if (pTrans->redoActions == NULL) goto _OVER; if (pTrans->undoActions == NULL) goto _OVER; if (pTrans->commitActions == NULL) goto _OVER; - int8_t unused = 0; - for (int32_t i = 0; i < redoActionNum; ++i) { - memset(&action, 0, sizeof(action)); - SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) - action.actionType = actionType; - SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) - action.stage = stage; - SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) - if (action.actionType == TRANS_ACTION_RAW) { - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.rawWritten*/, _OVER) - SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) - action.pRaw = taosMemoryMalloc(dataLen); - if (action.pRaw == NULL) goto _OVER; - mTrace("raw:%p, is created", action.pRaw); - SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); - if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; - action.pRaw = NULL; - } else if (action.actionType == TRANS_ACTION_MSG) { - SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); - tmsgUpdateDnodeEpSet(&action.epSet); - SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgSent*/, _OVER) - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgReceived*/, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) - action.pCont = taosMemoryMalloc(action.contLen); - if (action.pCont == NULL) goto _OVER; - SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); - if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; - action.pCont = NULL; - } else { - if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; - } - } - - for (int32_t i = 0; i < undoActionNum; ++i) { - memset(&action, 0, sizeof(action)); - SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) - action.actionType = actionType; - SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) - action.stage = stage; - SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) - if (action.actionType == TRANS_ACTION_RAW) { - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.rawWritten*/, _OVER) - SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) - action.pRaw = taosMemoryMalloc(dataLen); - if (action.pRaw == NULL) goto _OVER; - mTrace("raw:%p, is created", action.pRaw); - SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); - if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; - action.pRaw = NULL; - } else if (action.actionType == TRANS_ACTION_MSG) { - SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); - SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgSent*/, _OVER) - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgReceived*/, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) - action.pCont = taosMemoryMalloc(action.contLen); - if (action.pCont == NULL) goto _OVER; - SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); - if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; - action.pCont = NULL; - } else { - if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; - } - } - - for (int32_t i = 0; i < commitActionNum; ++i) { - memset(&action, 0, sizeof(action)); - SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) - action.actionType = actionType; - SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) - action.stage = stage; - SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER) - if (action.actionType) { - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.rawWritten*/, _OVER) - SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) - action.pRaw = taosMemoryMalloc(dataLen); - if (action.pRaw == NULL) goto _OVER; - mTrace("raw:%p, is created", action.pRaw); - SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); - if (taosArrayPush(pTrans->commitActions, &action) == NULL) goto _OVER; - action.pRaw = NULL; - } else if (action.actionType == TRANS_ACTION_MSG) { - SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); - SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgSent*/, _OVER) - SDB_GET_INT8(pRaw, dataPos, &unused /*&action.msgReceived*/, _OVER) - SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) - action.pCont = taosMemoryMalloc(action.contLen); - if (action.pCont == NULL) goto _OVER; - SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); - if (taosArrayPush(pTrans->commitActions, &action) == NULL) goto _OVER; - action.pCont = NULL; - } else { - if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; - } - } + if (mndTransDecodeAction(pRaw, &dataPos, pTrans->prepareActions, prepareActionNum) < 0) goto _OVER; + if (mndTransDecodeAction(pRaw, &dataPos, pTrans->redoActions, redoActionNum) < 0) goto _OVER; + if (mndTransDecodeAction(pRaw, &dataPos, pTrans->undoActions, undoActionNum) < 0) goto _OVER; + if (mndTransDecodeAction(pRaw, &dataPos, pTrans->commitActions, commitActionNum) < 0) goto _OVER; SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, _OVER) SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, _OVER) @@ -434,7 +356,6 @@ _OVER: mError("trans:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, terrstr()); mndTransDropData(pTrans); taosMemoryFreeClear(pRow); - taosMemoryFreeClear(action.pCont); return NULL; } @@ -458,7 +379,7 @@ static const char *mndTransStr(ETrnStage stage) { return "commit"; case TRN_STAGE_COMMIT_ACTION: return "commitAction"; - case TRN_STAGE_FINISHED: + case TRN_STAGE_FINISH: return "finished"; case TRN_STAGE_PRE_FINISH: return "pre-finish"; @@ -519,7 +440,11 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { return 0; } -static void mndTransDropData(STrans *pTrans) { +void mndTransDropData(STrans *pTrans) { + if (pTrans->prepareActions != NULL) { + mndTransDropActions(pTrans->prepareActions); + pTrans->prepareActions = NULL; + } if (pTrans->redoActions != NULL) { mndTransDropActions(pTrans->redoActions); pTrans->redoActions = NULL; @@ -549,7 +474,7 @@ static void mndTransDropData(STrans *pTrans) { (void)taosThreadMutexDestroy(&pTrans->mutex); } -static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { +static int32_t mndTransDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { mInfo("trans:%d, perform delete action, row:%p stage:%s callfunc:%d, stopFunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage), callFunc, pTrans->stopFunc); @@ -586,10 +511,11 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { pOld->id, pOld, mndTransStr(pOld->stage), pOld->createdTime, pNew, mndTransStr(pNew->stage), pNew->createdTime); // only occured while sync timeout - terrno = TSDB_CODE_MND_TRNAS_SYNC_TIMEOUT; + terrno = TSDB_CODE_MND_TRANS_SYNC_TIMEOUT; return -1; } + mndTransUpdateActions(pOld->prepareActions, pNew->prepareActions); mndTransUpdateActions(pOld->redoActions, pNew->redoActions); mndTransUpdateActions(pOld->undoActions, pNew->undoActions); mndTransUpdateActions(pOld->commitActions, pNew->commitActions); @@ -607,7 +533,7 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { } if (pOld->stage == TRN_STAGE_PRE_FINISH) { - pOld->stage = TRN_STAGE_FINISHED; + pOld->stage = TRN_STAGE_FINISH; mTrace("trans:%d, stage from pre-finish to finished since perform update action", pNew->id); } @@ -646,6 +572,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->conflict = conflict; pTrans->exec = TRN_EXEC_PARALLEL; pTrans->createdTime = taosGetTimestampMs(); + pTrans->prepareActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); @@ -728,6 +655,13 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) { return mndTransAppendAction(pTrans->commitActions, &action); } +int32_t mndTransAppendPrepareAction(STrans *pTrans, STransAction *pAction) { + pAction->stage = TRN_STAGE_PREPARE; + pAction->actionType = TRANS_ACTION_RAW; + pAction->mTraceId = pTrans->mTraceId; + return mndTransAppendAction(pTrans->prepareActions, pAction); +} + int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction) { pAction->stage = TRN_STAGE_REDO_ACTION; pAction->actionType = TRANS_ACTION_MSG; @@ -800,7 +734,7 @@ void mndTransSetParallel(STrans *pTrans) { pTrans->exec = TRN_EXEC_PARALLEL; } void mndTransSetOper(STrans *pTrans, EOperType oper) { pTrans->oper = oper; } static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { - SSdbRaw *pRaw = mndTransActionEncode(pTrans); + SSdbRaw *pRaw = mndTransEncode(pTrans); if (pRaw == NULL) { mError("trans:%d, failed to encode while sync trans since %s", pTrans->id, terrstr()); return -1; @@ -872,7 +806,7 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { return conflict; } -int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans) { +int32_t mndTransCheckConflict(SMnode *pMnode, STrans *pTrans) { if (pTrans->conflict == TRN_CONFLICT_DB || pTrans->conflict == TRN_CONFLICT_DB_INSIDE) { if (strlen(pTrans->dbname) == 0 && strlen(pTrans->stbname) == 0) { terrno = TSDB_CODE_MND_TRANS_CONFLICT; @@ -891,7 +825,7 @@ int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans) { } int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { - if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + if (mndTransCheckConflict(pMnode, pTrans) != 0) { return -1; } @@ -922,7 +856,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { pTrans->rpcRsp = NULL; pTrans->rpcRspLen = 0; - mndTransExecute(pMnode, pNew, true); + mndTransExecute(pMnode, pNew); mndReleaseTrans(pMnode, pNew); return 0; } @@ -961,7 +895,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { bool sendRsp = false; int32_t code = pTrans->code; - if (pTrans->stage == TRN_STAGE_FINISHED) { + if (pTrans->stage == TRN_STAGE_FINISH) { sendRsp = true; } @@ -1003,7 +937,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; } if (code == TSDB_CODE_SYN_TIMEOUT) { - code = TSDB_CODE_MND_TRNAS_SYNC_TIMEOUT; + code = TSDB_CODE_MND_TRANS_SYNC_TIMEOUT; } if (i != 0 && code == 0) { @@ -1104,7 +1038,7 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) { mInfo("trans:%d, invalid action, index:%d, code:0x%x", transId, action, pRsp->code); } - mndTransExecute(pMnode, pTrans, true); + mndTransExecute(pMnode, pTrans); _OVER: mndReleaseTrans(pMnode, pTrans); @@ -1392,8 +1326,25 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) return code; } -static bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { +bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { bool continueExec = true; + int32_t code = 0; + + int32_t numOfActions = taosArrayGetSize(pTrans->prepareActions); + if (numOfActions == 0) goto _OVER; + + mInfo("trans:%d, execute %d prepare actions.", pTrans->id, numOfActions); + + for (int32_t action = 0; action < numOfActions; ++action) { + STransAction *pAction = taosArrayGet(pTrans->prepareActions, action); + code = mndTransExecSingleAction(pMnode, pTrans, pAction); + if (code != 0) { + mError("trans:%d, failed to execute prepare action:%d, numOfActions:%d", pTrans->id, action, numOfActions); + return false; + } + } + +_OVER: pTrans->stage = TRN_STAGE_REDO_ACTION; mInfo("trans:%d, stage from prepare to redoAction", pTrans->id); return continueExec; @@ -1476,7 +1427,7 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans) { if (code == 0) { pTrans->code = 0; - pTrans->stage = TRN_STAGE_FINISHED; // TRN_STAGE_PRE_FINISH is not necessary + pTrans->stage = TRN_STAGE_FINISH; // TRN_STAGE_PRE_FINISH is not necessary mInfo("trans:%d, stage from commitAction to finished", pTrans->id); continueExec = true; } else { @@ -1528,14 +1479,14 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerfromPreFinishedStage(SMnode *pMnode, STrans *pTrans) { +static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans) { if (mndCannotExecuteTransAction(pMnode)) return false; bool continueExec = true; int32_t code = mndTransPreFinish(pMnode, pTrans); if (code == 0) { - pTrans->stage = TRN_STAGE_FINISHED; + pTrans->stage = TRN_STAGE_FINISH; mInfo("trans:%d, stage from pre-finish to finish", pTrans->id); continueExec = true; } else { @@ -1547,10 +1498,10 @@ static bool mndTransPerfromPreFinishedStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) { +static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans) { bool continueExec = false; - SSdbRaw *pRaw = mndTransActionEncode(pTrans); + SSdbRaw *pRaw = mndTransEncode(pTrans); if (pRaw == NULL) { mError("trans:%d, failed to encode while finish trans since %s", pTrans->id, terrstr()); return false; @@ -1567,12 +1518,12 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader) { +void mndTransExecuteImp(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = true; while (continueExec) { - mInfo("trans:%d, continue to execute, stage:%s createTime:%" PRId64 " leader:%d", pTrans->id, - mndTransStr(pTrans->stage), pTrans->createdTime, isLeader); + mInfo("trans:%d, continue to execute, stage:%s createTime:%" PRId64 " topHalf:%d", pTrans->id, + mndTransStr(pTrans->stage), pTrans->createdTime, topHalf); pTrans->lastExecTime = taosGetTimestampMs(); switch (pTrans->stage) { case TRN_STAGE_PREPARE: @@ -1582,7 +1533,7 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader) { continueExec = mndTransPerformRedoActionStage(pMnode, pTrans); break; case TRN_STAGE_COMMIT: - if (isLeader) { + if (topHalf) { continueExec = mndTransPerformCommitStage(pMnode, pTrans); } else { mInfo("trans:%d, can not commit since not leader", pTrans->id); @@ -1593,7 +1544,7 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader) { continueExec = mndTransPerformCommitActionStage(pMnode, pTrans); break; case TRN_STAGE_ROLLBACK: - if (isLeader) { + if (topHalf) { continueExec = mndTransPerformRollbackStage(pMnode, pTrans); } else { mInfo("trans:%d, can not rollback since not leader", pTrans->id); @@ -1604,15 +1555,15 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader) { continueExec = mndTransPerformUndoActionStage(pMnode, pTrans); break; case TRN_STAGE_PRE_FINISH: - if (isLeader) { - continueExec = mndTransPerfromPreFinishedStage(pMnode, pTrans); + if (topHalf) { + continueExec = mndTransPerformPreFinishStage(pMnode, pTrans); } else { mInfo("trans:%d, can not pre-finish since not leader", pTrans->id); continueExec = false; } break; - case TRN_STAGE_FINISHED: - continueExec = mndTransPerfromFinishedStage(pMnode, pTrans); + case TRN_STAGE_FINISH: + continueExec = mndTransPerformFinishStage(pMnode, pTrans); break; default: continueExec = false; @@ -1623,6 +1574,16 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader) { mndTransSendRpcRsp(pMnode, pTrans); } +void mndTransExecute(SMnode *pMnode, STrans *pTrans) { + bool topHalf = true; + return mndTransExecuteImp(pMnode, pTrans, topHalf); +} + +void mndTransRefresh(SMnode *pMnode, STrans *pTrans) { + bool topHalf = false; + return mndTransExecuteImp(pMnode, pTrans, topHalf); +} + static int32_t mndProcessTransTimer(SRpcMsg *pReq) { mTrace("start to process trans timer"); mndTransPullup(pReq->info.node); @@ -1649,7 +1610,7 @@ int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { pAction->errCode = 0; } - mndTransExecute(pMnode, pTrans, true); + mndTransExecute(pMnode, pTrans); return 0; } @@ -1707,7 +1668,7 @@ void mndTransPullup(SMnode *pMnode) { int32_t *pTransId = taosArrayGet(pArray, i); STrans *pTrans = mndAcquireTrans(pMnode, *pTransId); if (pTrans != NULL) { - mndTransExecute(pMnode, pTrans, true); + mndTransExecute(pMnode, pTrans); } mndReleaseTrans(pMnode, pTrans); } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 3da594109a1910fdd81ab654c3023d1cbfc48c9e..c59d23d252163dd964018b4ba0b31af024a8acde 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -488,7 +488,7 @@ SHashObj *mndDupUseDbHash(SHashObj *pOld) { return pNew; } -static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { +int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { memcpy(pNew, pUser, sizeof(SUserObj)); pNew->authVersion++; pNew->updateTime = taosGetTimestampMs(); @@ -508,7 +508,7 @@ static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { return 0; } -static void mndUserFreeObj(SUserObj *pUser) { +void mndUserFreeObj(SUserObj *pUser) { taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->writeDbs); taosHashCleanup(pUser->topics); @@ -630,6 +630,11 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } + if (strlen(createReq.pass) >= TSDB_PASSWORD_LEN){ + terrno = TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG; + goto _OVER; + } + pUser = mndAcquireUser(pMnode, createReq.user); if (pUser != NULL) { terrno = TSDB_CODE_MND_USER_ALREADY_EXIST; @@ -801,7 +806,8 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { goto _OVER; } - if (TSDB_ALTER_USER_PASSWD == alterReq.alterType && alterReq.pass[0] == 0) { + if (TSDB_ALTER_USER_PASSWD == alterReq.alterType && + (alterReq.pass[0] == 0 || strlen(alterReq.pass) >= TSDB_PASSWORD_LEN)) { terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT; goto _OVER; } @@ -824,7 +830,6 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { if (mndUserDupObj(pUser, &newUser) != 0) goto _OVER; - newUser.passVersion = pUser->passVersion; if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) { char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)alterReq.pass, strlen(alterReq.pass), pass); @@ -858,6 +863,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { mndReleaseDb(pMnode, pDb); goto _OVER; } + mndReleaseDb(pMnode, pDb); } else { while (1) { SDbObj *pDb = NULL; @@ -882,6 +888,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { mndReleaseDb(pMnode, pDb); goto _OVER; } + mndReleaseDb(pMnode, pDb); } else { while (1) { SDbObj *pDb = NULL; @@ -903,6 +910,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { goto _OVER; } taosHashRemove(newUser.readDbs, alterReq.objname, len); + mndReleaseDb(pMnode, pDb); } else { taosHashClear(newUser.readDbs); } @@ -917,24 +925,25 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { goto _OVER; } taosHashRemove(newUser.writeDbs, alterReq.objname, len); + mndReleaseDb(pMnode, pDb); } else { taosHashClear(newUser.writeDbs); } } - if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_TABLE) { + if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_TABLE || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_TABLE) { if (mndTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; } - if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_TABLE) { + if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_TABLE || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_TABLE) { if (mndTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; } - if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_TABLE) { + if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_TABLE || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_TABLE) { if (mndRemoveTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; } - if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_TABLE) { + if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_TABLE || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_TABLE) { if (mndRemoveTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; } @@ -1169,26 +1178,30 @@ static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int3 if (strcmp("t", value) != 0) { SNode *pAst = NULL; int32_t sqlLen = 0; - char sql[TSDB_EXPLAIN_RESULT_ROW_SIZE] = {0}; + size_t bufSz = strlen(value) + 1; + char* sql = taosMemoryMalloc(bufSz + 1); + char* obj = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); - if (nodesStringToNode(value, &pAst) == 0) { - nodesNodeToSQL(pAst, sql, TSDB_EXPLAIN_RESULT_ROW_SIZE, &sqlLen); + if (sql != NULL && obj != NULL && nodesStringToNode(value, &pAst) == 0) { + nodesNodeToSQL(pAst, sql, bufSz, &sqlLen); nodesDestroyNode(pAst); } else { sqlLen = 5; sprintf(sql, "error"); } - char obj[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(obj, sql, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, *numOfRows, (const char *)obj, false); + taosMemoryFree(obj); + taosMemoryFree(sql); } else { - char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, *numOfRows, (const char *)condition, false); + taosMemoryFree(condition); } (*numOfRows)++; @@ -1204,16 +1217,34 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock int32_t cols = 0; char *pWrite; + bool fetchNextUser = pShow->restore ? false : true; + pShow->restore = false; + while (numOfRows < rows) { - pShow->pIter = sdbFetch(pSdb, SDB_USER, pShow->pIter, (void **)&pUser); - if (pShow->pIter == NULL) break; + if (fetchNextUser) { + pShow->pIter = sdbFetch(pSdb, SDB_USER, pShow->pIter, (void **)&pUser); + if (pShow->pIter == NULL) break; + } else { + fetchNextUser = true; + void *pKey = taosHashGetKey(pShow->pIter, NULL); + pUser = sdbAcquire(pSdb, SDB_USER, pKey); + if (!pUser) { + continue; + } + } int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs); int32_t numOfTopics = taosHashGetSize(pUser->topics); int32_t numOfReadTbs = taosHashGetSize(pUser->readTbs); int32_t numOfWriteTbs = taosHashGetSize(pUser->writeTbs); - if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics + numOfReadTbs + numOfWriteTbs >= rows) break; + if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics + numOfReadTbs + numOfWriteTbs >= rows) { + mInfo("will restore. current num of rows: %d, read dbs %d, write dbs %d, topics %d, read tables %d, write tables %d", + numOfRows, numOfReadDbs, numOfWriteDbs, numOfTopics, numOfReadTbs, numOfWriteTbs); + pShow->restore = true; + sdbRelease(pSdb, pUser); + break; + } if (pUser->superUser) { cols = 0; @@ -1237,10 +1268,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); + taosMemoryFree(condition); numOfRows++; } @@ -1271,10 +1303,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); + taosMemoryFree(condition); numOfRows++; db = taosHashIterate(pUser->readDbs, db); @@ -1306,10 +1339,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); + taosMemoryFree(condition); numOfRows++; db = taosHashIterate(pUser->writeDbs, db); @@ -1343,10 +1377,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); + taosMemoryFree(condition); numOfRows++; topic = taosHashIterate(pUser->topics, topic); @@ -1431,69 +1466,6 @@ _OVER: return code; } -int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int32_t numOfUses, void **ppRsp, - int32_t *pRspLen) { - int32_t code = 0; - SUserPassBatchRsp batchRsp = {0}; - - for (int32_t i = 0; i < numOfUses; ++i) { - SUserObj *pUser = mndAcquireUser(pMnode, pUsers[i].user); - if (pUser == NULL) { - mError("user:%s, failed to validate user pass since %s", pUsers[i].user, terrstr()); - continue; - } - - pUsers[i].version = ntohl(pUsers[i].version); - if (pUser->passVersion <= pUsers[i].version) { - mTrace("user:%s, not update since mnd passVer %d <= client passVer %d", pUsers[i].user, pUser->passVersion, - pUsers[i].version); - mndReleaseUser(pMnode, pUser); - continue; - } - - SGetUserPassRsp rsp = {0}; - memcpy(rsp.user, pUser->user, TSDB_USER_LEN); - rsp.version = pUser->passVersion; - - if (!batchRsp.pArray && !(batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserPassRsp)))) { - code = TSDB_CODE_OUT_OF_MEMORY; - mndReleaseUser(pMnode, pUser); - goto _OVER; - } - - taosArrayPush(batchRsp.pArray, &rsp); - mndReleaseUser(pMnode, pUser); - } - - if (taosArrayGetSize(batchRsp.pArray) <= 0) { - goto _OVER; - } - - int32_t rspLen = tSerializeSUserPassBatchRsp(NULL, 0, &batchRsp); - if (rspLen < 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; - } - void *pRsp = taosMemoryMalloc(rspLen); - if (pRsp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; - } - tSerializeSUserPassBatchRsp(pRsp, rspLen, &batchRsp); - - *ppRsp = pRsp; - *pRspLen = rspLen; - -_OVER: - if (code) { - *ppRsp = NULL; - *pRspLen = 0; - } - - tFreeSUserPassBatchRsp(&batchRsp); - return code; -} - int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { int32_t code = 0; SSdb *pSdb = pMnode->pSdb; @@ -1507,7 +1479,9 @@ int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { if (pIter == NULL) break; code = -1; - if (mndUserDupObj(pUser, &newUser) != 0) break; + if (mndUserDupObj(pUser, &newUser) != 0) { + break; + } bool inRead = (taosHashGet(newUser.readDbs, db, len) != NULL); bool inWrite = (taosHashGet(newUser.writeDbs, db, len) != NULL); @@ -1516,7 +1490,9 @@ int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { (void)taosHashRemove(newUser.writeDbs, db, len); SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) break; + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + break; + } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); } @@ -1554,7 +1530,9 @@ int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) { if (inTopic) { (void)taosHashRemove(newUser.topics, topic, len); SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) break; + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + break; + } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index a8e9db28e9599e836d9deb6ec9adedf7449e9041..f3dded9c76e15f81bea2f7a5ca7313d91910c3a4 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -20,6 +20,9 @@ #include "mndMnode.h" #include "mndPrivilege.h" #include "mndShow.h" +#include "mndStb.h" +#include "mndStream.h" +#include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" #include "tmisce.h" @@ -27,7 +30,6 @@ #define VGROUP_VER_NUMBER 1 #define VGROUP_RESERVE_SIZE 64 -static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); @@ -482,15 +484,15 @@ static void *mndBuildDisableVnodeWriteReq(SMnode *pMnode, SDbObj *pDb, int32_t v return pReq; } -static void *mndBuildAlterVnodeHashRangeReq(SMnode *pMnode, SVgObj *pVgroup, int32_t dstVgId, int32_t *pContLen) { +static void *mndBuildAlterVnodeHashRangeReq(SMnode *pMnode, int32_t srcVgId, SVgObj *pVgroup, int32_t *pContLen) { SAlterVnodeHashRangeReq alterReq = { - .srcVgId = pVgroup->vgId, - .dstVgId = dstVgId, + .srcVgId = srcVgId, + .dstVgId = pVgroup->vgId, .hashBegin = pVgroup->hashBegin, .hashEnd = pVgroup->hashEnd, }; - mInfo("vgId:%d, build alter vnode hashrange req, dstVgId:%d, hashrange:[%u, %u]", pVgroup->vgId, dstVgId, + mInfo("vgId:%d, build alter vnode hashrange req, dstVgId:%d, hashrange:[%u, %u]", srcVgId, pVgroup->vgId, pVgroup->hashBegin, pVgroup->hashEnd); int32_t contLen = tSerializeSAlterVnodeHashRangeReq(NULL, 0, &alterReq); if (contLen < 0) { @@ -873,7 +875,7 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p // if (pDb == NULL || pDb->compactStartTime <= 0) { // colDataSetNULL(pColInfo, numOfRows); // } else { - // colDataAppend(pColInfo, numOfRows, (const char *)&pDb->compactStartTime, false); + // colDataSetVal(pColInfo, numOfRows, (const char *)&pDb->compactStartTime, false); // } numOfRows++; @@ -1206,17 +1208,18 @@ int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD return 0; } -static int32_t mndAddAlterVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, int32_t dstVgId) { +static int32_t mndAddAlterVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, int32_t srcVgId, SVgObj *pVgroup) { STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); int32_t contLen = 0; - void *pReq = mndBuildAlterVnodeHashRangeReq(pMnode, pVgroup, dstVgId, &contLen); + void *pReq = mndBuildAlterVnodeHashRangeReq(pMnode, srcVgId, pVgroup, &contLen); if (pReq == NULL) return -1; action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_ALTER_HASHRANGE; + action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); @@ -1246,6 +1249,21 @@ int32_t mndAddAlterVnodeConfigAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb return 0; } +int32_t mndAddPrepareNewVgAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVg) { + SSdbRaw *pRaw = mndVgroupActionEncode(pVg); + if (pRaw == NULL) goto _err; + + STransAction action = {.pRaw = pRaw, .msgType = TDMT_MND_CREATE_VG}; + if (mndTransAppendPrepareAction(pTrans, &action) != 0) goto _err; + (void)sdbSetRawStatus(pRaw, SDB_STATUS_CREATING); + pRaw = NULL; + return 0; + +_err: + sdbFreeRaw(pRaw); + return -1; +} + int32_t mndAddAlterVnodeReplicaAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) { SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); if (pDnode == NULL) return -1; @@ -1378,7 +1396,7 @@ int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgOb } int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vnIndex, - SArray *pArray, bool force) { + SArray *pArray, bool force, bool unsafe) { SVgObj newVg = {0}; memcpy(&newVg, pVgroup, sizeof(SVgObj)); @@ -1455,7 +1473,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, mInfo("vgId:%d, will add 1 vnode and force remove 1 vnode", pVgroup->vgId); if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1; newVg.replica--; - SVnodeGid del = newVg.vnodeGid[vnIndex]; + //SVnodeGid del = newVg.vnodeGid[vnIndex]; newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica]; memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid)); { @@ -1476,7 +1494,31 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[vnIndex]) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; - if (newVg.replica == 1) { + if(newVg.replica == 1){ + if(force && !unsafe){ + terrno = TSDB_CODE_VND_META_DATA_UNSAFE_DELETE; + return -1; + } + + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + + while (1) { + SStbObj *pStb = NULL; + pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pStb); + if (pIter == NULL) break; + + if (strcmp(pStb->db, pDb->name) == 0) { + if (mndSetForceDropCreateStbRedoActions(pMnode, pTrans, &newVg, pStb) != 0) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pStb); + return -1; + } + } + + sdbRelease(pSdb, pStb); + } + mInfo("vgId:%d, all data is dropped since replica=1", pVgroup->vgId); } } @@ -1498,7 +1540,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, return 0; } -int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t delDnodeId, bool force) { +int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t delDnodeId, bool force, bool unsafe) { int32_t code = 0; SArray *pArray = mndBuildDnodesArray(pMnode, delDnodeId); if (pArray == NULL) return -1; @@ -1521,7 +1563,7 @@ int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t del if (vnIndex != -1) { mInfo("vgId:%d, vnode:%d will be removed from dnode:%d, force:%d", pVgroup->vgId, vnIndex, delDnodeId, force); SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); - code = mndSetMoveVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, vnIndex, pArray, force); + code = mndSetMoveVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, vnIndex, pArray, force, unsafe); mndReleaseDb(pMnode, pDb); } @@ -2092,27 +2134,32 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); - //add first + //add second if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1; + //learner stage newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) return -1; if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1; + + //follower stage newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0) return -1; + if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) + return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; - //add second + //add third if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1; + newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; newVgroup.vnodeGid[2].nodeRole = TAOS_SYNC_ROLE_VOTER; - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) return -1; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0) @@ -2211,10 +2258,13 @@ static int32_t mndAddAdjustVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, return 0; } -static int32_t mndTransCommitVgStatus(STrans *pTrans, SVgObj *pVg, ESdbStatus vgStatus) { +typedef int32_t (*FpTransActionCb)(STrans *pTrans, SSdbRaw *pRaw); + +static int32_t mndAddVgStatusAction(STrans *pTrans, SVgObj *pVg, ESdbStatus vgStatus, ETrnStage stage) { + FpTransActionCb appendActionCb = (stage == TRN_STAGE_COMMIT_ACTION) ? mndTransAppendCommitlog : mndTransAppendRedolog; SSdbRaw *pRaw = mndVgroupActionEncode(pVg); if (pRaw == NULL) goto _err; - if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _err; + if (appendActionCb(pTrans, pRaw) != 0) goto _err; (void)sdbSetRawStatus(pRaw, vgStatus); pRaw = NULL; return 0; @@ -2223,18 +2273,50 @@ _err: return -1; } +static int32_t mndAddDbStatusAction(STrans *pTrans, SDbObj *pDb, ESdbStatus dbStatus, ETrnStage stage) { + FpTransActionCb appendActionCb = (stage == TRN_STAGE_COMMIT_ACTION) ? mndTransAppendCommitlog : mndTransAppendRedolog; + SSdbRaw *pRaw = mndDbActionEncode(pDb); + if (pRaw == NULL) goto _err; + if (appendActionCb(pTrans, pRaw) != 0) goto _err; + (void)sdbSetRawStatus(pRaw, dbStatus); + pRaw = NULL; + return 0; +_err: + sdbFreeRaw(pRaw); + return -1; +} + int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) { int32_t code = -1; STrans *pTrans = NULL; - SSdbRaw *pRaw = NULL; SDbObj dbObj = {0}; SArray *pArray = mndBuildDnodesArray(pMnode, 0); - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "split-vgroup"); + int32_t numOfTopics = 0; + if (mndGetNumOfTopics(pMnode, pDb->name, &numOfTopics) != 0) { + goto _OVER; + } + if (numOfTopics > 0) { + terrno = TSDB_CODE_MND_TOPIC_MUST_BE_DELETED; + goto _OVER; + } + + int32_t numOfStreams = 0; + if (mndGetNumOfStreams(pMnode, pDb->name, &numOfStreams) != 0) { + goto _OVER; + } + if (numOfStreams > 0) { + terrno = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; + goto _OVER; + } + + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "split-vgroup"); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to split vgroup, vgId:%d", pTrans->id, pVgroup->vgId); + mndTransSetDbName(pTrans, pDb->name, NULL); + SVgObj newVg1 = {0}; memcpy(&newVg1, pVgroup, sizeof(SVgObj)); mInfo("vgId:%d, vgroup info before split, replica:%d hashBegin:%u hashEnd:%u", newVg1.vgId, newVg1.replica, @@ -2286,45 +2368,54 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro // alter vgId and hash range int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); - if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, &newVg1, maxVgId) != 0) goto _OVER; + int32_t srcVgId = newVg1.vgId; newVg1.vgId = maxVgId; + if (mndAddPrepareNewVgAction(pMnode, pTrans, &newVg1) != 0) goto _OVER; + if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, srcVgId, &newVg1) != 0) goto _OVER; maxVgId++; - if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, &newVg2, maxVgId) != 0) goto _OVER; + srcVgId = newVg2.vgId; newVg2.vgId = maxVgId; + if (mndAddPrepareNewVgAction(pMnode, pTrans, &newVg2) != 0) goto _OVER; + if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, srcVgId, &newVg2) != 0) goto _OVER; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg2) != 0) goto _OVER; + if (mndAddVgStatusAction(pTrans, &newVg1, SDB_STATUS_READY, TRN_STAGE_REDO_ACTION) < 0) goto _OVER; + if (mndAddVgStatusAction(pTrans, &newVg2, SDB_STATUS_READY, TRN_STAGE_REDO_ACTION) < 0) goto _OVER; + if (mndAddVgStatusAction(pTrans, pVgroup, SDB_STATUS_DROPPED, TRN_STAGE_REDO_ACTION) < 0) goto _OVER; + + // update db status + memcpy(&dbObj, pDb, sizeof(SDbObj)); + if (dbObj.cfg.pRetensions != NULL) { + dbObj.cfg.pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); + if (dbObj.cfg.pRetensions == NULL) goto _OVER; + } + dbObj.vgVersion++; + dbObj.updateTime = taosGetTimestampMs(); + dbObj.cfg.numOfVgroups++; + if (mndAddDbStatusAction(pTrans, &dbObj, SDB_STATUS_READY, TRN_STAGE_REDO_ACTION) < 0) goto _OVER; + // adjust vgroup replica if (pDb->cfg.replications != newVg1.replica) { if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray) != 0) goto _OVER; } else { - if (mndTransCommitVgStatus(pTrans, &newVg1, SDB_STATUS_READY) < 0) goto _OVER; + if (mndAddVgStatusAction(pTrans, &newVg1, SDB_STATUS_READY, TRN_STAGE_COMMIT_ACTION) < 0) goto _OVER; } if (pDb->cfg.replications != newVg2.replica) { if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray) != 0) goto _OVER; } else { - if (mndTransCommitVgStatus(pTrans, &newVg2, SDB_STATUS_READY) < 0) goto _OVER; + if (mndAddVgStatusAction(pTrans, &newVg2, SDB_STATUS_READY, TRN_STAGE_COMMIT_ACTION) < 0) goto _OVER; } - if (mndTransCommitVgStatus(pTrans, pVgroup, SDB_STATUS_DROPPED) < 0) goto _OVER; + if (mndAddVgStatusAction(pTrans, pVgroup, SDB_STATUS_DROPPED, TRN_STAGE_COMMIT_ACTION) < 0) goto _OVER; - memcpy(&dbObj, pDb, sizeof(SDbObj)); - if (dbObj.cfg.pRetensions != NULL) { - dbObj.cfg.pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); - if (dbObj.cfg.pRetensions == NULL) goto _OVER; - } + // commit db status dbObj.vgVersion++; dbObj.updateTime = taosGetTimestampMs(); - dbObj.cfg.numOfVgroups++; - pRaw = mndDbActionEncode(&dbObj); - if (pRaw == NULL) goto _OVER; - if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; - (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); - pRaw = NULL; + if (mndAddDbStatusAction(pTrans, &dbObj, SDB_STATUS_READY, TRN_STAGE_COMMIT_ACTION) < 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -2332,7 +2423,6 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro _OVER: taosArrayDestroy(pArray); mndTransDrop(pTrans); - sdbFreeRaw(pRaw); taosArrayDestroy(dbObj.cfg.pRetensions); return code; } @@ -2501,6 +2591,7 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { pIter = sdbFetch(pMnode->pSdb, SDB_DNODE, pIter, (void **)&pDnode); if (pIter == NULL) break; if (!mndIsDnodeOnline(pDnode, curMs)) { + sdbCancelFetch(pMnode->pSdb, pIter); terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; mError("failed to balance vgroup since %s, dnode:%d", terrstr(), pDnode->id); sdbRelease(pMnode->pSdb, pDnode); diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index e9a9e425e3fd364a1702ca084edd8be4e0b4aeeb..3c96d8a2fd355de632392e4496a7a956b55bae45 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -122,6 +122,7 @@ typedef enum { SDB_STATUS_DROPPING = 2, SDB_STATUS_DROPPED = 3, SDB_STATUS_READY = 4, + SDB_STATUS_UPDATE = 5, } ESdbStatus; typedef enum { diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index f1cee6395b812c2bef8655530cd9f6ab211eedd3..258b22d8ee7f607b3239a2f5a278f942f7e5c428 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -256,6 +256,7 @@ int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw) { code = sdbInsertRow(pSdb, hash, pRaw, pRow, keySize); break; case SDB_STATUS_READY: + case SDB_STATUS_UPDATE: case SDB_STATUS_DROPPING: code = sdbUpdateRow(pSdb, hash, pRaw, pRow, keySize); break; diff --git a/source/dnode/qnode/CMakeLists.txt b/source/dnode/qnode/CMakeLists.txt index 5426cd55d3c1183f86e057baf205e199d6307fea..10bbbc1b26e94183da9ceeef6fadb700737fb1c4 100644 --- a/source/dnode/qnode/CMakeLists.txt +++ b/source/dnode/qnode/CMakeLists.txt @@ -14,4 +14,7 @@ target_link_libraries( PRIVATE qworker PRIVATE qcom PRIVATE executor + PRIVATE tdb + PRIVATE wal + PRIVATE index ) \ No newline at end of file diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 5efc714e95c85b528c24d64fc9642788d06c99ec..348235551271958318ec989056ff6d5b5037e7e7 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -14,7 +14,6 @@ */ #include "executor.h" -#include "libs/function/function.h" #include "qndInt.h" #include "query.h" #include "qworker.h" diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt index 6f1e7f9593a369afc0f3626514853873012da75f..ebfe80ecabe006f82bec5ec3c997064789ae69e8 100644 --- a/source/dnode/snode/CMakeLists.txt +++ b/source/dnode/snode/CMakeLists.txt @@ -14,4 +14,6 @@ target_link_libraries( PRIVATE util PRIVATE qcom PRIVATE stream + PRIVATE wal + PRIVATE index ) diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index 3fcee862a102acd202be4848af800f16a8ef3b66..68f7f756d50ed190ad2117b68f24408cbc773670 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -53,6 +53,8 @@ int32_t sndStopTaskOfStream(SStreamMeta* pMeta, int64_t streamId); int32_t sndResumeTaskOfStream(SStreamMeta* pMeta, int64_t streamId); #endif +void initStreamStateAPI(SStorageAPI* pAPI); + #ifdef __cplusplus } #endif diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 2ff338242f13e423f58d67ac1a45b13c102fa11f..4000e728359b7f88f339519474b2033b14502bf6 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -35,15 +35,13 @@ void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) { tDecoderClear(&decoder); - int32_t taskId = req.taskId; - - SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, taskId); + SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, req.taskId); if (pTask) { SRpcMsg rsp = { .info = pMsg->info, .code = 0, }; - streamProcessDispatchReq(pTask, &req, &rsp, false); + streamProcessDispatchMsg(pTask, &req, &rsp, false); streamMetaReleaseTask(pSnode->pMeta, pTask); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); @@ -52,31 +50,29 @@ void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) { FAIL: if (pMsg->info.handle == NULL) return; - SRpcMsg rsp = { - .code = code, - .info = pMsg->info, - }; + SRpcMsg rsp = { .code = code, .info = pMsg->info}; tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { - ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); - ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); + ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->pUpstreamEpInfoList) != 0); pTask->refCnt = 1; - pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; + pTask->id.idStr = createStreamTaskIdStr(pTask->id.streamId, pTask->id.taskId); - pTask->inputQueue = streamQueueOpen(0); - pTask->outputQueue = streamQueueOpen(0); + pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; + pTask->inputQueue = streamQueueOpen(512 << 10); + pTask->outputInfo.queue = streamQueueOpen(512 << 10); - if (pTask->inputQueue == NULL || pTask->outputQueue == NULL) { + if (pTask->inputQueue == NULL || pTask->outputInfo.queue == NULL) { return -1; } + pTask->tsInfo.init = taosGetTimestampMs(); pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; - pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; + pTask->outputInfo.status = TASK_OUTPUT_STATUS__NORMAL; pTask->pMsgCb = &pSnode->msgCb; pTask->chkInfo.version = ver; pTask->pMeta = pSnode->pMeta; @@ -86,13 +82,19 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { return -1; } - int32_t numOfChildEp = taosArrayGetSize(pTask->childEpInfo); - SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState }; + int32_t numOfChildEp = taosArrayGetSize(pTask->pUpstreamEpInfoList); + SReadHandle handle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState, .fillHistory = pTask->info.fillHistory }; + initStreamStateAPI(&handle.api); - pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, 0); + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, 0, pTask->id.taskId); ASSERT(pTask->exec.pExecutor); - streamSetupTrigger(pTask); + taosThreadMutexInit(&pTask->lock, NULL); + streamSetupScheduleTrigger(pTask); + + qDebug("snode:%d expand stream task on snode, s-task:%s, checkpoint ver:%" PRId64 " child id:%d, level:%d", SNODE_HANDLE, + pTask->id.idStr, pTask->chkInfo.version, pTask->info.selfChildId, pTask->info.taskLevel); + return 0; } @@ -149,39 +151,49 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { taosMemoryFree(pTask); return -1; } + tDecoderClear(&decoder); - ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); + ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG); // 2.save task taosWLockLatch(&pSnode->pMeta->lock); - code = streamMetaAddDeployedTask(pSnode->pMeta, -1, pTask); + + bool added = false; + code = streamMetaRegisterTask(pSnode->pMeta, -1, pTask, &added); if (code < 0) { taosWUnLockLatch(&pSnode->pMeta->lock); return -1; } + int32_t numOfTasks = streamMetaGetNumOfTasks(pSnode->pMeta); taosWUnLockLatch(&pSnode->pMeta->lock); + qDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE, pTask->id.idStr, + streamGetTaskStatusStr(pTask->status.taskStatus), numOfTasks); - // 3.go through recover steps to fill history - if (pTask->fillHistory) { - streamSetParamForRecover(pTask); - streamAggRecoverPrepare(pTask); - } - + streamTaskCheckDownstreamTasks(pTask); return 0; } int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) { SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg; - streamMetaRemoveTask(pSnode->pMeta, pReq->taskId); + qDebug("snode:%d receive msg to drop stream task:0x%x", pSnode->pMeta->vgId, pReq->taskId); + + SStreamTask* pTask = streamMetaAcquireTask(pSnode->pMeta, pReq->streamId, pReq->taskId); + if (pTask == NULL) { + qError("vgId:%d failed to acquire s-task:0x%x when dropping it", pSnode->pMeta->vgId, pReq->taskId); + return 0; + } + + streamMetaUnregisterTask(pSnode->pMeta, pReq->streamId, pReq->taskId); + streamMetaReleaseTask(pSnode->pMeta, pTask); return 0; } int32_t sndProcessTaskRunReq(SSnode *pSnode, SRpcMsg *pMsg) { SStreamTaskRunReq *pReq = pMsg->pCont; - int32_t taskId = pReq->taskId; - SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, taskId); + + SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, pReq->streamId, pReq->taskId); if (pTask) { streamProcessRunReq(pTask); streamMetaReleaseTask(pSnode->pMeta, pTask); @@ -199,21 +211,16 @@ int32_t sndProcessTaskDispatchReq(SSnode *pSnode, SRpcMsg *pMsg, bool exec) { SDecoder decoder; tDecoderInit(&decoder, (uint8_t *)msgBody, msgLen); tDecodeStreamDispatchReq(&decoder, &req); - int32_t taskId = req.taskId; - SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, taskId); + SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, req.taskId); if (pTask) { - SRpcMsg rsp = { - .info = pMsg->info, - .code = 0, - }; - streamProcessDispatchReq(pTask, &req, &rsp, exec); + SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; + streamProcessDispatchMsg(pTask, &req, &rsp, exec); streamMetaReleaseTask(pSnode->pMeta, pTask); return 0; } else { return -1; } - return 0; } int32_t sndProcessTaskRetrieveReq(SSnode *pSnode, SRpcMsg *pMsg) { @@ -225,13 +232,10 @@ int32_t sndProcessTaskRetrieveReq(SSnode *pSnode, SRpcMsg *pMsg) { tDecoderInit(&decoder, msgBody, msgLen); tDecodeStreamRetrieveReq(&decoder, &req); tDecoderClear(&decoder); - int32_t taskId = req.dstTaskId; - SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, taskId); + SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, req.dstTaskId); + if (pTask) { - SRpcMsg rsp = { - .info = pMsg->info, - .code = 0, - }; + SRpcMsg rsp = { .info = pMsg->info, .code = 0}; streamProcessRetrieveReq(pTask, &req, &rsp); streamMetaReleaseTask(pSnode->pMeta, pTask); tDeleteStreamRetrieveReq(&req); @@ -243,8 +247,11 @@ int32_t sndProcessTaskRetrieveReq(SSnode *pSnode, SRpcMsg *pMsg) { int32_t sndProcessTaskDispatchRsp(SSnode *pSnode, SRpcMsg *pMsg) { SStreamDispatchRsp *pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t taskId = ntohl(pRsp->upstreamTaskId); - SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, taskId); + + int32_t taskId = htonl(pRsp->upstreamTaskId); + int64_t streamId = htobe64(pRsp->streamId); + + SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, streamId, taskId); if (pTask) { streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamMetaReleaseTask(pSnode->pMeta, pTask); @@ -252,7 +259,6 @@ int32_t sndProcessTaskDispatchRsp(SSnode *pSnode, SRpcMsg *pMsg) { } else { return -1; } - return 0; } int32_t sndProcessTaskRetrieveRsp(SSnode *pSnode, SRpcMsg *pMsg) { @@ -261,38 +267,40 @@ int32_t sndProcessTaskRetrieveRsp(SSnode *pSnode, SRpcMsg *pMsg) { } int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { - void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); switch (pMsg->msgType) { - case TDMT_STREAM_TASK_DEPLOY: + case TDMT_STREAM_TASK_DEPLOY: { + void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); return sndProcessTaskDeployReq(pSnode, pReq, len); + } + case TDMT_STREAM_TASK_DROP: - return sndProcessTaskDropReq(pSnode, pReq, len); + return sndProcessTaskDropReq(pSnode, pMsg->pCont, pMsg->contLen); default: ASSERT(0); } return 0; } -int32_t sndProcessTaskRecoverFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { +int32_t sndProcessStreamTaskScanHistoryFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { char *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); // deserialize - SStreamRecoverFinishReq req; + SStreamScanHistoryFinishReq req; SDecoder decoder; tDecoderInit(&decoder, msg, msgLen); - tDecodeSStreamRecoverFinishReq(&decoder, &req); + tDecodeStreamScanHistoryFinishReq(&decoder, &req); tDecoderClear(&decoder); // find task - SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.taskId); + SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, req.downstreamTaskId); if (pTask == NULL) { return -1; } // do process request - if (streamProcessRecoverFinishReq(pTask, req.childId) < 0) { + if (streamProcessScanHistoryFinishReq(pTask, &req, &pMsg->info) < 0) { streamMetaReleaseTask(pSnode->pMeta, pTask); return -1; } @@ -306,6 +314,102 @@ int32_t sndProcessTaskRecoverFinishRsp(SSnode *pSnode, SRpcMsg *pMsg) { return 0; } +int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) { + char *msgStr = pMsg->pCont; + char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + SStreamTaskCheckReq req; + SDecoder decoder; + + tDecoderInit(&decoder, (uint8_t *)msgBody, msgLen); + tDecodeStreamTaskCheckReq(&decoder, &req); + tDecoderClear(&decoder); + + int32_t taskId = req.downstreamTaskId; + + SStreamTaskCheckRsp rsp = { + .reqId = req.reqId, + .streamId = req.streamId, + .childId = req.childId, + .downstreamNodeId = req.downstreamNodeId, + .downstreamTaskId = req.downstreamTaskId, + .upstreamNodeId = req.upstreamNodeId, + .upstreamTaskId = req.upstreamTaskId, + }; + + SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, taskId); + + if (pTask != NULL) { + rsp.status = streamTaskCheckStatus(pTask); + streamMetaReleaseTask(pSnode->pMeta, pTask); + + const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); + qDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d", + pTask->id.idStr, pStatus, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + } else { + rsp.status = 0; + qDebug("tq recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 + ") from task:0x%x (vgId:%d), rsp status %d", + taskId, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + } + + SEncoder encoder; + int32_t code; + int32_t len; + + tEncodeSize(tEncodeStreamTaskCheckRsp, &rsp, len, code); + if (code < 0) { + qError("vgId:%d failed to encode task check rsp, task:0x%x", pSnode->pMeta->vgId, taskId); + return -1; + } + + void *buf = rpcMallocCont(sizeof(SMsgHead) + len); + ((SMsgHead *)buf)->vgId = htonl(req.upstreamNodeId); + + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncoderInit(&encoder, (uint8_t *)abuf, len); + tEncodeStreamTaskCheckRsp(&encoder, &rsp); + tEncoderClear(&encoder); + + SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = pMsg->info}; + + tmsgSendRsp(&rspMsg); + return 0; +} + +int32_t sndProcessStreamTaskCheckRsp(SSnode* pSnode, SRpcMsg* pMsg) { + char* pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + + int32_t code; + SStreamTaskCheckRsp rsp; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)pReq, len); + code = tDecodeStreamTaskCheckRsp(&decoder, &rsp); + + if (code < 0) { + tDecoderClear(&decoder); + return -1; + } + + tDecoderClear(&decoder); + qDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", + rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); + + SStreamTask* pTask = streamMetaAcquireTask(pSnode->pMeta, rsp.streamId, rsp.upstreamTaskId); + if (pTask == NULL) { + qError("tq failed to locate the stream task:0x%x (vgId:%d), it may have been destroyed", rsp.upstreamTaskId, + pSnode->pMeta->vgId); + return -1; + } + + code = streamProcessCheckRsp(pTask, &rsp); + streamMetaReleaseTask(pSnode->pMeta, pTask); + return code; +} + int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { case TDMT_STREAM_TASK_RUN: @@ -318,10 +422,14 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { return sndProcessTaskRetrieveReq(pSnode, pMsg); case TDMT_STREAM_RETRIEVE_RSP: return sndProcessTaskRetrieveRsp(pSnode, pMsg); - case TDMT_STREAM_RECOVER_FINISH: - return sndProcessTaskRecoverFinishReq(pSnode, pMsg); - case TDMT_STREAM_RECOVER_FINISH_RSP: + case TDMT_STREAM_SCAN_HISTORY_FINISH: + return sndProcessStreamTaskScanHistoryFinishReq(pSnode, pMsg); + case TDMT_STREAM_SCAN_HISTORY_FINISH_RSP: return sndProcessTaskRecoverFinishRsp(pSnode, pMsg); + case TDMT_STREAM_TASK_CHECK: + return sndProcessStreamTaskCheckReq(pSnode, pMsg); + case TDMT_STREAM_TASK_CHECK_RSP: + return sndProcessStreamTaskCheckRsp(pSnode, pMsg); default: ASSERT(0); } diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c new file mode 100644 index 0000000000000000000000000000000000000000..e737e3fa373ed621ec87c3267ce0cb964ae0c19e --- /dev/null +++ b/source/dnode/snode/src/snodeInitApi.c @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "storageapi.h" +#include "tstreamUpdate.h" +#include "streamState.h" + +static void initStateStoreAPI(SStateStore* pStore); +static void initFunctionStateStore(SFunctionStateStore* pStore); + +void initStreamStateAPI(SStorageAPI* pAPI) { + initStateStoreAPI(&pAPI->stateStore); + initFunctionStateStore(&pAPI->functionStore); +} + +void initStateStoreAPI(SStateStore* pStore) { + pStore->streamFileStateInit = streamFileStateInit; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + + pStore->streamStateGetByPos = streamStateGetByPos; + + pStore->streamStatePutParName = streamStatePutParName; + pStore->streamStateGetParName = streamStateGetParName; + + pStore->streamStateAddIfNotExist = streamStateAddIfNotExist; + pStore->streamStateReleaseBuf = streamStateReleaseBuf; + pStore->streamStateFreeVal = streamStateFreeVal; + + pStore->streamStatePut = streamStatePut; + pStore->streamStateGet = streamStateGet; + pStore->streamStateCheck = streamStateCheck; + pStore->streamStateGetByPos = streamStateGetByPos; + pStore->streamStateDel = streamStateDel; + pStore->streamStateClear = streamStateClear; + pStore->streamStateSaveInfo = streamStateSaveInfo; + pStore->streamStateGetInfo = streamStateGetInfo; + pStore->streamStateSetNumber = streamStateSetNumber; + + pStore->streamStateFillPut = streamStateFillPut; + pStore->streamStateFillGet = streamStateFillGet; + pStore->streamStateFillDel = streamStateFillDel; + + pStore->streamStateCurNext = streamStateCurNext; + pStore->streamStateCurPrev = streamStateCurPrev; + + pStore->streamStateGetAndCheckCur = streamStateGetAndCheckCur; + pStore->streamStateSeekKeyNext = streamStateSeekKeyNext; + pStore->streamStateFillSeekKeyNext = streamStateFillSeekKeyNext; + pStore->streamStateFillSeekKeyPrev = streamStateFillSeekKeyPrev; + pStore->streamStateFreeCur = streamStateFreeCur; + + pStore->streamStateGetGroupKVByCur = streamStateGetGroupKVByCur; + pStore->streamStateGetKVByCur = streamStateGetKVByCur; + + pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist; + pStore->streamStateSessionPut = streamStateSessionPut; + pStore->streamStateSessionGet = streamStateSessionGet; + pStore->streamStateSessionDel = streamStateSessionDel; + pStore->streamStateSessionClear = streamStateSessionClear; + pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur; + pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist; + pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange; + + pStore->updateInfoInit = updateInfoInit; + pStore->updateInfoFillBlockData = updateInfoFillBlockData; + pStore->updateInfoIsUpdated = updateInfoIsUpdated; + pStore->updateInfoIsTableInserted = updateInfoIsTableInserted; + pStore->updateInfoDestroy = updateInfoDestroy; + pStore->windowSBfDelete = windowSBfDelete; + pStore->windowSBfAdd = windowSBfAdd; + + pStore->updateInfoInitP = updateInfoInitP; + pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + pStore->updateInfoSerialize = updateInfoSerialize; + pStore->updateInfoDeserialize = updateInfoDeserialize; + + pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext; + pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev; + pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext; + + pStore->streamFileStateInit = streamFileStateInit; + + pStore->streamFileStateDestroy = streamFileStateDestroy; + pStore->streamFileStateClear = streamFileStateClear; + pStore->needClearDiskBuff = needClearDiskBuff; + + pStore->streamStateOpen = streamStateOpen; + pStore->streamStateClose = streamStateClose; + pStore->streamStateBegin = streamStateBegin; + pStore->streamStateCommit = streamStateCommit; + pStore->streamStateDestroy= streamStateDestroy; + pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; + pStore->streamStateReloadInfo = streamStateReloadInfo; +} + +void initFunctionStateStore(SFunctionStateStore* pStore) { + pStore->streamStateFuncPut = streamStateFuncPut; + pStore->streamStateFuncGet = streamStateFuncGet; +} \ No newline at end of file diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e8660cd6adff1283f3852abf52aa3e7769250dc5..194ffa16f671b8b63466d3bb1de559b470547de4 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -1,20 +1,18 @@ # vnode add_library(vnode STATIC "") -target_sources( - vnode - PRIVATE - - # vnode - "src/vnd/vnodeOpen.c" - "src/vnd/vnodeBufPool.c" - "src/vnd/vnodeCfg.c" - "src/vnd/vnodeCommit.c" - "src/vnd/vnodeQuery.c" - "src/vnd/vnodeModule.c" - "src/vnd/vnodeSvr.c" - "src/vnd/vnodeSync.c" - "src/vnd/vnodeSnapshot.c" - "src/vnd/vnodeRetention.c" +set( + VNODE_SOURCE_FILES + "src/vnd/vnodeOpen.c" + "src/vnd/vnodeBufPool.c" + "src/vnd/vnodeCfg.c" + "src/vnd/vnodeCommit.c" + "src/vnd/vnodeQuery.c" + "src/vnd/vnodeModule.c" + "src/vnd/vnodeSvr.c" + "src/vnd/vnodeSync.c" + "src/vnd/vnodeSnapshot.c" + "src/vnd/vnodeRetention.c" + "src/vnd/vnodeInitApi.c" # meta "src/meta/metaOpen.c" @@ -26,34 +24,34 @@ target_sources( "src/meta/metaEntry.c" "src/meta/metaSnapshot.c" "src/meta/metaCache.c" + "src/meta/metaTtl.c" # sma "src/sma/smaEnv.c" "src/sma/smaUtil.c" - "src/sma/smaFS.c" "src/sma/smaOpen.c" "src/sma/smaCommit.c" "src/sma/smaRollup.c" "src/sma/smaSnapshot.c" "src/sma/smaTimeRange.c" - # tsdb - "src/tsdb/tsdbCommit.c" - "src/tsdb/tsdbFile.c" - "src/tsdb/tsdbFS.c" - "src/tsdb/tsdbOpen.c" - "src/tsdb/tsdbMemTable.c" - "src/tsdb/tsdbRead.c" - "src/tsdb/tsdbCache.c" - "src/tsdb/tsdbWrite.c" - "src/tsdb/tsdbReaderWriter.c" - "src/tsdb/tsdbUtil.c" - "src/tsdb/tsdbSnapshot.c" - "src/tsdb/tsdbCacheRead.c" - "src/tsdb/tsdbRetention.c" - "src/tsdb/tsdbDiskData.c" - "src/tsdb/tsdbMergeTree.c" - "src/tsdb/tsdbDataIter.c" + # # tsdb + # "src/tsdb/tsdbCommit.c" + # "src/tsdb/tsdbFile.c" + # "src/tsdb/tsdbFS.c" + # "src/tsdb/tsdbOpen.c" + # "src/tsdb/tsdbMemTable.c" + # "src/tsdb/tsdbRead.c" + # "src/tsdb/tsdbCache.c" + # "src/tsdb/tsdbWrite.c" + # "src/tsdb/tsdbReaderWriter.c" + # "src/tsdb/tsdbUtil.c" + # "src/tsdb/tsdbSnapshot.c" + # "src/tsdb/tsdbCacheRead.c" + # "src/tsdb/tsdbRetention.c" + # "src/tsdb/tsdbDiskData.c" + # "src/tsdb/tsdbMergeTree.c" + # "src/tsdb/tsdbDataIter.c" # tq "src/tq/tq.c" @@ -70,6 +68,19 @@ target_sources( "src/tq/tqOffsetSnapshot.c" ) +aux_source_directory("src/tsdb/" TSDB_SOURCE_FILES) +list( + APPEND + VNODE_SOURCE_FILES + ${TSDB_SOURCE_FILES} +) + +target_sources( + vnode + PRIVATE + ${VNODE_SOURCE_FILES} +) + IF (TD_VNODE_PLUGINS) target_sources( vnode @@ -79,13 +90,50 @@ IF (TD_VNODE_PLUGINS) ) ENDIF () -target_include_directories( - vnode - PUBLIC "inc" - PUBLIC "src/inc" - PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" - PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include" -) +# IF (NOT ${TD_LINUX}) +# target_include_directories( +# vnode +# PUBLIC "inc" +# PUBLIC "src/inc" +# PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" +# PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include" +# ) +# ELSE() +# target_include_directories( +# vnode +# PUBLIC "inc" +# PUBLIC "src/inc" +# PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" +# ) +#ENDIF(NOT ${TD_LINUX}) + +if (${BUILD_CONTRIB}) + target_include_directories( + vnode + PUBLIC "inc" + PUBLIC "src/inc" + PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" + PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include" + ) +else() + target_include_directories( + vnode + PUBLIC "inc" + PUBLIC "src/inc" + PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" + ) + if (${TD_LINUX}) + target_include_directories( + vnode + PUBLIC "${TD_SOURCE_DIR}/deps/${TD_DEPS_DIR}/rocksdb_static" + ) + target_link_directories( + vnode + PUBLIC "${TD_SOURCE_DIR}/deps/${TD_DEPS_DIR}/rocksdb_static" + ) + endif() +endif() + target_link_libraries( vnode PUBLIC os diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index d098ec9be2d9332ba7698065af19e1b48ad603b7..a7ce18198dfcd4fe93675a936d43585b83a43d79 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -26,6 +26,7 @@ #include "tfs.h" #include "wal.h" +#include "filter.h" #include "tcommon.h" #include "tfs.h" #include "tgrant.h" @@ -33,6 +34,7 @@ #include "trow.h" #include "tdb.h" +#include "storageapi.h" #ifdef __cplusplus extern "C" { @@ -49,11 +51,14 @@ extern const SVnodeCfg vnodeCfgDefault; int32_t vnodeInit(int32_t nthreads); void vnodeCleanup(); -int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs); -int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs); -int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq, STfs *pTfs); +int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs *pTfs); +int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t diskPrimary, STfs *pTfs); +int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq, + int32_t diskPrimary, STfs *pTfs); +int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, + int32_t diskPrimary, STfs *pTfs); void vnodeDestroy(const char *path, STfs *pTfs); -SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb); +SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb); void vnodePreClose(SVnode *pVnode); void vnodePostClose(SVnode *pVnode); void vnodeSyncCheckTimeout(SVnode *pVnode); @@ -65,17 +70,19 @@ int32_t vnodeStart(SVnode *pVnode); void vnodeStop(SVnode *pVnode); int64_t vnodeGetSyncHandle(SVnode *pVnode); void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); -void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId); +void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); +int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList); int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list); int32_t vnodeIsCatchUp(SVnode *pVnode); ESyncRole vnodeGetRole(SVnode *pVnode); -int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list); +int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list); -void *vnodeGetIdx(SVnode *pVnode); -void *vnodeGetIvtIdx(SVnode *pVnode); +int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void* arg1), void *arg); +void *vnodeGetIdx(void *pVnode); +void *vnodeGetIvtIdx(void *pVnode); int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num); int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num); @@ -92,76 +99,41 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); +int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit); // meta -typedef struct SMeta SMeta; // todo: remove -typedef struct SMetaReader SMetaReader; -typedef struct SMetaEntry SMetaEntry; - -#define META_READER_NOLOCK 0x1 - -void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); +void _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta* pAPI); void metaReaderReleaseLock(SMetaReader *pReader); void metaReaderClear(SMetaReader *pReader); -int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); -int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); -int metaGetTableEntryByName(SMetaReader *pReader, const char *name); -int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); -int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList); +int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); +int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); +int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *uidList); +int32_t metaGetTableTagsByUids(void* pVnode, int64_t suid, SArray *uidList); int32_t metaReadNext(SMetaReader *pReader); -const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); +const void *metaGetTableTagVal(const void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); -int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); +int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid); int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); -bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); -int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, +int metaGetTableTtlByUid(void *meta, uint64_t uid, int64_t *ttlDays); +bool metaIsTableExist(void* pVnode, tb_uid_t uid); +int32_t metaGetCachedTableUidList(void *pVnode, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, bool *acquired); -int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, +int32_t metaUidFilterCachePut(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen, double selectivityRatio); -int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); -int64_t metaGetTbNum(SMeta *pMeta); -int64_t metaGetNtbNum(SMeta *pMeta); -typedef struct { - int64_t uid; - int64_t ctbNum; -} SMetaStbStats; -int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); - -typedef struct SMetaFltParam { - tb_uid_t suid; - int16_t cid; - int16_t type; - void *val; - bool reverse; - bool equal; - int (*filterFunc)(void *a, void *b, int16_t type); - -} SMetaFltParam; - -// TODO, refactor later -int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *results); -int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *parm, SArray *pUids); -int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); -int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); - -#if 1 // refact APIs below (TODO) -typedef SVCreateTbReq STbCfg; -typedef SVCreateTSmaReq SSmaCfg; - -typedef struct SMTbCursor SMTbCursor; - -SMTbCursor *metaOpenTbCursor(SMeta *pMeta); -void metaCloseTbCursor(SMTbCursor *pTbCur); -int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); -int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); +int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); +int32_t metaPutTbGroupToCache(void* pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, + int32_t payloadLen); +bool metaTbInFilterCache(void *pVnode, tb_uid_t suid, int8_t type); +int32_t metaPutTbToFilterCache(void *pVnode, tb_uid_t suid, int8_t type); +int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type); -#endif +int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables); // tsdb typedef struct STsdbReader STsdbReader; @@ -177,14 +149,14 @@ typedef struct STsdbReader STsdbReader; #define CACHESCAN_RETRIEVE_LAST_ROW 0x4 #define CACHESCAN_RETRIEVE_LAST 0x8 -int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); -int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly); - +int32_t tsdbReaderOpen(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, + SSDataBlock *pResBlock, void **ppReader, const char *idstr, bool countOnly, + SHashObj **pIgnoreTables); +int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); void tsdbReaderSetId(STsdbReader *pReader, const char *idstr); void tsdbReaderClose(STsdbReader *pReader); int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); void tsdbReleaseDataBlock(STsdbReader *pReader); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); @@ -193,49 +165,49 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); void *tsdbGetIdx(SMeta *pMeta); void *tsdbGetIvtIdx(SMeta *pMeta); uint64_t tsdbGetReaderMaxVersion(STsdbReader *pReader); -int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); -void tsdbReaderSetId(STsdbReader *pReader, const char *idstr); void tsdbReaderSetCloseFlag(STsdbReader *pReader); - +int64_t tsdbGetLastTimestamp(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); + +//====================================================================================================================== +int32_t tsdbReaderOpen2(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, + SSDataBlock *pResBlock, void **ppReader, const char *idstr, bool countOnly, + SHashObj **pIgnoreTables); +int32_t tsdbSetTableList2(STsdbReader *pReader, const void *pTableList, int32_t num); +void tsdbReaderSetId2(STsdbReader *pReader, const char *idstr); +void tsdbReaderClose2(STsdbReader *pReader); +int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext); +int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); +void tsdbReleaseDataBlock2(STsdbReader *pReader); +SSDataBlock *tsdbRetrieveDataBlock2(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); +int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond); +int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); +int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle); +void *tsdbGetIdx2(SMeta *pMeta); +void *tsdbGetIvtIdx2(SMeta *pMeta); +uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader); +void tsdbReaderSetCloseFlag2(STsdbReader *pReader); +int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); +//====================================================================================================================== + +int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, SArray *pTableUids); void *tsdbCacherowsReaderClose(void *pReader); -int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); +int32_t tsdbGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); size_t tsdbCacheGetCapacity(SVnode *pVnode); size_t tsdbCacheGetUsage(SVnode *pVnode); int32_t tsdbCacheGetElems(SVnode *pVnode); -// tq -typedef struct SMetaTableInfo { - int64_t suid; - int64_t uid; - SSchemaWrapper *schema; - char tbName[TSDB_TABLE_NAME_LEN]; -} SMetaTableInfo; - +//// tq typedef struct SIdInfo { int64_t version; int32_t index; } SIdInfo; -typedef struct SSnapContext { - SMeta *pMeta; - int64_t snapVersion; - TBC *pCur; - int64_t suid; - int8_t subType; - SHashObj *idVersion; - SHashObj *suidInfo; - SArray *idList; - int32_t index; - bool withMeta; - bool queryMeta; // true-get meta, false-get data -} SSnapContext; - typedef struct STqReader { SPackedData msg; SSubmitReq2 submit; @@ -253,21 +225,26 @@ typedef struct STqReader { } STqReader; STqReader *tqReaderOpen(SVnode *pVnode); -void tqCloseReader(STqReader *); +void tqReaderClose(STqReader *); void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); -int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList, const char* id); int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); -int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); -int32_t tqNextBlockInWal(STqReader* pReader); -bool tqNextBlockImpl(STqReader *pReader, const char* idstr); +bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid); +bool tqCurrentBlockConsumed(const STqReader* pReader); + +int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); +bool tqNextBlockInWal(STqReader *pReader, const char *idstr); +bool tqNextBlockImpl(STqReader *pReader, const char *idstr); +SWalReader* tqGetWalReader(STqReader* pReader); +SSDataBlock* tqGetResultBlock (STqReader* pReader); -int32_t extractSubmitMsgFromWal(SWalReader *pReader, SPackedData *pPackedData); +int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, int64_t maxVer, const char *id); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(STqReader *pReader, const char* idstr); +int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock** pRes, const char* idstr); int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); @@ -284,10 +261,10 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot); int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData); -int32_t buildSnapContext(SMeta *pMeta, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, +int32_t buildSnapContext(SVnode *pVnode, int64_t snapVersion, int64_t suid, int8_t subType, int8_t withMeta, SSnapContext **ctxRet); -int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); -SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx); +int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); +SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx); int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); int32_t destroySnapContext(SSnapContext *ctx); @@ -346,67 +323,9 @@ struct SVnodeCfg { int32_t tsdbPageSize; }; -typedef struct { - uint64_t uid; - uint64_t groupId; -} STableKeyInfo; - #define TABLE_ROLLUP_ON ((int8_t)0x1) #define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0) #define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON) -struct SMetaEntry { - int64_t version; - int8_t type; - int8_t flags; // TODO: need refactor? - tb_uid_t uid; - char *name; - union { - struct { - SSchemaWrapper schemaRow; - SSchemaWrapper schemaTag; - SRSmaParam rsmaParam; - } stbEntry; - struct { - int64_t ctime; - int32_t ttlDays; - int32_t commentLen; - char *comment; - tb_uid_t suid; - uint8_t *pTags; - } ctbEntry; - struct { - int64_t ctime; - int32_t ttlDays; - int32_t commentLen; - char *comment; - int32_t ncid; // next column id - SSchemaWrapper schemaRow; - } ntbEntry; - struct { - STSma *tsma; - } smaEntry; - }; - - uint8_t *pBuf; -}; - -struct SMetaReader { - int32_t flags; - SMeta *pMeta; - SDecoder coder; - SMetaEntry me; - void *pBuf; - int32_t szBuf; -}; - -struct SMTbCursor { - TBC *pDbc; - void *pKey; - void *pVal; - int32_t kLen; - int32_t vLen; - SMetaReader mr; -}; #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 3999aa0b7f22a53197413048eca7842e6fe5e57d..4f25bf31a210863aa75877b87894a8f9159dfa7e 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -17,6 +17,7 @@ #define _TD_VNODE_META_H_ #include "index.h" +#include "metaTtl.h" #include "vnodeInt.h" #ifdef __cplusplus @@ -89,10 +90,10 @@ struct SMeta { // ivt idx and idx void* pTagIvtIdx; - TTB* pTagIdx; - TTB* pTtlIdx; + TTB* pTagIdx; + STtlManger* pTtlMgr; - TTB* pCtimeIdx; // table created time idx + TTB* pBtimeIdx; // table created time idx TTB* pNcolIdx; // ncol of table idx, normal table only TTB* pSmaIdx; @@ -138,20 +139,15 @@ typedef struct { } STagIdxKey; #pragma pack(pop) -typedef struct { - int64_t dtime; - tb_uid_t uid; -} STtlIdxKey; - typedef struct { tb_uid_t uid; int64_t smaUid; } SSmaIdxKey; typedef struct { - int64_t ctime; + int64_t btime; tb_uid_t uid; -} SCtimeIdxKey; +} SBtimeIdxKey; typedef struct { int64_t ncol; @@ -162,6 +158,12 @@ typedef struct { int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey); +// TODO, refactor later +int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *param, SArray *results); +int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *parm, SArray *pUids); +int32_t metaFilterTableName(void *pVnode, SMetaFltParam *param, SArray *pUids); +int32_t metaFilterTtl(void *pVnode, SMetaFltParam *param, SArray *pUids); + #ifndef META_REFACT // SMetaDB int metaOpenDB(SMeta* pMeta); diff --git a/source/dnode/vnode/src/inc/metaTtl.h b/source/dnode/vnode/src/inc/metaTtl.h new file mode 100644 index 0000000000000000000000000000000000000000..45faceb1ea2bfa7468e486ab547cc3cd83521ba7 --- /dev/null +++ b/source/dnode/vnode/src/inc/metaTtl.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_TTL_H_ +#define _TD_VNODE_TTL_H_ + +#include "taosdef.h" +#include "thash.h" + +#include "tdb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum DirtyEntryType { + ENTRY_TYPE_DEL = 1, + ENTRY_TYPE_UPSERT = 2, +} DirtyEntryType; + +typedef struct STtlManger { + TdThreadRwlock lock; + + TTB* pOldTtlIdx; // btree<{deleteTime, tuid}, NULL> + + SHashObj* pTtlCache; // key: tuid, value: {ttl, ctime} + SHashObj* pDirtyUids; // dirty tuid + TTB* pTtlIdx; // btree<{deleteTime, tuid}, ttl> + + char* logPrefix; +} STtlManger; + +typedef struct { + int64_t ttlDays; + int64_t changeTimeMs; +} STtlCacheEntry; + +typedef struct { + DirtyEntryType type; +} STtlDirtyEntry; + +typedef struct { + int64_t deleteTimeSec; + tb_uid_t uid; +} STtlIdxKey; + +typedef struct { + int64_t deleteTimeMs; + tb_uid_t uid; +} STtlIdxKeyV1; + +typedef struct { + int64_t ttlDays; +} STtlIdxValue; + +typedef struct { + tb_uid_t uid; + int64_t changeTimeMs; +} STtlUpdCtimeCtx; + +typedef struct { + tb_uid_t uid; + int64_t changeTimeMs; + int64_t ttlDays; +} STtlUpdTtlCtx; + +typedef struct { + tb_uid_t uid; + TXN* pTxn; + int64_t ttlDays; +} STtlDelTtlCtx; + +int ttlMgrOpen(STtlManger** ppTtlMgr, TDB* pEnv, int8_t rollback, const char* logPrefix); +void ttlMgrClose(STtlManger* pTtlMgr); +int ttlMgrPostOpen(STtlManger* pTtlMgr, void* pMeta); + +bool ttlMgrNeedUpgrade(TDB* pEnv); +int ttlMgrUpgrade(STtlManger* pTtlMgr, void* pMeta); + +int ttlMgrInsertTtl(STtlManger* pTtlMgr, const STtlUpdTtlCtx* pUpdCtx); +int ttlMgrDeleteTtl(STtlManger* pTtlMgr, const STtlDelTtlCtx* pDelCtx); +int ttlMgrUpdateChangeTime(STtlManger* pTtlMgr, const STtlUpdCtimeCtx* pUpdCtimeCtx); + +int ttlMgrFlush(STtlManger* pTtlMgr, TXN* pTxn); +int ttlMgrFindExpired(STtlManger* pTtlMgr, int64_t timePointMs, SArray* pTbUids); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_TTL_H_*/ diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index c3e8d7ef1dacd2d0c7aaf143f09d3a9de88eec2e..aaf0973b41a6965f4aaeca44877511bbd3c62727 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -105,17 +105,16 @@ struct SRSmaFS { struct SRSmaStat { SSma *pSma; - int64_t commitAppliedVer; // vnode applied version for async commit - int64_t refId; // shared by fetch tasks - volatile int64_t nBufItems; // number of items in queue buffer - SRWLatch lock; // r/w lock for rsma fs(e.g. qtaskinfo) - volatile int32_t nFetchAll; // active number of fetch all - volatile int8_t triggerStat; // shared by fetch tasks - volatile int8_t commitStat; // 0 not in committing, 1 in committing - volatile int8_t delFlag; // 0 no deleted SRSmaInfo, 1 has deleted SRSmaInfo - SRSmaFS fs; // for recovery/snapshot r/w - SHashObj *infoHash; // key: suid, value: SRSmaInfo - tsem_t notEmpty; // has items in queue buffer + int64_t refId; // shared by fetch tasks + volatile int64_t nBufItems; // number of items in queue buffer + SRWLatch lock; // r/w lock for rsma fs(e.g. qtaskinfo) + volatile int32_t nFetchAll; // active number of fetch all + volatile int8_t triggerStat; // shared by fetch tasks + volatile int8_t commitStat; // 0 not in committing, 1 in committing + volatile int8_t delFlag; // 0 no deleted SRSmaInfo, 1 has deleted SRSmaInfo + SRSmaFS fs; // for recovery/snapshot r/w + SHashObj *infoHash; // key: suid, value: SRSmaInfo + tsem_t notEmpty; // has items in queue buffer }; struct SSmaStat { @@ -156,12 +155,9 @@ struct SRSmaInfo { int16_t padding; T_REF_DECLARE() SRSmaInfoItem items[TSDB_RETENTION_L2]; - void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t - STaosQueue *queue; // buffer queue of SubmitReq - STaosQall *qall; // buffer qall of SubmitReq - void *iTaskInfo[TSDB_RETENTION_L2]; // immutable qTaskInfo_t - STaosQueue *iQueue; // immutable buffer queue of SubmitReq - STaosQall *iQall; // immutable buffer qall of SubmitReq + void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t + STaosQueue *queue; // buffer queue of SubmitReq + STaosQall *qall; // buffer qall of SubmitReq }; #define RSMA_INFO_HEAD_LEN offsetof(SRSmaInfo, items) @@ -191,6 +187,12 @@ typedef enum { RSMA_EXEC_COMMIT = 3, // triggered by commit } ERsmaExecType; +#define TD_SMA_LOOPS_CHECK(n, limit) \ + if (++(n) > limit) { \ + sched_yield(); \ + (n) = 0; \ + } + // sma int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType); void tdDestroySmaEnv(SSmaEnv *pSmaEnv); @@ -213,27 +215,12 @@ int32_t smaPreClose(SSma *pSma); // rsma void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); -int32_t tdRSmaFSOpen(SSma *pSma, int64_t version, int8_t rollback); -void tdRSmaFSClose(SRSmaFS *fs); -int32_t tdRSmaFSPrepareCommit(SSma *pSma, SRSmaFS *pFSNew); -int32_t tdRSmaFSCommit(SSma *pSma); -int32_t tdRSmaFSFinishCommit(SSma *pSma); -int32_t tdRSmaFSCopy(SSma *pSma, SRSmaFS *pFS); -int32_t tdRSmaFSTakeSnapshot(SSma *pSma, SRSmaFS *pFS); -int32_t tdRSmaFSRef(SSma *pSma, SRSmaFS *pFS); -void tdRSmaFSUnRef(SSma *pSma, SRSmaFS *pFS); -int32_t tdRSmaFSUpsertQTaskFile(SSma *pSma, SRSmaFS *pFS, SQTaskFile *qTaskFile, int32_t nSize); -int32_t tdRSmaFSRollback(SSma *pSma); int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback); int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type); -int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); +// int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer, int8_t rollback); -void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t suid, int8_t level, int64_t version, char *outputName); -void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t suid, int8_t level, int64_t version, const char *path, - char *outputName); -void tdRSmaQTaskInfoGetFullPath(int32_t vgId, int8_t level, const char *path, char *outputName); -void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, const char *path, char *outputName); +void tdRSmaQTaskInfoGetFullPath(SVnode *pVnode, tb_uid_t suid, int8_t level, STfs *pTfs, char *outputName); static FORCE_INLINE void tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { int32_t ref = T_REF_INC(pRSmaInfo); @@ -244,9 +231,7 @@ static FORCE_INLINE void tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { smaTrace("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); } -void tdRSmaGetFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t suid, - int8_t level, int64_t version, char *outputName); -void tdRSmaGetDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName); +void tdRSmaGetDirName(SVnode *pVnode, STfs *pTfs, bool endWithSep, char *outputName); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index b5b8259fb4abfb32c144cd00dc4191e0cedad90e..a6a84075b59081bc245f2b9bac763dd386fc36c7 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -45,33 +45,18 @@ extern "C" { typedef struct STqOffsetStore STqOffsetStore; // tqPush - -//typedef struct { -// // msg info -// int64_t consumerId; -// int64_t reqOffset; -// int64_t processedVer; -// int32_t epoch; -// // rpc info -// int64_t reqId; -// SRpcHandleInfo rpcInfo; -// tmr_h timerId; -// int8_t tmrStopped; -// // exec -// int8_t inputStatus; -// int8_t execStatus; -// SStreamQueue inputQ; -// SRWLatch lock; -//} STqPushHandle; +#define EXTRACT_DATA_FROM_WAL_ID (-1) +#define STREAM_TASK_STATUS_CHECK_ID (-2) // tqExec - typedef struct { - char* qmsg; // SubPlanToString + char* qmsg; // SubPlanToString } STqExecCol; typedef struct { int64_t suid; + char* qmsg; // SubPlanToString + SNode* node; } STqExecTb; typedef struct { @@ -79,21 +64,21 @@ typedef struct { } STqExecDb; typedef struct { - int8_t subType; - STqReader* pTqReader; - qTaskInfo_t task; + int8_t subType; + STqReader* pTqReader; + qTaskInfo_t task; union { STqExecCol execCol; STqExecTb execTb; STqExecDb execDb; }; - int32_t numOfCols; // number of out pout column, temporarily used + int32_t numOfCols; // number of out pout column, temporarily used } STqExecHandle; -typedef enum tq_handle_status{ +typedef enum tq_handle_status { TMQ_HANDLE_STATUS_IDLE = 0, TMQ_HANDLE_STATUS_EXEC = 1, -}tq_handle_status; +} tq_handle_status; typedef struct { char subKey[TSDB_SUBSCRIBE_KEY_LEN]; @@ -106,16 +91,9 @@ typedef struct { // STqPushHandle pushHandle; // push STqExecHandle execHandle; // exec SRpcMsg* msg; - int32_t noDataPollCnt; tq_handle_status status; } STqHandle; -//typedef struct { -// SMqDataRsp* pDataRsp; -// char subKey[TSDB_SUBSCRIBE_KEY_LEN]; -// SRpcHandleInfo info; -//} STqPushEntry; - struct STQ { SVnode* pVnode; char* path; @@ -144,6 +122,7 @@ static STqMgmt tqMgmt = {0}; int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle); int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle); +void tqDestroyTqHandle(void* data); // tqRead int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset); @@ -153,9 +132,10 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea // tqExec int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows); int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); -int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type, - int32_t vgId); -int32_t tqPushDataRsp(STqHandle* pHandle, int32_t vgId); +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, + int32_t type, int32_t vgId); +//int32_t tqPushDataRsp(STqHandle* pHandle, int32_t vgId); +int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId); // tqMeta int32_t tqMetaOpen(STQ* pTq); @@ -166,6 +146,8 @@ int32_t tqMetaRestoreHandle(STQ* pTq); int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen); int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key); int32_t tqMetaRestoreCheckInfo(STQ* pTq); +int32_t tqMetaGetHandle(STQ* pTq, const char* key); +int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle); STqOffsetStore* tqOffsetOpen(STQ* pTq); void tqOffsetClose(STqOffsetStore*); @@ -186,14 +168,14 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); // tqStream int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); int32_t tqStreamTasksScanWal(STQ* pTq); +int32_t tqStreamTasksStatusCheck(STQ* pTq); // tq util -char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); -int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver); +int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, SStreamRefDataBlock** pRefBlock); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever); -int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq); +int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset); #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index b2bc9abf33aec05468d779bfbcc2eef65a6b38e3..fa42248c695a522b21e71fac00bebe7f7a66bae0 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -16,6 +16,9 @@ #ifndef _TD_VNODE_TSDB_H_ #define _TD_VNODE_TSDB_H_ +// #include "../tsdb/tsdbFile2.h" +// #include "../tsdb/tsdbMerge.h" +// #include "../tsdb/tsdbSttFileRW.h" #include "tsimplehash.h" #include "vnodeInt.h" @@ -64,7 +67,6 @@ typedef struct STsdbReadSnap STsdbReadSnap; typedef struct SBlockInfo SBlockInfo; typedef struct SSmaInfo SSmaInfo; typedef struct SBlockCol SBlockCol; -typedef struct SVersionRange SVersionRange; typedef struct SLDataIter SLDataIter; typedef struct SDiskCol SDiskCol; typedef struct SDiskData SDiskData; @@ -76,9 +78,8 @@ typedef struct STsdbFilterInfo STsdbFilterInfo; #define TSDBROW_ROW_FMT ((int8_t)0x0) #define TSDBROW_COL_FMT ((int8_t)0x1) -#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) -#define TSDB_MAX_SUBBLOCKS 8 -#define TSDB_FHDR_SIZE 512 +#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) +#define TSDB_FHDR_SIZE 512 #define VERSION_MIN 0 #define VERSION_MAX INT64_MAX @@ -166,6 +167,7 @@ void tBlockDataDestroy(SBlockData *pBlockData); int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid); void tBlockDataReset(SBlockData *pBlockData); int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); +int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema); int32_t tBlockDataTryUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, int64_t uid); int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); void tBlockDataClear(SBlockData *pBlockData); @@ -199,7 +201,7 @@ int32_t tMapDataToArray(SMapData *pMapData, int32_t itemSize, int32_t (*tGetItem // other int32_t tsdbKeyFid(TSKEY key, int32_t minutes, int8_t precision); void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minKey, TSKEY *maxKey); -int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now); +int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t nowSec); int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline); int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); @@ -297,17 +299,17 @@ int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter); // SDelFReader int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFReaderClose(SDelFReader **ppReader); +int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer); int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData); int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== int32_t tsdbTakeReadSnap(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap); void tsdbUntakeReadSnap(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive); -// tsdbMerge.c ============================================================================================== -int32_t tsdbMerge(STsdb *pTsdb); -#define TSDB_CACHE_NO(c) ((c).cacheLast == 0) -#define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0) -#define TSDB_CACHE_LAST(c) (((c).cacheLast & 2) > 0) +int32_t tsdbTakeReadSnap2(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap); +void tsdbUntakeReadSnap2(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive); +// tsdbMerge.c ============================================================================================== +int32_t tsdbMerge(void *arg); // tsdbDiskData ============================================================================================== int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder); @@ -346,28 +348,40 @@ struct STsdbFS { }; typedef struct { - rocksdb_t *db; - rocksdb_options_t *options; - rocksdb_flushoptions_t *flushoptions; - rocksdb_writeoptions_t *writeoptions; - rocksdb_readoptions_t *readoptions; - rocksdb_writebatch_t *writebatch; - TdThreadMutex rMutex; + rocksdb_t *db; + rocksdb_comparator_t *my_comparator; + rocksdb_cache_t *blockcache; + rocksdb_block_based_table_options_t *tableoptions; + rocksdb_options_t *options; + rocksdb_flushoptions_t *flushoptions; + rocksdb_writeoptions_t *writeoptions; + rocksdb_readoptions_t *readoptions; + rocksdb_writebatch_t *writebatch; + rocksdb_writebatch_t *rwritebatch; + TdThreadMutex rMutex; + STSchema *pTSchema; } SRocksCache; +typedef struct { + STsdb *pTsdb; + int flush_count; +} SCacheFlushState; + struct STsdb { - char *path; - SVnode *pVnode; - STsdbKeepCfg keepCfg; - TdThreadRwlock rwLock; - SMemTable *mem; - SMemTable *imem; - STsdbFS fs; - SLRUCache *lruCache; - TdThreadMutex lruMutex; - SLRUCache *biCache; - TdThreadMutex biMutex; - SRocksCache rCache; + char *path; + SVnode *pVnode; + STsdbKeepCfg keepCfg; + TdThreadRwlock rwLock; + SMemTable *mem; + SMemTable *imem; + STsdbFS fs; // old + SLRUCache *lruCache; + SCacheFlushState flushState; + TdThreadMutex lruMutex; + SLRUCache *biCache; + TdThreadMutex biMutex; + struct STFileSystem *pFS; // new + SRocksCache rCache; }; struct TSDBKEY { @@ -375,11 +389,6 @@ struct TSDBKEY { TSKEY ts; }; -struct SVersionRange { - uint64_t minVer; - uint64_t maxVer; -}; - typedef struct SMemSkipListNode SMemSkipListNode; struct SMemSkipListNode { int8_t level; @@ -408,6 +417,7 @@ struct STbData { SDelData *pTail; SMemSkipList sl; STbData *next; + SRBTreeNode rbtn[1]; }; struct SMemTable { @@ -421,11 +431,10 @@ struct SMemTable { TSKEY maxKey; int64_t nRow; int64_t nDel; - struct { - int32_t nTbData; - int32_t nBucket; - STbData **aBucket; - }; + int32_t nTbData; + int32_t nBucket; + STbData **aBucket; + SRBTree tbDataTree[1]; }; struct TSDBROW { @@ -498,7 +507,7 @@ struct SDataBlk { int32_t nRow; int8_t hasDup; int8_t nSubBlock; - SBlockInfo aSubBlock[TSDB_MAX_SUBBLOCKS]; + SBlockInfo aSubBlock[1]; SSmaInfo smaInfo; }; @@ -650,12 +659,19 @@ struct SDelFWriter { uint8_t *aBuf[1]; }; +#include "tarray2.h" +//#include "tsdbFS2.h" +// struct STFileSet; +typedef struct STFileSet STFileSet; +typedef TARRAY2(STFileSet *) TFileSetArray; + struct STsdbReadSnap { - SMemTable *pMem; - SQueryNode *pNode; - SMemTable *pIMem; - SQueryNode *pINode; - STsdbFS fs; + SMemTable *pMem; + SQueryNode *pNode; + SMemTable *pIMem; + SQueryNode *pINode; + TFileSetArray *pfSetArray; + STsdbFS fs; }; struct SDataFWriter { @@ -694,6 +710,7 @@ typedef struct { typedef struct SSttBlockLoadInfo { SBlockData blockData[2]; + void *pSttStatisBlkArray; SArray *aSttBlk; int32_t blockIndex[2]; // to denote the loaded block in the corresponding position. int32_t currentLoadBlockIndex; @@ -702,10 +719,9 @@ typedef struct SSttBlockLoadInfo { STSchema *pSchema; int16_t *colIds; int32_t numOfCols; - bool checkRemainingRow; + bool checkRemainingRow; // todo: no assign value? bool isLast; bool sttBlockLoaded; - int32_t numOfStt; // keep the last access position, this position may be used to reduce the binary times for // starting last block data for a new table @@ -764,77 +780,80 @@ struct SDiskDataBuilder { }; typedef struct SLDataIter { - SRBTreeNode node; - SSttBlk *pSttBlk; - SDataFReader *pReader; - int32_t iStt; - int8_t backward; - int32_t iSttBlk; - int32_t iRow; - SRowInfo rInfo; - uint64_t uid; - STimeWindow timeWindow; - SVersionRange verRange; - SSttBlockLoadInfo *pBlockLoadInfo; - bool ignoreEarlierTs; + SRBTreeNode node; + SSttBlk *pSttBlk; + int32_t iStt; // for debug purpose + int8_t backward; + int32_t iSttBlk; + int32_t iRow; + SRowInfo rInfo; + uint64_t uid; + STimeWindow timeWindow; + SVersionRange verRange; + SSttBlockLoadInfo *pBlockLoadInfo; + bool ignoreEarlierTs; + struct SSttFileReader *pReader; } SLDataIter; #define tMergeTreeGetRow(_t) (&((_t)->pIter->rInfo.row)) int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, - bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter); -void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); -bool tMergeTreeNext(SMergeTree *pMTree); -bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); -void tMergeTreeClose(SMergeTree *pMTree); + bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter *pLDataIter); + +struct SSttFileReader; +typedef int32_t (*_load_tomb_fn)(STsdbReader *pReader, struct SSttFileReader *pSttFileReader, + SSttBlockLoadInfo *pLoadInfo); + +typedef struct { + int8_t backward; + STsdb *pTsdb; + uint64_t suid; + uint64_t uid; + STimeWindow timewindow; + SVersionRange verRange; + bool strictTimeRange; + SArray *pSttFileBlockIterArray; + void *pCurrentFileset; + STSchema *pSchema; + int16_t *pCols; + int32_t numOfCols; + _load_tomb_fn loadTombFn; + void *pReader; + void *idstr; +} SMergeTreeConf; +int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf); + +void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); +bool tMergeTreeNext(SMergeTree *pMTree); +bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); +void tMergeTreeClose(SMergeTree *pMTree); SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, int32_t numOfStt); +SSttBlockLoadInfo *tCreateOneLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols); void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el); void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); +void *destroySttBlockReader(SArray *pLDataIterArray, int64_t *blocks, double *el); // tsdbCache ============================================================================================== -typedef struct SCacheRowsReader { - STsdb *pTsdb; - SVersionRange verRange; - TdThreadMutex readerMutex; - SVnode *pVnode; - STSchema *pSchema; - STSchema *pCurrSchema; - uint64_t uid; - uint64_t suid; - char **transferBuf; // todo remove it soon - int32_t numOfCols; - SArray *pCidList; - int32_t *pSlotIds; - int32_t type; - int32_t tableIndex; // currently returned result tables - STableKeyInfo *pTableList; // table id list - int32_t numOfTables; - SSttBlockLoadInfo *pLoadInfo; - SLDataIter *pDataIter; - STsdbReadSnap *pReadSnap; - SDataFReader *pDataFReader; - SDataFReader *pDataFReaderLast; - const char *idstr; - int64_t lastTs; -} SCacheRowsReader; +typedef enum { + READ_MODE_COUNT_ONLY = 0x1, + READ_MODE_ALL, +} EReadMode; typedef struct { TSKEY ts; + int8_t dirty; SColVal colVal; } SLastCol; int32_t tsdbOpenCache(STsdb *pTsdb); void tsdbCloseCache(STsdb *pTsdb); int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *row); -int32_t tsdbCacheGet(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int32_t ltype); int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb); int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup); -int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h); -int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h); int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h); int32_t tsdbCacheGetBlockIdx(SLRUCache *pCache, SDataFReader *pFileReader, LRUHandle **handle); @@ -844,11 +863,6 @@ int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); -void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); -size_t tsdbCacheGetCapacity(SVnode *pVnode); - -// int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); - // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index a67f246e73d9656f9c184e99b796edf8ee8dc954..85ef384ea98a79c11c6e7774f9bb797e5de21e8e 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -49,7 +49,8 @@ int32_t vnodeEncodeConfig(const void* pObj, SJson* pJson); int32_t vnodeDecodeConfig(const SJson* pJson, void* pObj); // vnodeModule.c -int32_t vnodeScheduleTask(int32_t (*execute)(void*), void* arg); +int vnodeScheduleTask(int (*execute)(void*), void* arg); +int vnodeScheduleTaskEx(int tpid, int (*execute)(void*), void* arg); // vnodeBufPool.c typedef struct SVBufPoolNode SVBufPoolNode; @@ -86,6 +87,9 @@ void vnodeBufPoolReset(SVBufPool* pPool); void vnodeBufPoolAddToFreeList(SVBufPool* pPool); int32_t vnodeBufPoolRecycle(SVBufPool* pPool); +// vnodeOpen.c +int32_t vnodeGetPrimaryDir(const char* relPath, int32_t diskPrimary, STfs* pTfs, char* buf, size_t bufLen); + // vnodeQuery.c int32_t vnodeQueryOpen(SVnode* pVnode); void vnodeQueryPreClose(SVnode* pVnode); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index d7f0ef041a8db9d9e66f3cd8cda6a598b5d5a466..cd7704940b3a017dae7d12e15647fc61ca8fd542 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -93,6 +93,7 @@ typedef struct SQueryNode SQueryNode; #define VNODE_BUFPOOL_SEGMENTS 3 #define VND_INFO_FNAME "vnode.json" +#define VND_INFO_FNAME_TMP "vnode_tmp.json" // vnd.h typedef int32_t (*_query_reseek_func_t)(void* pQHandle); @@ -103,6 +104,19 @@ struct SQueryNode { _query_reseek_func_t reseek; }; +#if 1 // refact APIs below (TODO) +typedef SVCreateTbReq STbCfg; +typedef SVCreateTSmaReq SSmaCfg; + +SMTbCursor* metaOpenTbCursor(void* pVnode); +void metaCloseTbCursor(SMTbCursor* pTbCur); +void metaPauseTbCursor(SMTbCursor* pTbCur); +void metaResumeTbCursor(SMTbCursor* pTbCur, int8_t first); +int32_t metaTbCursorNext(SMTbCursor* pTbCur, ETableType jumpTableType); +int32_t metaTbCursorPrev(SMTbCursor* pTbCur, ETableType jumpTableType); + +#endif + void* vnodeBufPoolMalloc(SVBufPool* pPool, int size); void* vnodeBufPoolMallocAligned(SVBufPool* pPool, int size); void vnodeBufPoolFree(SVBufPool* pPool, void* p); @@ -123,6 +137,7 @@ typedef struct STbUidStore STbUidStore; #define META_BEGIN_HEAP_NIL 2 int metaOpen(SVnode* pVnode, SMeta** ppMeta, int8_t rollback); +int metaUpgrade(SVnode* pVnode, SMeta** ppMeta); int metaClose(SMeta** pMeta); int metaBegin(SMeta* pMeta, int8_t fromSys); TXN* metaGetTxn(SMeta* pMeta); @@ -135,14 +150,19 @@ int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* p int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp); int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t* tbUid); -int metaTtlDropTable(SMeta* pMeta, int64_t ttl, SArray* tbUids); +int32_t metaTrimTables(SMeta* pMeta); +int metaTtlDropTable(SMeta* pMeta, int64_t timePointMs, SArray* tbUids); int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); +int metaUpdateChangeTime(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock); int32_t metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sver, STSchema** ppTSchema); int metaGetTableEntryByName(SMetaReader* pReader, const char* name); int metaAlterCache(SMeta* pMeta, int32_t nPage); +int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid); +int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid); + int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); @@ -159,7 +179,9 @@ SArray* metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid); SArray* metaGetSmaTbUids(SMeta* pMeta); void* metaGetIdx(SMeta* pMeta); void* metaGetIvtIdx(SMeta* pMeta); -int metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList); + +int64_t metaGetTbNum(SMeta* pMeta); +void metaReaderDoInit(SMetaReader* pReader, SMeta* pMeta, int32_t flags); int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); @@ -176,12 +198,12 @@ int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pR int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback); int tsdbClose(STsdb** pTsdb); int32_t tsdbBegin(STsdb* pTsdb); -int32_t tsdbPrepareCommit(STsdb* pTsdb); -int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); +// int32_t tsdbPrepareCommit(STsdb* pTsdb); +// int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); int32_t tsdbCacheCommit(STsdb* pTsdb); int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo); -int32_t tsdbFinishCommit(STsdb* pTsdb); -int32_t tsdbRollbackCommit(STsdb* pTsdb); +// int32_t tsdbFinishCommit(STsdb* pTsdb); +// int32_t tsdbRollbackCommit(STsdb* pTsdb); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); @@ -197,7 +219,8 @@ void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); int tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg); int tqUnregisterPushHandle(STQ* pTq, void* pHandle); -int tqStartStreamTasks(STQ* pTq); // restore all stream tasks after vnode launching completed. +int tqStartStreamTasks(STQ* pTq); // restore all stream tasks after vnode launching completed. +int tqCheckStreamStatus(STQ* pTq); int tqCommit(STQ*); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); @@ -208,9 +231,11 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t version, char* msg, int32_t m int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessSeekReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen); +int32_t tqProcessSeekReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg); // tq-stream int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); @@ -218,18 +243,16 @@ int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLe int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg); -int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessSubmitReqForSubscribe(STQ* pTq); -int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver); +int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, SRpcMsg* pMsg); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); -int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg); -int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg); -int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskTransferStateReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskScanHistoryFinishReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskScanHistoryFinishRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqCheckLogInWal(STQ* pTq, int64_t version); // sma @@ -298,7 +321,6 @@ int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData); // SRSmaSnapWriter ======================================== int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWriter** ppWriter); int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); -int32_t rsmaSnapWriterPrepareClose(SRSmaSnapWriter* pWriter); int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback); typedef struct { @@ -365,6 +387,7 @@ struct SVnode { SVState state; SVStatis statis; STfs* pTfs; + int32_t diskPrimary; SMsgCb msgCb; // Buffer Pool @@ -405,6 +428,10 @@ struct SVnode { #define VND_IS_RSMA(v) ((v)->config.isRsma == 1) #define VND_IS_TSMA(v) ((v)->config.isTsma == 1) +#define TSDB_CACHE_NO(c) ((c).cacheLast == 0) +#define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0) +#define TSDB_CACHE_LAST(c) (((c).cacheLast & 2) > 0) + struct STbUidStore { tb_uid_t suid; SArray* tbUids; @@ -471,6 +498,8 @@ struct SCompactInfo { STimeWindow tw; }; +void initStorageAPI(SStorageAPI* pAPI); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 795f281ab2107acdce93ec2373ab99e1391b61ba..6918634b5daa4268ca401752608b42d5b82ea595 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -59,6 +59,17 @@ struct SMetaCache { SHashObj* pTableEntry; SLRUCache* pUidResCache; } sTagFilterResCache; + + struct STbGroupResCache { + TdThreadMutex lock; + uint32_t accTimes; + SHashObj* pTableEntry; + SLRUCache* pResCache; + } STbGroupResCache; + + struct STbFilterCache { + SHashObj* pStb; + } STbFilterCache; }; static void entryCacheClose(SMeta* pMeta) { @@ -144,6 +155,29 @@ int32_t metaCacheOpen(SMeta* pMeta) { taosHashSetFreeFp(pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp); taosThreadMutexInit(&pCache->sTagFilterResCache.lock, NULL); + pCache->STbGroupResCache.pResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); + if (pCache->STbGroupResCache.pResCache == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err2; + } + + pCache->STbGroupResCache.accTimes = 0; + pCache->STbGroupResCache.pTableEntry = + taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); + if (pCache->STbGroupResCache.pTableEntry == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err2; + } + + taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp); + taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL); + + pCache->STbFilterCache.pStb = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (pCache->STbFilterCache.pStb == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err2; + } + pMeta->pCache = pCache; return code; @@ -165,6 +199,12 @@ void metaCacheClose(SMeta* pMeta) { taosThreadMutexDestroy(&pMeta->pCache->sTagFilterResCache.lock); taosHashCleanup(pMeta->pCache->sTagFilterResCache.pTableEntry); + taosLRUCacheCleanup(pMeta->pCache->STbGroupResCache.pResCache); + taosThreadMutexDestroy(&pMeta->pCache->STbGroupResCache.lock); + taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry); + + taosHashCleanup(pMeta->pCache->STbFilterCache.pStb); + taosMemoryFree(pMeta->pCache); pMeta->pCache = NULL; } @@ -456,21 +496,22 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv } static FORCE_INLINE void setMD5DigestInKey(uint64_t* pBuf, const char* key, int32_t keyLen) { -// ASSERT(keyLen == sizeof(int64_t) * 2); + // ASSERT(keyLen == sizeof(int64_t) * 2); memcpy(&pBuf[2], key, keyLen); } // the format of key: // hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes) static void initCacheKey(uint64_t* buf, const SHashObj* pHashMap, uint64_t suid, const char* key, int32_t keyLen) { - buf[0] = (uint64_t) pHashMap; + buf[0] = (uint64_t)pHashMap; buf[1] = suid; setMD5DigestInKey(buf, key, keyLen); ASSERT(keyLen == sizeof(uint64_t) * 2); } -int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, +int32_t metaGetCachedTableUidList(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, bool* acquireRes) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; int32_t vgId = TD_VID(pMeta->pVnode); // generate the composed key for LRU cache @@ -510,7 +551,8 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK uint32_t acc = pMeta->pCache->sTagFilterResCache.accTimes; if ((*pEntry)->hitTimes % 5000 == 0 && (*pEntry)->hitTimes > 0) { - metaInfo("vgId:%d cache hit:%d, total acc:%d, rate:%.2f", vgId, (*pEntry)->hitTimes, acc, ((double)(*pEntry)->hitTimes) / acc); + metaInfo("vgId:%d cache hit:%d, total acc:%d, rate:%.2f", vgId, (*pEntry)->hitTimes, acc, + ((double)(*pEntry)->hitTimes) / acc); } taosLRUCacheRelease(pCache, pHandle, false); @@ -520,7 +562,8 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK return TSDB_CODE_SUCCESS; } -static void freePayload(const void* key, size_t keyLen, void* value) { +static void freeUidCachePayload(const void* key, size_t keyLen, void* value, void* ud) { + (void)ud; if (value == NULL) { return; } @@ -573,9 +616,10 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL } // check both the payload size and selectivity ratio -int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, +int32_t metaUidFilterCachePut(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen, double selectivityRatio) { int32_t code = 0; + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; int32_t vgId = TD_VID(pMeta->pVnode); if (selectivityRatio > tsSelectivityRatio) { @@ -608,7 +652,7 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int if (code != TSDB_CODE_SUCCESS) { goto _end; } - } else { // check if it exists or not + } else { // check if it exists or not size_t size = listNEles(&(*pEntry)->list); if (size == 0) { tdListAppend(&(*pEntry)->list, pKey); @@ -626,8 +670,8 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int } // add to cache. - taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freePayload, NULL, - TAOS_LRU_PRIORITY_LOW); + taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeUidCachePayload, NULL, + TAOS_LRU_PRIORITY_LOW, NULL); _end: taosThreadMutexUnlock(pLock); metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", vgId, suid, @@ -643,7 +687,7 @@ int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) { SHashObj* pEntryHashMap = pMeta->pCache->sTagFilterResCache.pTableEntry; uint64_t dummy[2] = {0}; - initCacheKey(p, pEntryHashMap, suid, (char*) &dummy[0], 16); + initCacheKey(p, pEntryHashMap, suid, (char*)&dummy[0], 16); TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock; taosThreadMutexLock(pLock); @@ -668,6 +712,211 @@ int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) { tdListEmpty(&(*pEntry)->list); taosThreadMutexUnlock(pLock); - metaDebug("vgId:%d suid:%"PRId64" cached related tag filter uid list cleared", vgId, suid); + metaDebug("vgId:%d suid:%" PRId64 " cached related tag filter uid list cleared", vgId, suid); + return TSDB_CODE_SUCCESS; +} + +int32_t metaGetCachedTbGroup(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + int32_t vgId = TD_VID(pMeta->pVnode); + + // generate the composed key for LRU cache + SLRUCache* pCache = pMeta->pCache->STbGroupResCache.pResCache; + SHashObj* pTableMap = pMeta->pCache->STbGroupResCache.pTableEntry; + TdThreadMutex* pLock = &pMeta->pCache->STbGroupResCache.lock; + + *pList = NULL; + uint64_t key[4]; + initCacheKey(key, pTableMap, suid, (const char*)pKey, keyLen); + + taosThreadMutexLock(pLock); + pMeta->pCache->STbGroupResCache.accTimes += 1; + + LRUHandle* pHandle = taosLRUCacheLookup(pCache, key, TAG_FILTER_RES_KEY_LEN); + if (pHandle == NULL) { + taosThreadMutexUnlock(pLock); + return TSDB_CODE_SUCCESS; + } + + STagFilterResEntry** pEntry = taosHashGet(pTableMap, &suid, sizeof(uint64_t)); + if (NULL == pEntry) { + metaDebug("suid %" PRIu64 " not in tb group cache", suid); + return TSDB_CODE_FAILED; + } + + *pList = taosArrayDup(taosLRUCacheValue(pCache, pHandle), NULL); + + (*pEntry)->hitTimes += 1; + + uint32_t acc = pMeta->pCache->STbGroupResCache.accTimes; + if ((*pEntry)->hitTimes % 5000 == 0 && (*pEntry)->hitTimes > 0) { + metaInfo("vgId:%d tb group cache hit:%d, total acc:%d, rate:%.2f", vgId, (*pEntry)->hitTimes, acc, + ((double)(*pEntry)->hitTimes) / acc); + } + + taosLRUCacheRelease(pCache, pHandle, false); + + // unlock meta + taosThreadMutexUnlock(pLock); + return TSDB_CODE_SUCCESS; +} + +static void freeTbGroupCachePayload(const void* key, size_t keyLen, void* value, void* ud) { + (void)ud; + if (value == NULL) { + return; + } + + const uint64_t* p = key; + if (keyLen != sizeof(int64_t) * 4) { + metaError("tb group key length is invalid, length:%d, expect:%d", (int32_t)keyLen, (int32_t)sizeof(uint64_t) * 2); + return; + } + + SHashObj* pHashObj = (SHashObj*)p[0]; + + STagFilterResEntry** pEntry = taosHashGet(pHashObj, &p[1], sizeof(uint64_t)); + + if (pEntry != NULL && (*pEntry) != NULL) { + int64_t st = taosGetTimestampUs(); + + SListIter iter = {0}; + tdListInitIter((SList*)&((*pEntry)->list), &iter, TD_LIST_FORWARD); + + SListNode* pNode = NULL; + while ((pNode = tdListNext(&iter)) != NULL) { + uint64_t* digest = (uint64_t*)pNode->data; + if (digest[0] == p[2] && digest[1] == p[3]) { + void* tmp = tdListPopNode(&((*pEntry)->list), pNode); + taosMemoryFree(tmp); + + double el = (taosGetTimestampUs() - st) / 1000.0; + metaDebug("clear one item in tb group cache, remain cached item:%d, elapsed time:%.2fms", + listNEles(&((*pEntry)->list)), el); + break; + } + } + } + + taosArrayDestroy((SArray*)value); +} + +int32_t metaPutTbGroupToCache(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen) { + int32_t code = 0; + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + int32_t vgId = TD_VID(pMeta->pVnode); + + if (payloadLen > tsTagFilterResCacheSize) { + metaDebug("vgId:%d, suid:%" PRIu64 + " ignore to add to tb group cache, due to payload length %d greater than threshold %d", + vgId, suid, payloadLen, tsTagFilterResCacheSize); + taosArrayDestroy((SArray*)pPayload); + return TSDB_CODE_SUCCESS; + } + + SLRUCache* pCache = pMeta->pCache->STbGroupResCache.pResCache; + SHashObj* pTableEntry = pMeta->pCache->STbGroupResCache.pTableEntry; + TdThreadMutex* pLock = &pMeta->pCache->STbGroupResCache.lock; + + uint64_t key[4] = {0}; + initCacheKey(key, pTableEntry, suid, pKey, keyLen); + + taosThreadMutexLock(pLock); + STagFilterResEntry** pEntry = taosHashGet(pTableEntry, &suid, sizeof(uint64_t)); + if (pEntry == NULL) { + code = addNewEntry(pTableEntry, pKey, keyLen, suid); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + } else { // check if it exists or not + size_t size = listNEles(&(*pEntry)->list); + if (size == 0) { + tdListAppend(&(*pEntry)->list, pKey); + } else { + SListNode* pNode = listHead(&(*pEntry)->list); + uint64_t* p = (uint64_t*)pNode->data; + if (p[1] == ((uint64_t*)pKey)[1] && p[0] == ((uint64_t*)pKey)[0]) { + // we have already found the existed items, no need to added to cache anymore. + taosThreadMutexUnlock(pLock); + return TSDB_CODE_SUCCESS; + } else { // not equal, append it + tdListAppend(&(*pEntry)->list, pKey); + } + } + } + + // add to cache. + taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeTbGroupCachePayload, NULL, + TAOS_LRU_PRIORITY_LOW, NULL); +_end: + taosThreadMutexUnlock(pLock); + metaDebug("vgId:%d, suid:%" PRIu64 " tb group added into cache, total:%d, tables:%d", vgId, suid, + (int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry)); + + return code; +} + +// remove the lru cache that are expired due to the tags value update, or creating, or dropping, of child tables +int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) { + uint64_t p[4] = {0}; + int32_t vgId = TD_VID(pMeta->pVnode); + SHashObj* pEntryHashMap = pMeta->pCache->STbGroupResCache.pTableEntry; + + uint64_t dummy[2] = {0}; + initCacheKey(p, pEntryHashMap, suid, (char*)&dummy[0], 16); + + TdThreadMutex* pLock = &pMeta->pCache->STbGroupResCache.lock; + taosThreadMutexLock(pLock); + + STagFilterResEntry** pEntry = taosHashGet(pEntryHashMap, &suid, sizeof(uint64_t)); + if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) { + taosThreadMutexUnlock(pLock); + return TSDB_CODE_SUCCESS; + } + + (*pEntry)->hitTimes = 0; + + SListIter iter = {0}; + tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD); + + SListNode* pNode = NULL; + while ((pNode = tdListNext(&iter)) != NULL) { + setMD5DigestInKey(p, pNode->data, 2 * sizeof(uint64_t)); + taosLRUCacheErase(pMeta->pCache->STbGroupResCache.pResCache, p, TAG_FILTER_RES_KEY_LEN); + } + + tdListEmpty(&(*pEntry)->list); + taosThreadMutexUnlock(pLock); + + metaDebug("vgId:%d suid:%" PRId64 " cached related tb group cleared", vgId, suid); return TSDB_CODE_SUCCESS; } + +bool metaTbInFilterCache(void* pVnode, tb_uid_t suid, int8_t type) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + + if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid))) { + return true; + } + + return false; +} + +int32_t metaPutTbToFilterCache(void* pVnode, tb_uid_t suid, int8_t type) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + + if (type == 0) { + return taosHashPut(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid), NULL, 0); + } + + return 0; +} + +int32_t metaSizeOfTbFilterCache(void* pVnode, int8_t type) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + if (type == 0) { + return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb); + } + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index f597c100d0588dae9123af444dc442c0b7de2c2b..d26256795361301d2c80f3488a0aee0b1590a20d 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -40,6 +40,8 @@ int metaBegin(SMeta *pMeta, int8_t heap) { return -1; } + tdbCommit(pMeta->pEnv, pMeta->txn); + return 0; } @@ -50,6 +52,7 @@ int metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv int metaPrepareAsyncCommit(SMeta *pMeta) { // return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn); int code = 0; + code = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn); code = tdbCommit(pMeta->pEnv, pMeta->txn); return code; diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index e50931ac0618252eea58afc0d1a734d252a05d1e..01877a523a942c9a3e5bcf889046358334ef9621 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -31,7 +31,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; } } else if (pME->type == TSDB_CHILD_TABLE) { - if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1; + if (tEncodeI64(pCoder, pME->ctbEntry.btime) < 0) return -1; if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1; if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1; if (pME->ctbEntry.commentLen > 0) { @@ -40,7 +40,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1; if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1; } else if (pME->type == TSDB_NORMAL_TABLE) { - if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1; + if (tEncodeI64(pCoder, pME->ntbEntry.btime) < 0) return -1; if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1; if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1; if (pME->ntbEntry.commentLen > 0) { @@ -76,7 +76,7 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; } } else if (pME->type == TSDB_CHILD_TABLE) { - if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1; + if (tDecodeI64(pCoder, &pME->ctbEntry.btime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1; if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1; if (pME->ctbEntry.commentLen > 0) { @@ -85,7 +85,7 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1; if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO) } else if (pME->type == TSDB_NORMAL_TABLE) { - if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1; + if (tDecodeI64(pCoder, &pME->ntbEntry.btime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1; if (pME->ntbEntry.commentLen > 0) { diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 1d0b11e26a58d047216196bf3fed7b8f13807922..517d9692c75f1831038698f6193eb02f994002a3 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -14,49 +14,48 @@ */ #include "meta.h" +#include "vnd.h" static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); -static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int taskIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); -static int ctimeIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int btimeIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int ncolIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int32_t metaInitLock(SMeta *pMeta) { return taosThreadRwlockInit(&pMeta->lock, NULL); } static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&pMeta->lock); } +static void metaCleanup(SMeta **ppMeta); + int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { SMeta *pMeta = NULL; int ret; - int slen; + int offset; + char path[TSDB_FILENAME_LEN] = {0}; *ppMeta = NULL; // create handle - if (pVnode->pTfs) { - slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(VNODE_META_DIR) + 3; - } else { - slen = strlen(pVnode->path) + strlen(VNODE_META_DIR) + 2; - } - if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + slen)) == NULL) { + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN); + offset = strlen(path); + snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VNODE_META_DIR); + + if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + strlen(path) + 1)) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } metaInitLock(pMeta); + pMeta->path = (char *)&pMeta[1]; - if (pVnode->pTfs) { - sprintf(pMeta->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, - VNODE_META_DIR); - } else { - sprintf(pMeta->path, "%s%s%s", pVnode->path, TD_DIRSEP, VNODE_META_DIR); - } - taosRealPath(pMeta->path, NULL, slen); + strcpy(pMeta->path, path); + taosRealPath(pMeta->path, NULL, strlen(path) + 1); + pMeta->pVnode = pVnode; // create path if not created yet @@ -128,8 +127,10 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { goto _err; } - // open pTtlIdx - ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx, 0); + // open pTtlMgr ("ttlv1.idx") + char logPrefix[128] = {0}; + sprintf(logPrefix, "vgId:%d", TD_VID(pVnode)); + ret = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix); if (ret < 0) { metaError("vgId:%d, failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; @@ -143,7 +144,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { } // idx table create time - ret = tdbTbOpen("ctime.idx", sizeof(SCtimeIdxKey), 0, ctimeIdxCmpr, pMeta->pEnv, &pMeta->pCtimeIdx, 0); + ret = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0); if (ret < 0) { metaError("vgId:%d, failed to open meta ctime index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; @@ -181,51 +182,43 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { return 0; _err: - if (pMeta->pIdx) metaCloseIdx(pMeta); - if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb); - if (pMeta->pNcolIdx) tdbTbClose(pMeta->pNcolIdx); - if (pMeta->pCtimeIdx) tdbTbClose(pMeta->pCtimeIdx); - if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); - if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx); - if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); - if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); - if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); - if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx); - if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); - if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); - if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); - if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb); - if (pMeta->pEnv) tdbClose(pMeta->pEnv); - metaDestroyLock(pMeta); - taosMemoryFree(pMeta); + metaCleanup(&pMeta); return -1; } -int metaClose(SMeta **ppMeta) { +int metaUpgrade(SVnode *pVnode, SMeta **ppMeta) { + int code = TSDB_CODE_SUCCESS; SMeta *pMeta = *ppMeta; - if (pMeta) { - if (pMeta->pEnv) metaAbort(pMeta); - if (pMeta->pCache) metaCacheClose(pMeta); - if (pMeta->pIdx) metaCloseIdx(pMeta); - if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb); - if (pMeta->pNcolIdx) tdbTbClose(pMeta->pNcolIdx); - if (pMeta->pCtimeIdx) tdbTbClose(pMeta->pCtimeIdx); - if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); - if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx); - if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); - if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); - if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); - if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx); - if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); - if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); - if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); - if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb); - if (pMeta->pEnv) tdbClose(pMeta->pEnv); - metaDestroyLock(pMeta); - taosMemoryFreeClear(*ppMeta); + if (ttlMgrNeedUpgrade(pMeta->pEnv)) { + code = metaBegin(pMeta, META_BEGIN_HEAP_OS); + if (code < 0) { + metaError("vgId:%d, failed to upgrade meta, meta begin failed since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + + code = ttlMgrUpgrade(pMeta->pTtlMgr, pMeta); + if (code < 0) { + metaError("vgId:%d, failed to upgrade meta ttl since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + + code = metaCommit(pMeta, pMeta->txn); + if (code < 0) { + metaError("vgId:%d, failed to upgrade meta ttl, meta commit failed since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } } + return TSDB_CODE_SUCCESS; + +_err: + metaCleanup(ppMeta); + return code; +} + +int metaClose(SMeta **ppMeta) { + metaCleanup(ppMeta); return 0; } @@ -271,6 +264,32 @@ int32_t metaULock(SMeta *pMeta) { return ret; } +static void metaCleanup(SMeta **ppMeta) { + SMeta *pMeta = *ppMeta; + if (pMeta) { + if (pMeta->pEnv) metaAbort(pMeta); + if (pMeta->pCache) metaCacheClose(pMeta); + if (pMeta->pIdx) metaCloseIdx(pMeta); + if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb); + if (pMeta->pNcolIdx) tdbTbClose(pMeta->pNcolIdx); + if (pMeta->pBtimeIdx) tdbTbClose(pMeta->pBtimeIdx); + if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); + if (pMeta->pTtlMgr) ttlMgrClose(pMeta->pTtlMgr); + if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); + if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); + if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); + if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx); + if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); + if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); + if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); + if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb); + if (pMeta->pEnv) tdbClose(pMeta->pEnv); + metaDestroyLock(pMeta); + + taosMemoryFreeClear(*ppMeta); + } +} + static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { STbDbKey *pTbDbKey1 = (STbDbKey *)pKey1; STbDbKey *pTbDbKey2 = (STbDbKey *)pKey2; @@ -399,37 +418,18 @@ static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL return 0; } -static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { - STtlIdxKey *pTtlIdxKey1 = (STtlIdxKey *)pKey1; - STtlIdxKey *pTtlIdxKey2 = (STtlIdxKey *)pKey2; - - if (pTtlIdxKey1->dtime > pTtlIdxKey2->dtime) { - return 1; - } else if (pTtlIdxKey1->dtime < pTtlIdxKey2->dtime) { - return -1; - } - - if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) { - return 1; - } else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) { - return -1; - } - - return 0; -} - -static int ctimeIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { - SCtimeIdxKey *pCtimeIdxKey1 = (SCtimeIdxKey *)pKey1; - SCtimeIdxKey *pCtimeIdxKey2 = (SCtimeIdxKey *)pKey2; - if (pCtimeIdxKey1->ctime > pCtimeIdxKey2->ctime) { +static int btimeIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + SBtimeIdxKey *pBtimeIdxKey1 = (SBtimeIdxKey *)pKey1; + SBtimeIdxKey *pBtimeIdxKey2 = (SBtimeIdxKey *)pKey2; + if (pBtimeIdxKey1->btime > pBtimeIdxKey2->btime) { return 1; - } else if (pCtimeIdxKey1->ctime < pCtimeIdxKey2->ctime) { + } else if (pBtimeIdxKey1->btime < pBtimeIdxKey2->btime) { return -1; } - if (pCtimeIdxKey1->uid > pCtimeIdxKey2->uid) { + if (pBtimeIdxKey1->uid > pBtimeIdxKey2->uid) { return 1; - } else if (pCtimeIdxKey1->uid < pCtimeIdxKey2->uid) { + } else if (pBtimeIdxKey1->uid < pBtimeIdxKey2->uid) { return -1; } diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index d464f64de327969f4c302498fe2f6370676845d6..c26bb45c2b51b91ae4e4d9f53d7a895fc832ec9e 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -17,10 +17,16 @@ #include "osMemory.h" #include "tencode.h" -void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { +void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags, SStoreMeta* pAPI) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + metaReaderDoInit(pReader, pMeta, flags); + pReader->pAPI = pAPI; +} + +void metaReaderDoInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { memset(pReader, 0, sizeof(*pReader)); - pReader->flags = flags; pReader->pMeta = pMeta; + pReader->flags = flags; if (pReader->pMeta && !(flags & META_READER_NOLOCK)) { metaRLock(pMeta); } @@ -64,96 +70,20 @@ _err: return -1; } -// int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { -// -// SArray* readerList = taosArrayInit(taosArrayGetSize(uidList), sizeof(SMetaReader)); -// SArray* uidVersion = taosArrayInit(taosArrayGetSize(uidList), sizeof(STbDbKey)); -// SMeta *pMeta = meta; -// int64_t version; -// SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); -// -// int64_t stt1 = taosGetTimestampUs(); -// for(int i = 0; i < taosArrayGetSize(uidList); i++) { -// void* ppVal = NULL; -// int vlen = 0; -// uint64_t * uid = taosArrayGet(uidList, i); -// // query uid.idx -// if (tdbTbGet(pMeta->pUidIdx, uid, sizeof(*uid), &ppVal, &vlen) < 0) { -// continue; -// } -// version = *(int64_t *)ppVal; -// -// STbDbKey tbDbKey = {.version = version, .uid = *uid}; -// taosArrayPush(uidVersion, &tbDbKey); -// taosHashPut(uHash, uid, sizeof(int64_t), ppVal, sizeof(int64_t)); -// } -// int64_t stt2 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest1 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt2-stt1); -// -// TBC *pCur = NULL; -// tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); -// tdbTbcMoveToFirst(pCur); -// void *pKey = NULL; -// int kLen = 0; -// -// while(1){ -// SMetaReader pReader = {0}; -// int32_t ret = tdbTbcNext(pCur, &pKey, &kLen, &pReader.pBuf, &pReader.szBuf); -// if (ret < 0) break; -// STbDbKey *tmp = (STbDbKey*)pKey; -// int64_t *ver = (int64_t*)taosHashGet(uHash, &tmp->uid, sizeof(int64_t)); -// if(ver == NULL || *ver != tmp->version) continue; -// taosArrayPush(readerList, &pReader); -// } -// tdbTbcClose(pCur); -// -// taosArrayClear(readerList); -// int64_t stt3 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2); -// for(int i = 0; i < taosArrayGetSize(uidVersion); i++) { -// SMetaReader pReader = {0}; -// -// STbDbKey *tbDbKey = taosArrayGet(uidVersion, i); -// // query table.db -// if (tdbTbGet(pMeta->pTbDb, tbDbKey, sizeof(STbDbKey), &pReader.pBuf, &pReader.szBuf) < 0) { -// continue; -// } -// taosArrayPush(readerList, &pReader); -// } -// int64_t stt4 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt4-stt3); -// -// for(int i = 0; i < taosArrayGetSize(readerList); i++){ -// SMetaReader* pReader = taosArrayGet(readerList, i); -// metaReaderInit(pReader, meta, 0); -// // decode the entry -// tDecoderInit(&pReader->coder, pReader->pBuf, pReader->szBuf); -// -// if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) { -// } -// metaReaderClear(pReader); -// } -// int64_t stt5 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest4 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt5-stt4); -// return 0; -// } - -bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) { - // query uid.idx - metaRLock(pMeta); - - if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) { - metaULock(pMeta); +bool metaIsTableExist(void *pVnode, tb_uid_t uid) { + SVnode *pVnodeObj = pVnode; + metaRLock(pVnodeObj->pMeta); // query uid.idx + if (tdbTbGet(pVnodeObj->pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) { + metaULock(pVnodeObj->pMeta); return false; } - metaULock(pMeta); - + metaULock(pVnodeObj->pMeta); return true; } -int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { +int metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { SMeta *pMeta = pReader->pMeta; int64_t version1; @@ -167,7 +97,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { return metaGetTableEntryByVersion(pReader, version1, uid); } -int metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid) { +int metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid) { SMeta *pMeta = pReader->pMeta; SMetaInfo info; @@ -190,7 +120,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) { } uid = *(tb_uid_t *)pReader->pBuf; - return metaGetTableEntryByUid(pReader, uid); + return metaReaderGetTableEntryByUid(pReader, uid); } tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) { @@ -210,11 +140,11 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) { return uid; } -int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) { +int metaGetTableNameByUid(void *pVnode, uint64_t uid, char *tbName) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, (SMeta *)meta, 0); - code = metaGetTableEntryByUid(&mr, uid); + metaReaderDoInit(&mr, ((SVnode*)pVnode)->pMeta, 0); + code = metaReaderGetTableEntryByUid(&mr, uid); if (code < 0) { metaReaderClear(&mr); return -1; @@ -229,8 +159,8 @@ int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) { int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, (SMeta *)meta, 0); - code = metaGetTableEntryByUid(&mr, uid); + metaReaderDoInit(&mr, (SMeta *)meta, 0); + code = metaReaderGetTableEntryByUid(&mr, uid); if (code < 0) { metaReaderClear(&mr); return -1; @@ -241,15 +171,15 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { return 0; } -int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) { +int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, (SMeta *)meta, 0); + metaReaderDoInit(&mr, ((SVnode *)pVnode)->pMeta, 0); SMetaReader *pReader = &mr; // query name.idx - if (tdbTbGet(pReader->pMeta->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(((SMeta *)pReader->pMeta)->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; metaReaderClear(&mr); return -1; @@ -262,10 +192,10 @@ int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) { return 0; } -int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType) { +int metaGetTableTypeByName(void *pVnode, char *tbName, ETableType *tbType) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, (SMeta *)meta, 0); + metaReaderDoInit(&mr, ((SVnode*)pVnode)->pMeta, 0); code = metaGetTableEntryByName(&mr, tbName); if (code == 0) *tbType = mr.me.type; @@ -282,8 +212,31 @@ int metaReadNext(SMetaReader *pReader) { return 0; } +int metaGetTableTtlByUid(void *meta, uint64_t uid, int64_t *ttlDays) { + int code = -1; + SMetaReader mr = {0}; + metaReaderDoInit(&mr, (SMeta *)meta, 0); + code = metaReaderGetTableEntryByUid(&mr, uid); + if (code < 0) { + goto _exit; + } + if (mr.me.type == TSDB_CHILD_TABLE) { + *ttlDays = mr.me.ctbEntry.ttlDays; + } else if (mr.me.type == TSDB_NORMAL_TABLE) { + *ttlDays = mr.me.ntbEntry.ttlDays; + } else { + goto _exit; + } + + code = 0; + +_exit: + metaReaderClear(&mr); + return code; +} + #if 1 // =================================================== -SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { +SMTbCursor *metaOpenTbCursor(void *pVnode) { SMTbCursor *pTbCur = NULL; pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur)); @@ -291,12 +244,11 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { return NULL; } - metaReaderInit(&pTbCur->mr, pMeta, 0); - - tdbTbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL); - - tdbTbcMoveToFirst(pTbCur->pDbc); - + SVnode* pVnodeObj = pVnode; + // tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); + pTbCur->pMeta = pVnodeObj->pMeta; + pTbCur->paused = 1; + metaResumeTbCursor(pTbCur, 1); return pTbCur; } @@ -304,21 +256,52 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) { if (pTbCur) { tdbFree(pTbCur->pKey); tdbFree(pTbCur->pVal); - metaReaderClear(&pTbCur->mr); - if (pTbCur->pDbc) { - tdbTbcClose(pTbCur->pDbc); + if (!pTbCur->paused) { + metaReaderClear(&pTbCur->mr); + if (pTbCur->pDbc) { + tdbTbcClose((TBC *)pTbCur->pDbc); + } } taosMemoryFree(pTbCur); } } +void metaPauseTbCursor(SMTbCursor *pTbCur) { + if (!pTbCur->paused) { + metaReaderClear(&pTbCur->mr); + tdbTbcClose((TBC *)pTbCur->pDbc); + pTbCur->paused = 1; + } +} +void metaResumeTbCursor(SMTbCursor *pTbCur, int8_t first) { + if (pTbCur->paused) { + metaReaderDoInit(&pTbCur->mr, pTbCur->pMeta, 0); + + tdbTbcOpen(((SMeta *)pTbCur->pMeta)->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); + + if (first) { + tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); + } else { + int c = 0; + tdbTbcMoveTo(pTbCur->pDbc, pTbCur->pKey, pTbCur->kLen, &c); + if (c < 0) { + tdbTbcMoveToPrev(pTbCur->pDbc); + } else { + tdbTbcMoveToNext(pTbCur->pDbc); + } + } + + pTbCur->paused = 0; + } +} + int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType) { int ret; void *pBuf; STbCfg tbCfg; for (;;) { - ret = tdbTbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); + ret = tdbTbcNext((TBC *)pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); if (ret < 0) { return -1; } @@ -342,7 +325,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) { STbCfg tbCfg; for (;;) { - ret = tdbTbcPrev(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); + ret = tdbTbcPrev((TBC *)pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); if (ret < 0) { return -1; } @@ -387,42 +370,6 @@ _query: tDecoderClear(&dc); goto _exit; } - { // Traverse to find the previous qualified data - TBC *pCur; - tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); - STbDbKey key = {.version = sver, .uid = INT64_MAX}; - int c = 0; - tdbTbcMoveTo(pCur, &key, sizeof(key), &c); - if (c < 0) { - tdbTbcMoveToPrev(pCur); - } - - void *pKey = NULL; - void *pVal = NULL; - int vLen = 0, kLen = 0; - while (1) { - int32_t ret = tdbTbcPrev(pCur, &pKey, &kLen, &pVal, &vLen); - if (ret < 0) break; - - STbDbKey *tmp = (STbDbKey *)pKey; - if (tmp->uid != uid) { - continue; - } - SDecoder dcNew = {0}; - SMetaEntry meNew = {0}; - tDecoderInit(&dcNew, pVal, vLen); - metaDecodeEntry(&dcNew, &meNew); - pSchema = tCloneSSchemaWrapper(&meNew.stbEntry.schemaRow); - tDecoderClear(&dcNew); - tdbTbcClose(pCur); - tdbFree(pKey); - tdbFree(pVal); - goto _exit; - } - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCur); - } } else if (me.type == TSDB_CHILD_TABLE) { uid = me.ctbEntry.suid; tDecoderClear(&dc); @@ -447,7 +394,6 @@ _query: tDecoderClear(&dc); _exit: - tDecoderClear(&dc); if (lock) { metaULock(pMeta); } @@ -455,7 +401,6 @@ _exit: return pSchema; _err: - tDecoderClear(&dc); if (lock) { metaULock(pMeta); } @@ -463,37 +408,6 @@ _err: return NULL; } -int metaTtlSmaller(SMeta *pMeta, uint64_t ttl, SArray *uidList) { - TBC *pCur; - int ret = tdbTbcOpen(pMeta->pTtlIdx, &pCur, NULL); - if (ret < 0) { - return ret; - } - - STtlIdxKey ttlKey = {0}; - ttlKey.dtime = ttl; - ttlKey.uid = INT64_MAX; - int c = 0; - tdbTbcMoveTo(pCur, &ttlKey, sizeof(ttlKey), &c); - if (c < 0) { - tdbTbcMoveToPrev(pCur); - } - - void *pKey = NULL; - int kLen = 0; - while (1) { - ret = tdbTbcPrev(pCur, &pKey, &kLen, NULL, NULL); - if (ret < 0) { - break; - } - ttlKey = *(STtlIdxKey *)pKey; - taosArrayPush(uidList, &ttlKey.uid); - } - tdbFree(pKey); - tdbTbcClose(pCur); - return 0; -} - struct SMCtbCursor { SMeta *pMeta; TBC *pCur; @@ -770,8 +684,6 @@ int64_t metaGetTimeSeriesNum(SMeta *pMeta) { return pMeta->pVnode->config.vndStats.numOfTimeSeries + pMeta->pVnode->config.vndStats.numOfNTimeSeries; } -int64_t metaGetNtbNum(SMeta *pMeta) { return pMeta->pVnode->config.vndStats.numOfNTables; } - typedef struct { SMeta *pMeta; TBC *pCur; @@ -870,13 +782,13 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { } SMetaReader mr = {0}; - metaReaderInit(&mr, pMeta, 0); + metaReaderDoInit(&mr, pMeta, 0); int64_t smaId; int smaIdx = 0; STSma *pTSma = NULL; for (int i = 0; i < pSW->number; ++i) { smaId = *(tb_uid_t *)taosArrayGet(pSmaIds, i); - if (metaGetTableEntryByUid(&mr, smaId) < 0) { + if (metaReaderGetTableEntryByUid(&mr, smaId) < 0) { tDecoderClear(&mr.coder); metaWarn("vgId:%d, no entry for tbId:%" PRIi64 ", smaId:%" PRIi64, TD_VID(pMeta->pVnode), uid, smaId); continue; @@ -925,8 +837,8 @@ _err: STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { STSma *pTSma = NULL; SMetaReader mr = {0}; - metaReaderInit(&mr, pMeta, 0); - if (metaGetTableEntryByUid(&mr, indexUid) < 0) { + metaReaderDoInit(&mr, pMeta, 0); + if (metaReaderGetTableEntryByUid(&mr, indexUid) < 0) { metaWarn("vgId:%d, failed to get table entry for smaId:%" PRIi64, TD_VID(pMeta->pVnode), indexUid); metaReaderClear(&mr); return NULL; @@ -1027,7 +939,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) { #endif -const void *metaGetTableTagVal(void *pTag, int16_t type, STagVal *val) { +const void *metaGetTableTagVal(const void *pTag, int16_t type, STagVal *val) { STag *tag = (STag *)pTag; if (type == TSDB_DATA_TYPE_JSON) { return tag; @@ -1083,8 +995,10 @@ typedef struct { int32_t vLen; } SIdxCursor; -int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { - int32_t ret = 0; +int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *arg, SArray *pUids) { + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; + SMetaFltParam *param = arg; + int32_t ret = 0; SIdxCursor *pCursor = NULL; pCursor = (SIdxCursor *)taosMemoryCalloc(1, sizeof(SIdxCursor)); @@ -1094,17 +1008,17 @@ int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) pCursor->type = param->type; metaRLock(pMeta); - ret = tdbTbcOpen(pMeta->pCtimeIdx, &pCursor->pCur, NULL); + ret = tdbTbcOpen(pMeta->pBtimeIdx, &pCursor->pCur, NULL); if (ret != 0) { goto END; } int64_t uidLimit = param->reverse ? INT64_MAX : 0; - SCtimeIdxKey ctimeKey = {.ctime = *(int64_t *)(param->val), .uid = uidLimit}; - SCtimeIdxKey *pCtimeKey = &ctimeKey; + SBtimeIdxKey btimeKey = {.btime = *(int64_t *)(param->val), .uid = uidLimit}; + SBtimeIdxKey *pBtimeKey = &btimeKey; int cmp = 0; - if (tdbTbcMoveTo(pCursor->pCur, &ctimeKey, sizeof(ctimeKey), &cmp) < 0) { + if (tdbTbcMoveTo(pCursor->pCur, &btimeKey, sizeof(btimeKey), &cmp) < 0) { goto END; } @@ -1118,10 +1032,10 @@ int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, NULL, NULL); if (valid < 0) break; - SCtimeIdxKey *p = entryKey; + SBtimeIdxKey *p = entryKey; if (count > TRY_ERROR_LIMIT) break; - int32_t cmp = (*param->filterFunc)((void *)&p->ctime, (void *)&pCtimeKey->ctime, param->type); + int32_t cmp = (*param->filterFunc)((void *)&p->btime, (void *)&pBtimeKey->btime, param->type); if (cmp == 0) taosArrayPush(pUids, &p->uid); else { @@ -1141,9 +1055,11 @@ END: return ret; } -int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { - int32_t ret = 0; - char *buf = NULL; +int32_t metaFilterTableName(void *pVnode, SMetaFltParam *arg, SArray *pUids) { + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; + SMetaFltParam *param = arg; + int32_t ret = 0; + char *buf = NULL; STagIdxKey *pKey = NULL; int32_t nKey = 0; @@ -1206,9 +1122,11 @@ END: return ret; } -int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { - int32_t ret = 0; - char *buf = NULL; +int32_t metaFilterTtl(void *pVnode, SMetaFltParam *arg, SArray *pUids) { + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; + SMetaFltParam *param = arg; + int32_t ret = 0; + char *buf = NULL; STtlIdxKey *pKey = NULL; int32_t nKey = 0; @@ -1221,7 +1139,7 @@ int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { pCursor->type = param->type; metaRLock(pMeta); - ret = tdbTbcOpen(pMeta->pTtlIdx, &pCursor->pCur, NULL); + //ret = tdbTbcOpen(pMeta->pTtlIdx, &pCursor->pCur, NULL); END: if (pCursor->pMeta) metaULock(pCursor->pMeta); @@ -1235,7 +1153,10 @@ END: // impl later return 0; } -int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { +int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) { + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; + SMetaFltParam *param = arg; + SMetaEntry oStbEntry = {0}; int32_t ret = -1; char *buf = NULL; @@ -1418,7 +1339,8 @@ static int32_t metaGetTableTagByUid(SMeta *pMeta, int64_t suid, int64_t uid, voi return ret; } -int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList) { +int32_t metaGetTableTagsByUids(void *pVnode, int64_t suid, SArray *uidList) { + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; const int32_t LIMIT = 128; int32_t isLock = false; @@ -1450,8 +1372,8 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList) { return 0; } -int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *pUidTagInfo) { - SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid, 1); +int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *pUidTagInfo) { + SMCtbCursor *pCur = metaOpenCtbCursor(((SVnode *)pVnode)->pMeta, suid, 1); // If len > 0 means there already have uids, and we only want the // tags of the specified tables, of which uid in the uid list. Otherwise, all table tags are retrieved and kept @@ -1556,30 +1478,35 @@ _exit: return code; } -int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo) { +int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables) { int32_t code = 0; + *numOfTables = 0; - metaRLock(pMeta); + SVnode *pVnodeObj = pVnode; + metaRLock(pVnodeObj->pMeta); // fast path: search cache - if (metaStatsCacheGet(pMeta, uid, pInfo) == TSDB_CODE_SUCCESS) { - metaULock(pMeta); + SMetaStbStats state = {0}; + if (metaStatsCacheGet(pVnodeObj->pMeta, uid, &state) == TSDB_CODE_SUCCESS) { + metaULock(pVnodeObj->pMeta); + *numOfTables = state.ctbNum; goto _exit; } // slow path: search TDB int64_t ctbNum = 0; - vnodeGetCtbNum(pMeta->pVnode, uid, &ctbNum); + vnodeGetCtbNum(pVnode, uid, &ctbNum); - metaULock(pMeta); + metaULock(pVnodeObj->pMeta); + *numOfTables = ctbNum; - pInfo->uid = uid; - pInfo->ctbNum = ctbNum; + state.uid = uid; + state.ctbNum = ctbNum; // upsert the cache - metaWLock(pMeta); - metaStatsCacheUpsert(pMeta, pInfo); - metaULock(pMeta); + metaWLock(pVnodeObj->pMeta); + metaStatsCacheUpsert(pVnodeObj->pMeta, &state); + metaULock(pVnodeObj->pMeta); _exit: return code; diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 8d5821e28bfa2c33a5582e34bc32127970af6320..91704f5c7ab1558a5610496757763e053dad570d 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -13,8 +13,10 @@ * along with this program. If not, see . */ +#include "vnodeInt.h" #include "meta.h" + static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME); static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME); @@ -35,8 +37,8 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { // validate req // save smaIndex - metaReaderInit(&mr, pMeta, 0); - if (metaGetTableEntryByUidCache(&mr, pCfg->indexUid) == 0) { + metaReaderDoInit(&mr, pMeta, 0); + if (metaReaderGetTableEntryByUidCache(&mr, pCfg->indexUid) == 0) { #if 1 terrno = TSDB_CODE_TSMA_ALREADY_EXIST; metaReaderClear(&mr); diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 707dd66e3011dd98cd8449f20bf748a2b6f2ff7c..18190ac533df4ab817e496f02b7bd02c1f4215a6 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -222,29 +222,29 @@ static void destroySTableInfoForChildTable(void* data) { } static void MoveToSnapShotVersion(SSnapContext* ctx) { - tdbTbcClose(ctx->pCur); - tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL); + tdbTbcClose((TBC*)ctx->pCur); + tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL); STbDbKey key = {.version = ctx->snapVersion, .uid = INT64_MAX}; int c = 0; - tdbTbcMoveTo(ctx->pCur, &key, sizeof(key), &c); + tdbTbcMoveTo((TBC*)ctx->pCur, &key, sizeof(key), &c); if (c < 0) { - tdbTbcMoveToPrev(ctx->pCur); + tdbTbcMoveToPrev((TBC*)ctx->pCur); } } static int32_t MoveToPosition(SSnapContext* ctx, int64_t ver, int64_t uid) { - tdbTbcClose(ctx->pCur); - tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL); + tdbTbcClose((TBC*)ctx->pCur); + tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL); STbDbKey key = {.version = ver, .uid = uid}; int c = 0; - tdbTbcMoveTo(ctx->pCur, &key, sizeof(key), &c); + tdbTbcMoveTo((TBC*)ctx->pCur, &key, sizeof(key), &c); return c; } static void MoveToFirst(SSnapContext* ctx) { - tdbTbcClose(ctx->pCur); - tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL); - tdbTbcMoveToFirst(ctx->pCur); + tdbTbcClose((TBC*)ctx->pCur); + tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL); + tdbTbcMoveToFirst((TBC*)ctx->pCur); } static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo) { @@ -260,12 +260,12 @@ static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo) taosHashPut(suidInfo, &me->uid, sizeof(tb_uid_t), &dataTmp, sizeof(STableInfoForChildTable)); } -int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, +int32_t buildSnapContext(SVnode* pVnode, int64_t snapVersion, int64_t suid, int8_t subType, int8_t withMeta, SSnapContext** ctxRet) { SSnapContext* ctx = taosMemoryCalloc(1, sizeof(SSnapContext)); if (ctx == NULL) return -1; *ctxRet = ctx; - ctx->pMeta = pMeta; + ctx->pMeta = pVnode->pMeta; ctx->snapVersion = snapVersion; ctx->suid = suid; ctx->subType = subType; @@ -291,7 +291,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t metaDebug("tmqsnap init snapVersion:%" PRIi64, ctx->snapVersion); MoveToFirst(ctx); while (1) { - int32_t ret = tdbTbcNext(ctx->pCur, &pKey, &kLen, &pVal, &vLen); + int32_t ret = tdbTbcNext((TBC*)ctx->pCur, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; STbDbKey* tmp = (STbDbKey*)pKey; if (tmp->version > ctx->snapVersion) break; @@ -301,7 +301,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t continue; } - if (tdbTbGet(pMeta->pUidIdx, &tmp->uid, sizeof(tb_uid_t), NULL, NULL) < + if (tdbTbGet(ctx->pMeta->pUidIdx, &tmp->uid, sizeof(tb_uid_t), NULL, NULL) < 0) { // check if table exist for now, need optimize later continue; } @@ -329,7 +329,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t MoveToSnapShotVersion(ctx); while (1) { - int32_t ret = tdbTbcPrev(ctx->pCur, &pKey, &kLen, &pVal, &vLen); + int32_t ret = tdbTbcPrev((TBC*)ctx->pCur, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; STbDbKey* tmp = (STbDbKey*)pKey; @@ -378,7 +378,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t } int32_t destroySnapContext(SSnapContext* ctx) { - tdbTbcClose(ctx->pCur); + tdbTbcClose((TBC*)ctx->pCur); taosArrayDestroy(ctx->idList); taosHashCleanup(ctx->idVersion); taosHashCleanup(ctx->suidInfo); @@ -466,7 +466,7 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) { return c; } -int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) { +int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) { int32_t ret = 0; void* pKey = NULL; void* pVal = NULL; @@ -476,7 +476,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in if (ctx->index >= taosArrayGetSize(ctx->idList)) { metaDebug("tmqsnap get meta end"); ctx->index = 0; - ctx->queryMeta = false; // change to get data + ctx->queryMeta = 0; // change to get data return 0; } @@ -496,7 +496,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in metaDebug("tmqsnap get meta not exist uid:%" PRIi64 " version:%" PRIi64, *uid, idInfo->version); } - tdbTbcGet(ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); + tdbTbcGet((TBC*)ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); SDecoder dc = {0}; SMetaEntry me = {0}; tDecoderInit(&dc, pVal, vLen); @@ -598,7 +598,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in return ret; } -SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) { +SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext* ctx) { SMetaTableInfo result = {0}; void* pKey = NULL; void* pVal = NULL; @@ -619,10 +619,10 @@ SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) { int32_t ret = MoveToPosition(ctx, idInfo->version, *uidTmp); if (ret != 0) { - metaDebug("tmqsnap getUidfromSnapShot not exist uid:%" PRIi64 " version:%" PRIi64, *uidTmp, idInfo->version); + metaDebug("tmqsnap getMetaTableInfoFromSnapshot not exist uid:%" PRIi64 " version:%" PRIi64, *uidTmp, idInfo->version); continue; } - tdbTbcGet(ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); + tdbTbcGet((TBC*)ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); SDecoder dc = {0}; SMetaEntry me = {0}; tDecoderInit(&dc, pVal, vLen); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 83f2ece571e985e08a29cc7a4089659362780a20..632e6dd872be9b7381309653404b88f4dbf5152d 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -20,7 +20,7 @@ static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, con static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME); static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME); @@ -28,8 +28,8 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type); static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); // opt ins_tables query -static int metaUpdateCtimeIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME); @@ -207,7 +207,10 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { tb_uid_t uid = *(tb_uid_t *)pData; tdbFree(pData); SMetaInfo info; - metaGetInfo(pMeta, uid, &info, NULL); + if (metaGetInfo(pMeta, uid, &info, NULL) == TSDB_CODE_NOT_FOUND) { + terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; + return -1; + } if (info.uid == info.suid) { return 0; } else { @@ -452,7 +455,7 @@ int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { } } - if (diffIdx == -1 && diffIdx == 0) { + if (diffIdx == -1 || diffIdx == 0) { goto _err; } @@ -690,7 +693,7 @@ _err: return -1; } -int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) { +int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) { SMetaEntry me = {0}; SMetaReader mr = {0}; @@ -709,7 +712,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe } // validate req - metaReaderInit(&mr, pMeta, 0); + metaReaderDoInit(&mr, pMeta, 0); if (metaGetTableEntryByName(&mr, pReq->name) == 0) { if (pReq->type == TSDB_CHILD_TABLE && pReq->ctb.suid != mr.me.ctbEntry.suid) { terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; @@ -729,12 +732,12 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe metaReaderClear(&mr); // build SMetaEntry - me.version = version; + me.version = ver; me.type = pReq->type; me.uid = pReq->uid; me.name = pReq->name; if (me.type == TSDB_CHILD_TABLE) { - me.ctbEntry.ctime = pReq->ctime; + me.ctbEntry.btime = pReq->btime; me.ctbEntry.ttlDays = pReq->ttl; me.ctbEntry.commentLen = pReq->commentLen; me.ctbEntry.comment = pReq->comment; @@ -767,9 +770,10 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe metaWLock(pMeta); metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1); metaUidCacheClear(pMeta, me.ctbEntry.suid); + metaTbGroupCacheClear(pMeta, me.ctbEntry.suid); metaULock(pMeta); } else { - me.ntbEntry.ctime = pReq->ctime; + me.ntbEntry.btime = pReq->btime; me.ntbEntry.ttlDays = pReq->ttl; me.ntbEntry.commentLen = pReq->commentLen; me.ntbEntry.comment = pReq->comment; @@ -837,56 +841,125 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi return 0; } -int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) { - int ret = metaTtlSmaller(pMeta, ttl, tbUids); - if (ret != 0) { - return ret; +static void metaDropTables(SMeta *pMeta, SArray *tbUids) { + metaWLock(pMeta); + for (int i = 0; i < TARRAY_SIZE(tbUids); ++i) { + tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUids, i); + metaDropTableByUid(pMeta, uid, NULL); + metaDebug("batch drop table:%" PRId64, uid); } - if (taosArrayGetSize(tbUids) == 0) { - return 0; + metaULock(pMeta); +} + +static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { + int32_t code = 0; + // 1, tranverse table's + // 2, validate table name using vnodeValidateTableHash + // 3, push invalidated table's uid into uidList + + TBC *pCur; + code = tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); + if (code < 0) { + return code; } - metaWLock(pMeta); - for (int i = 0; i < taosArrayGetSize(tbUids); ++i) { - tb_uid_t *uid = (tb_uid_t *)taosArrayGet(tbUids, i); - metaDropTableByUid(pMeta, *uid, NULL); - metaDebug("ttl drop table:%" PRId64, *uid); + code = tdbTbcMoveToFirst(pCur); + if (code) { + tdbTbcClose(pCur); + return code; } - metaULock(pMeta); + + void *pData = NULL, *pKey = NULL; + int nData = 0, nKey = 0; + + while (1) { + int32_t ret = tdbTbcNext(pCur, &pKey, &nKey, &pData, &nData); + if (ret < 0) { + break; + } + + SMetaEntry me = {0}; + SDecoder dc = {0}; + tDecoderInit(&dc, pData, nData); + metaDecodeEntry(&dc, &me); + + if (me.type != TSDB_SUPER_TABLE) { + char tbFName[TSDB_TABLE_FNAME_LEN + 1]; + snprintf(tbFName, sizeof(tbFName), "%s.%s", pMeta->pVnode->config.dbname, me.name); + tbFName[TSDB_TABLE_FNAME_LEN] = '\0'; + int32_t ret = vnodeValidateTableHash(pMeta->pVnode, tbFName); + if (ret < 0 && terrno == TSDB_CODE_VND_HASH_MISMATCH) { + taosArrayPush(uidList, &me.uid); + } + } + tDecoderClear(&dc); + } + tdbFree(pData); + tdbFree(pKey); + tdbTbcClose(pCur); + return 0; } -static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME) { - int64_t ttlDays = 0; - int64_t ctime = 0; - if (pME->type == TSDB_CHILD_TABLE) { - ctime = pME->ctbEntry.ctime; - ttlDays = pME->ctbEntry.ttlDays; - } else if (pME->type == TSDB_NORMAL_TABLE) { - ctime = pME->ntbEntry.ctime; - ttlDays = pME->ntbEntry.ttlDays; - } else { - metaError("meta/table: invalide table type: %" PRId8 " build ttl idx key failed.", pME->type); - return; +int32_t metaTrimTables(SMeta *pMeta) { + int32_t code = 0; + + SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); + if (tbUids == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + code = metaFilterTableByHash(pMeta, tbUids); + if (code != 0) { + goto end; + } + if (TARRAY_SIZE(tbUids) == 0) { + goto end; } - if (ttlDays <= 0) return; + metaInfo("vgId:%d, trim %ld tables", TD_VID(pMeta->pVnode), taosArrayGetSize(tbUids)); + metaDropTables(pMeta, tbUids); + +end: + taosArrayDestroy(tbUids); - ttlKey->dtime = ctime / 1000 + ttlDays * tsTtlUnit; - ttlKey->uid = pME->uid; + return code; } -static int metaBuildCtimeIdxKey(SCtimeIdxKey *ctimeKey, const SMetaEntry *pME) { - int64_t ctime; + +int metaTtlDropTable(SMeta *pMeta, int64_t timePointMs, SArray *tbUids) { + int ret = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn); + if (ret != 0) { + metaError("ttl failed to flush, ret:%d", ret); + return ret; + } + + ret = ttlMgrFindExpired(pMeta->pTtlMgr, timePointMs, tbUids); + if (ret != 0) { + metaError("ttl failed to find expired table, ret:%d", ret); + return ret; + } + if (TARRAY_SIZE(tbUids) == 0) { + return 0; + } + + metaInfo("ttl find expired table count: %zu", TARRAY_SIZE(tbUids)); + + metaDropTables(pMeta, tbUids); + return 0; +} + +static int metaBuildBtimeIdxKey(SBtimeIdxKey *btimeKey, const SMetaEntry *pME) { + int64_t btime; if (pME->type == TSDB_CHILD_TABLE) { - ctime = pME->ctbEntry.ctime; + btime = pME->ctbEntry.btime; } else if (pME->type == TSDB_NORMAL_TABLE) { - ctime = pME->ntbEntry.ctime; + btime = pME->ntbEntry.btime; } else { return -1; } - ctimeKey->ctime = ctime; - ctimeKey->uid = pME->uid; + btimeKey->btime = btime; + btimeKey->uid = pME->uid; return 0; } @@ -900,11 +973,17 @@ static int metaBuildNColIdxKey(SNcolIdxKey *ncolKey, const SMetaEntry *pME) { return 0; } -static int metaDeleteTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { - STtlIdxKey ttlKey = {0}; - metaBuildTtlIdxKey(&ttlKey, pME); - if (ttlKey.dtime == 0) return 0; - return tdbTbDelete(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), pMeta->txn); +static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { + if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return 0; + + STtlDelTtlCtx ctx = {.uid = pME->uid, .pTxn = pMeta->txn}; + if (pME->type == TSDB_CHILD_TABLE) { + ctx.ttlDays = pME->ctbEntry.ttlDays; + } else { + ctx.ttlDays = pME->ntbEntry.ttlDays; + } + + return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); } static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { @@ -986,10 +1065,10 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, pMeta->txn); tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), pMeta->txn); - if (e.type == TSDB_CHILD_TABLE || e.type == TSDB_NORMAL_TABLE) metaDeleteCtimeIdx(pMeta, &e); + if (e.type == TSDB_CHILD_TABLE || e.type == TSDB_NORMAL_TABLE) metaDeleteBtimeIdx(pMeta, &e); if (e.type == TSDB_NORMAL_TABLE) metaDeleteNcolIdx(pMeta, &e); - if (e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e); + if (e.type != TSDB_SUPER_TABLE) metaDeleteTtl(pMeta, &e); if (e.type == TSDB_CHILD_TABLE) { tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), pMeta->txn); @@ -998,6 +1077,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { metaUpdateStbStats(pMeta, e.ctbEntry.suid, -1); metaUidCacheClear(pMeta, e.ctbEntry.suid); + metaTbGroupCacheClear(pMeta, e.ctbEntry.suid); } else if (e.type == TSDB_NORMAL_TABLE) { // drop schema.db (todo) @@ -1009,6 +1089,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { metaStatsCacheDrop(pMeta, uid); metaUidCacheClear(pMeta, uid); + metaTbGroupCacheClear(pMeta, uid); --pMeta->pVnode->config.vndStats.numOfSTables; } @@ -1020,23 +1101,23 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { return 0; } // opt ins_tables -int metaUpdateCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { - SCtimeIdxKey ctimeKey = {0}; - if (metaBuildCtimeIdxKey(&ctimeKey, pME) < 0) { +int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { + SBtimeIdxKey btimeKey = {0}; + if (metaBuildBtimeIdxKey(&btimeKey, pME) < 0) { return 0; } - metaTrace("vgId:%d, start to save version:%" PRId64 " uid:%" PRId64 " ctime:%" PRId64, TD_VID(pMeta->pVnode), - pME->version, pME->uid, ctimeKey.ctime); + metaTrace("vgId:%d, start to save version:%" PRId64 " uid:%" PRId64 " btime:%" PRId64, TD_VID(pMeta->pVnode), + pME->version, pME->uid, btimeKey.btime); - return tdbTbUpsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn); + return tdbTbUpsert(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), NULL, 0, pMeta->txn); } -int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { - SCtimeIdxKey ctimeKey = {0}; - if (metaBuildCtimeIdxKey(&ctimeKey, pME) < 0) { +int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { + SBtimeIdxKey btimeKey = {0}; + if (metaBuildBtimeIdxKey(&btimeKey, pME) < 0) { return 0; } - return tdbTbDelete(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), pMeta->txn); + return tdbTbDelete(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), pMeta->txn); } int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { SNcolIdxKey ncolKey = {0}; @@ -1246,6 +1327,8 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl metaULock(pMeta); + metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs); + metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp); if (entry.pBuf) taosMemoryFree(entry.pBuf); @@ -1429,9 +1512,12 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn); metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid); + metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid); metaULock(pMeta); + metaUpdateChangeTime(pMeta, ctbEntry.uid, pAlterTbReq->ctimeMs); + tDecoderClear(&dc1); tDecoderClear(&dc2); taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); @@ -1520,9 +1606,9 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p // build SMetaEntry if (entry.type == TSDB_CHILD_TABLE) { if (pAlterTbReq->updateTTL) { - metaDeleteTtlIdx(pMeta, &entry); + metaDeleteTtl(pMeta, &entry); entry.ctbEntry.ttlDays = pAlterTbReq->newTTL; - metaUpdateTtlIdx(pMeta, &entry); + metaUpdateTtl(pMeta, &entry); } if (pAlterTbReq->newCommentLen >= 0) { entry.ctbEntry.commentLen = pAlterTbReq->newCommentLen; @@ -1530,9 +1616,9 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p } } else { if (pAlterTbReq->updateTTL) { - metaDeleteTtlIdx(pMeta, &entry); + metaDeleteTtl(pMeta, &entry); entry.ntbEntry.ttlDays = pAlterTbReq->newTTL; - metaUpdateTtlIdx(pMeta, &entry); + metaUpdateTtl(pMeta, &entry); } if (pAlterTbReq->newCommentLen >= 0) { entry.ntbEntry.commentLen = pAlterTbReq->newCommentLen; @@ -1545,6 +1631,8 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p metaUpdateUidIdx(pMeta, &entry); metaULock(pMeta); + metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs); + tdbTbcClose(pTbDbc); tdbTbcClose(pUidIdxc); tDecoderClear(&dc); @@ -1574,10 +1662,11 @@ static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTb if (ret < 0) { terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; return -1; + } else { + uid = *(tb_uid_t *)pVal; + tdbFree(pVal); + pVal = NULL; } - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { ret = -1; @@ -1656,12 +1745,16 @@ static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTb nTagData = tDataTypes[pCol->type].bytes; } if (metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, uid, &pTagIdxKey, &nTagIdxKey) < 0) { + tdbFree(pKey); + tdbFree(pVal); metaDestroyTagIdxKey(pTagIdxKey); + tdbTbcClose(pCtbIdxc); goto _err; } tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); metaDestroyTagIdxKey(pTagIdxKey); } + tdbTbcClose(pCtbIdxc); return 0; _err: @@ -1884,11 +1977,32 @@ static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbUpsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn); } -static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { - STtlIdxKey ttlKey = {0}; - metaBuildTtlIdxKey(&ttlKey, pME); - if (ttlKey.dtime == 0) return 0; - return tdbTbUpsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, pMeta->txn); +static int metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME) { + if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return 0; + + STtlUpdTtlCtx ctx = {.uid = pME->uid}; + if (pME->type == TSDB_CHILD_TABLE) { + ctx.ttlDays = pME->ctbEntry.ttlDays; + ctx.changeTimeMs = pME->ctbEntry.btime; + } else { + ctx.ttlDays = pME->ntbEntry.ttlDays; + ctx.changeTimeMs = pME->ntbEntry.btime; + } + + return ttlMgrInsertTtl(pMeta->pTtlMgr, &ctx); +} + +int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs) { + if (!tsTtlChangeOnWrite) return 0; + + if (changeTimeMs <= 0) { + metaWarn("Skip to change ttl deletetion time on write, uid: %" PRId64, uid); + return TSDB_CODE_VERSION_NOT_COMPATIBLE; + } + + STtlUpdCtimeCtx ctx = {.uid = uid, .changeTimeMs = changeTimeMs}; + + return ttlMgrUpdateChangeTime(pMeta->pTtlMgr, &ctx); } static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -2099,7 +2213,7 @@ int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { } } - code = metaUpdateCtimeIdx(pMeta, pME); + code = metaUpdateBtimeIdx(pMeta, pME); VND_CHECK_CODE(code, line, _err); if (pME->type == TSDB_NORMAL_TABLE) { @@ -2108,7 +2222,7 @@ int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { } if (pME->type != TSDB_SUPER_TABLE) { - code = metaUpdateTtlIdx(pMeta, pME); + code = metaUpdateTtl(pMeta, pME); VND_CHECK_CODE(code, line, _err); } diff --git a/source/dnode/vnode/src/meta/metaTtl.c b/source/dnode/vnode/src/meta/metaTtl.c new file mode 100644 index 0000000000000000000000000000000000000000..45f697258c4e8e599f1d0a2179ce10db03b6f3f5 --- /dev/null +++ b/source/dnode/vnode/src/meta/metaTtl.c @@ -0,0 +1,469 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "metaTtl.h" +#include "meta.h" + +typedef struct { + TTB *pNewTtlIdx; + SMeta *pMeta; +} SConvertData; + +static void ttlMgrCleanup(STtlManger *pTtlMgr); + +static int ttlMgrConvert(TTB *pOldTtlIdx, TTB *pNewTtlIdx, void *pMeta); + +static void ttlMgrBuildKey(STtlIdxKeyV1 *pTtlKey, int64_t ttlDays, int64_t changeTimeMs, tb_uid_t uid); +static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int ttlIdxKeyV1Cmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int ttlMgrFillCache(STtlManger *pTtlMgr); +static int32_t ttlMgrFillCacheOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen, void *pTtlCache); +static int32_t ttlMgrConvertOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen, void *pConvertData); + +static int32_t ttlMgrWLock(STtlManger *pTtlMgr); +static int32_t ttlMgrRLock(STtlManger *pTtlMgr); +static int32_t ttlMgrULock(STtlManger *pTtlMgr); + +const char *ttlTbname = "ttl.idx"; +const char *ttlV1Tbname = "ttlv1.idx"; + +int ttlMgrOpen(STtlManger **ppTtlMgr, TDB *pEnv, int8_t rollback, const char *logPrefix) { + int ret = TSDB_CODE_SUCCESS; + int64_t startNs = taosGetTimestampNs(); + + *ppTtlMgr = NULL; + + STtlManger *pTtlMgr = (STtlManger *)tdbOsCalloc(1, sizeof(*pTtlMgr)); + if (pTtlMgr == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + char *logBuffer = (char *)tdbOsCalloc(1, strlen(logPrefix) + 1); + if (logBuffer == NULL) { + tdbOsFree(pTtlMgr); + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(logBuffer, logPrefix); + pTtlMgr->logPrefix = logBuffer; + + ret = tdbTbOpen(ttlV1Tbname, TDB_VARIANT_LEN, TDB_VARIANT_LEN, ttlIdxKeyV1Cmpr, pEnv, &pTtlMgr->pTtlIdx, rollback); + if (ret < 0) { + metaError("%s, failed to open %s since %s", pTtlMgr->logPrefix, ttlV1Tbname, tstrerror(terrno)); + tdbOsFree(pTtlMgr); + return ret; + } + + pTtlMgr->pTtlCache = taosHashInit(8192, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + pTtlMgr->pDirtyUids = taosHashInit(8192, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + + taosThreadRwlockInit(&pTtlMgr->lock, NULL); + + ret = ttlMgrFillCache(pTtlMgr); + if (ret < 0) { + metaError("%s, failed to fill hash since %s", pTtlMgr->logPrefix, tstrerror(terrno)); + ttlMgrCleanup(pTtlMgr); + return ret; + } + + int64_t endNs = taosGetTimestampNs(); + metaInfo("%s, ttl mgr open end, hash size: %d, time consumed: %" PRId64 " ns", pTtlMgr->logPrefix, + taosHashGetSize(pTtlMgr->pTtlCache), endNs - startNs); + + *ppTtlMgr = pTtlMgr; + return TSDB_CODE_SUCCESS; +} + +void ttlMgrClose(STtlManger *pTtlMgr) { ttlMgrCleanup(pTtlMgr); } + +bool ttlMgrNeedUpgrade(TDB *pEnv) { + bool needUpgrade = tdbTbExist(ttlTbname, pEnv); + if (needUpgrade) { + metaInfo("find ttl idx in old version , will convert"); + } + return needUpgrade; +} + +int ttlMgrUpgrade(STtlManger *pTtlMgr, void *pMeta) { + SMeta *meta = (SMeta *)pMeta; + int ret = TSDB_CODE_SUCCESS; + + if (!tdbTbExist(ttlTbname, meta->pEnv)) return TSDB_CODE_SUCCESS; + + metaInfo("%s, ttl mgr start upgrade", pTtlMgr->logPrefix); + + int64_t startNs = taosGetTimestampNs(); + + ret = tdbTbOpen(ttlTbname, sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, meta->pEnv, &pTtlMgr->pOldTtlIdx, 0); + if (ret < 0) { + metaError("%s, failed to open %s index since %s", pTtlMgr->logPrefix, ttlTbname, tstrerror(terrno)); + goto _out; + } + + ret = ttlMgrConvert(pTtlMgr->pOldTtlIdx, pTtlMgr->pTtlIdx, pMeta); + if (ret < 0) { + metaError("%s, failed to convert ttl index since %s", pTtlMgr->logPrefix, tstrerror(terrno)); + goto _out; + } + + ret = tdbTbDropByName(ttlTbname, meta->pEnv, meta->txn); + if (ret < 0) { + metaError("%s, failed to drop old ttl index since %s", pTtlMgr->logPrefix, tstrerror(terrno)); + goto _out; + } + + ret = ttlMgrFillCache(pTtlMgr); + if (ret < 0) { + metaError("%s, failed to fill hash since %s", pTtlMgr->logPrefix, tstrerror(terrno)); + goto _out; + } + + int64_t endNs = taosGetTimestampNs(); + metaInfo("%s, ttl mgr upgrade end, hash size: %d, time consumed: %" PRId64 " ns", pTtlMgr->logPrefix, + taosHashGetSize(pTtlMgr->pTtlCache), endNs - startNs); +_out: + tdbTbClose(pTtlMgr->pOldTtlIdx); + pTtlMgr->pOldTtlIdx = NULL; + + return ret; +} + +static void ttlMgrCleanup(STtlManger *pTtlMgr) { + taosMemoryFree(pTtlMgr->logPrefix); + taosHashCleanup(pTtlMgr->pTtlCache); + taosHashCleanup(pTtlMgr->pDirtyUids); + tdbTbClose(pTtlMgr->pTtlIdx); + taosThreadRwlockDestroy(&pTtlMgr->lock); + taosMemoryFree(pTtlMgr); +} + +static void ttlMgrBuildKey(STtlIdxKeyV1 *pTtlKey, int64_t ttlDays, int64_t changeTimeMs, tb_uid_t uid) { + if (ttlDays <= 0) return; + + pTtlKey->deleteTimeMs = changeTimeMs + ttlDays * tsTtlUnit * 1000; + pTtlKey->uid = uid; +} + +static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + STtlIdxKey *pTtlIdxKey1 = (STtlIdxKey *)pKey1; + STtlIdxKey *pTtlIdxKey2 = (STtlIdxKey *)pKey2; + + if (pTtlIdxKey1->deleteTimeSec > pTtlIdxKey2->deleteTimeSec) { + return 1; + } else if (pTtlIdxKey1->deleteTimeSec < pTtlIdxKey2->deleteTimeSec) { + return -1; + } + + if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) { + return 1; + } else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) { + return -1; + } + + return 0; +} + +static int ttlIdxKeyV1Cmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + STtlIdxKeyV1 *pTtlIdxKey1 = (STtlIdxKeyV1 *)pKey1; + STtlIdxKeyV1 *pTtlIdxKey2 = (STtlIdxKeyV1 *)pKey2; + + if (pTtlIdxKey1->deleteTimeMs > pTtlIdxKey2->deleteTimeMs) { + return 1; + } else if (pTtlIdxKey1->deleteTimeMs < pTtlIdxKey2->deleteTimeMs) { + return -1; + } + + if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) { + return 1; + } else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) { + return -1; + } + + return 0; +} + +static int ttlMgrFillCache(STtlManger *pTtlMgr) { + return tdbTbTraversal(pTtlMgr->pTtlIdx, pTtlMgr->pTtlCache, ttlMgrFillCacheOneEntry); +} + +static int32_t ttlMgrFillCacheOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen, void *pTtlCache) { + SHashObj *pCache = (SHashObj *)pTtlCache; + + STtlIdxKeyV1 *ttlKey = (STtlIdxKeyV1 *)pKey; + tb_uid_t uid = ttlKey->uid; + int64_t ttlDays = *(int64_t *)pVal; + int64_t changeTimeMs = ttlKey->deleteTimeMs - ttlDays * tsTtlUnit * 1000; + + STtlCacheEntry data = {.ttlDays = ttlDays, .changeTimeMs = changeTimeMs}; + + return taosHashPut(pCache, &uid, sizeof(uid), &data, sizeof(data)); +} + +static int ttlMgrConvertOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen, void *pConvertData) { + SConvertData *pData = (SConvertData *)pConvertData; + + STtlIdxKey *ttlKey = (STtlIdxKey *)pKey; + tb_uid_t uid = ttlKey->uid; + int64_t ttlDays = 0; + + int ret = metaGetTableTtlByUid(pData->pMeta, uid, &ttlDays); + if (ret < 0) { + metaError("ttlMgr convert failed to get ttl since %s", tstrerror(terrno)); + goto _out; + } + + STtlIdxKeyV1 ttlKeyV1 = {.deleteTimeMs = ttlKey->deleteTimeSec * 1000, .uid = uid}; + ret = tdbTbUpsert(pData->pNewTtlIdx, &ttlKeyV1, sizeof(ttlKeyV1), &ttlDays, sizeof(ttlDays), pData->pMeta->txn); + if (ret < 0) { + metaError("ttlMgr convert failed to upsert since %s", tstrerror(terrno)); + goto _out; + } + + ret = 0; +_out: + return ret; +} + +static int ttlMgrConvert(TTB *pOldTtlIdx, TTB *pNewTtlIdx, void *pMeta) { + SMeta *meta = pMeta; + + metaInfo("ttlMgr convert ttl start."); + + SConvertData cvData = {.pNewTtlIdx = pNewTtlIdx, .pMeta = meta}; + + int ret = tdbTbTraversal(pOldTtlIdx, &cvData, ttlMgrConvertOneEntry); + if (ret < 0) { + metaError("failed to convert ttl since %s", tstrerror(terrno)); + } + + metaInfo("ttlMgr convert ttl end."); + return ret; +} + +int ttlMgrInsertTtl(STtlManger *pTtlMgr, const STtlUpdTtlCtx *updCtx) { + if (updCtx->ttlDays == 0) return 0; + + STtlCacheEntry cacheEntry = {.ttlDays = updCtx->ttlDays, .changeTimeMs = updCtx->changeTimeMs}; + STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_UPSERT}; + + ttlMgrWLock(pTtlMgr); + + int ret = taosHashPut(pTtlMgr->pTtlCache, &updCtx->uid, sizeof(updCtx->uid), &cacheEntry, sizeof(cacheEntry)); + if (ret < 0) { + metaError("%s, ttlMgr insert failed to update ttl cache since %s", pTtlMgr->logPrefix, tstrerror(terrno)); + goto _out; + } + + ret = taosHashPut(pTtlMgr->pDirtyUids, &updCtx->uid, sizeof(updCtx->uid), &dirtryEntry, sizeof(dirtryEntry)); + if (ret < 0) { + metaError("%s, ttlMgr insert failed to update ttl dirty uids since %s", pTtlMgr->logPrefix, tstrerror(terrno)); + goto _out; + } + + ret = 0; +_out: + ttlMgrULock(pTtlMgr); + + metaDebug("%s, ttl mgr insert ttl, uid: %" PRId64 ", ctime: %" PRId64 ", ttlDays: %" PRId64, pTtlMgr->logPrefix, + updCtx->uid, updCtx->changeTimeMs, updCtx->ttlDays); + + return ret; +} + +int ttlMgrDeleteTtl(STtlManger *pTtlMgr, const STtlDelTtlCtx *delCtx) { + if (delCtx->ttlDays == 0) return 0; + ttlMgrWLock(pTtlMgr); + + STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_DEL}; + + int ret = taosHashPut(pTtlMgr->pDirtyUids, &delCtx->uid, sizeof(delCtx->uid), &dirtryEntry, sizeof(dirtryEntry)); + if (ret < 0) { + metaError("%s, ttlMgr del failed to update ttl dirty uids since %s", pTtlMgr->logPrefix, tstrerror(terrno)); + goto _out; + } + + ret = 0; +_out: + ttlMgrULock(pTtlMgr); + + metaDebug("%s, ttl mgr delete ttl, uid: %" PRId64, pTtlMgr->logPrefix, delCtx->uid); + + return ret; +} + +int ttlMgrUpdateChangeTime(STtlManger *pTtlMgr, const STtlUpdCtimeCtx *pUpdCtimeCtx) { + ttlMgrWLock(pTtlMgr); + + int ret = 0; + + STtlCacheEntry *oldData = taosHashGet(pTtlMgr->pTtlCache, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid)); + if (oldData == NULL) { + goto _out; + } + + STtlCacheEntry cacheEntry = {.ttlDays = oldData->ttlDays, .changeTimeMs = pUpdCtimeCtx->changeTimeMs}; + STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_UPSERT}; + + ret = taosHashPut(pTtlMgr->pTtlCache, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid), &cacheEntry, sizeof(cacheEntry)); + if (ret < 0) { + metaError("%s, ttlMgr update ctime failed to update ttl cache since %s", pTtlMgr->logPrefix, tstrerror(terrno)); + goto _out; + } + + ret = taosHashPut(pTtlMgr->pDirtyUids, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid), &dirtryEntry, + sizeof(dirtryEntry)); + if (ret < 0) { + metaError("%s, ttlMgr update ctime failed to update ttl dirty uids since %s", pTtlMgr->logPrefix, + tstrerror(terrno)); + goto _out; + } + + ret = 0; +_out: + ttlMgrULock(pTtlMgr); + + metaDebug("%s, ttl mgr update ctime, uid: %" PRId64 ", ctime: %" PRId64, pTtlMgr->logPrefix, pUpdCtimeCtx->uid, + pUpdCtimeCtx->changeTimeMs); + + return ret; +} + +int ttlMgrFindExpired(STtlManger *pTtlMgr, int64_t timePointMs, SArray *pTbUids) { + ttlMgrRLock(pTtlMgr); + + TBC *pCur; + int ret = tdbTbcOpen(pTtlMgr->pTtlIdx, &pCur, NULL); + if (ret < 0) { + goto _out; + } + + STtlIdxKeyV1 ttlKey = {0}; + ttlKey.deleteTimeMs = timePointMs; + ttlKey.uid = INT64_MAX; + int c = 0; + tdbTbcMoveTo(pCur, &ttlKey, sizeof(ttlKey), &c); + if (c < 0) { + tdbTbcMoveToPrev(pCur); + } + + void *pKey = NULL; + int kLen = 0; + while (1) { + ret = tdbTbcPrev(pCur, &pKey, &kLen, NULL, NULL); + if (ret < 0) { + ret = 0; + break; + } + ttlKey = *(STtlIdxKeyV1 *)pKey; + taosArrayPush(pTbUids, &ttlKey.uid); + } + + tdbFree(pKey); + tdbTbcClose(pCur); + + ret = 0; +_out: + ttlMgrULock(pTtlMgr); + return ret; +} + +int ttlMgrFlush(STtlManger *pTtlMgr, TXN *pTxn) { + ttlMgrWLock(pTtlMgr); + + metaDebug("%s, ttl mgr flush start. dirty uids:%d", pTtlMgr->logPrefix, taosHashGetSize(pTtlMgr->pDirtyUids)); + + int ret = -1; + + void *pIter = taosHashIterate(pTtlMgr->pDirtyUids, NULL); + while (pIter != NULL) { + STtlDirtyEntry *pEntry = (STtlDirtyEntry *)pIter; + tb_uid_t *pUid = taosHashGetKey(pIter, NULL); + + STtlCacheEntry *cacheEntry = taosHashGet(pTtlMgr->pTtlCache, pUid, sizeof(*pUid)); + if (cacheEntry == NULL) { + metaError("%s, ttlMgr flush failed to get ttl cache since %s, uid: %" PRId64 ", type: %d", pTtlMgr->logPrefix, + tstrerror(terrno), *pUid, pEntry->type); + continue; + } + + STtlIdxKeyV1 ttlKey; + ttlMgrBuildKey(&ttlKey, cacheEntry->ttlDays, cacheEntry->changeTimeMs, *pUid); + + if (pEntry->type == ENTRY_TYPE_UPSERT) { + ret = tdbTbUpsert(pTtlMgr->pTtlIdx, &ttlKey, sizeof(ttlKey), &cacheEntry->ttlDays, sizeof(cacheEntry->ttlDays), + pTxn); + if (ret < 0) { + metaError("%s, ttlMgr flush failed to flush ttl cache upsert since %s", pTtlMgr->logPrefix, tstrerror(terrno)); + goto _out; + } + } else if (pEntry->type == ENTRY_TYPE_DEL) { + ret = tdbTbDelete(pTtlMgr->pTtlIdx, &ttlKey, sizeof(ttlKey), pTxn); + if (ret < 0) { + metaError("%s, ttlMgr flush failed to flush ttl cache del since %s", pTtlMgr->logPrefix, tstrerror(terrno)); + goto _out; + } + + ret = taosHashRemove(pTtlMgr->pTtlCache, pUid, sizeof(*pUid)); + if (ret < 0) { + metaError("%s, ttlMgr flush failed to delete ttl cache since %s", pTtlMgr->logPrefix, tstrerror(terrno)); + goto _out; + } + } else { + metaError("%s, ttlMgr flush failed to flush ttl cache, unknown type: %d", pTtlMgr->logPrefix, pEntry->type); + goto _out; + } + + void *pIterTmp = pIter; + pIter = taosHashIterate(pTtlMgr->pDirtyUids, pIterTmp); + taosHashRemove(pTtlMgr->pDirtyUids, pUid, sizeof(tb_uid_t)); + } + + taosHashClear(pTtlMgr->pDirtyUids); + + ret = 0; +_out: + ttlMgrULock(pTtlMgr); + + metaDebug("%s, ttl mgr flush end.", pTtlMgr->logPrefix); + + return ret; +} + +static int32_t ttlMgrRLock(STtlManger *pTtlMgr) { + int32_t ret = 0; + + metaTrace("%s, ttlMgr rlock %p", pTtlMgr->logPrefix, &pTtlMgr->lock); + + ret = taosThreadRwlockRdlock(&pTtlMgr->lock); + + return ret; +} + +static int32_t ttlMgrWLock(STtlManger *pTtlMgr) { + int32_t ret = 0; + + metaTrace("%s, ttlMgr wlock %p", pTtlMgr->logPrefix, &pTtlMgr->lock); + + ret = taosThreadRwlockWrlock(&pTtlMgr->lock); + + return ret; +} + +static int32_t ttlMgrULock(STtlManger *pTtlMgr) { + int32_t ret = 0; + + metaTrace("%s, ttlMgr ulock %p", pTtlMgr->logPrefix, &pTtlMgr->lock); + + ret = taosThreadRwlockUnlock(&pTtlMgr->lock); + + return ret; +} diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 51011ef7916a7f5b4173418a6d67186a8ff7869f..c26157f4b7ab260eda5a0244bbc9c6bc5ee812d1 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -103,18 +103,16 @@ _exit: return code; } -int32_t smaFinishCommit(SSma *pSma) { +extern int32_t tsdbCommitCommit(STsdb *tsdb); +int32_t smaFinishCommit(SSma *pSma) { int32_t code = 0; int32_t lino = 0; SVnode *pVnode = pSma->pVnode; - code = tdRSmaFSFinishCommit(pSma); - TSDB_CHECK_CODE(code, lino, _exit); - - if (VND_RSMA1(pVnode) && (code = tsdbFinishCommit(VND_RSMA1(pVnode))) < 0) { + if (VND_RSMA1(pVnode) && (code = tsdbCommitCommit(VND_RSMA1(pVnode))) < 0) { TSDB_CHECK_CODE(code, lino, _exit); } - if (VND_RSMA2(pVnode) && (code = tsdbFinishCommit(VND_RSMA2(pVnode))) < 0) { + if (VND_RSMA2(pVnode) && (code = tsdbCommitCommit(VND_RSMA2(pVnode))) < 0) { TSDB_CHECK_CODE(code, lino, _exit); } _exit: @@ -133,6 +131,7 @@ _exit: * @param isCommit * @return int32_t */ +extern int32_t tsdbPreCommit(STsdb *tsdb); static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) { int32_t code = 0; int32_t lino = 0; @@ -150,18 +149,7 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) { atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); if (isCommit) { while (atomic_val_compare_exchange_8(RSMA_COMMIT_STAT(pRSmaStat), 0, 1) != 0) { - ++nLoops; - if (nLoops > 1000) { - sched_yield(); - nLoops = 0; - } - } - - pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; - if (ASSERTS(pRSmaStat->commitAppliedVer >= -1, "commit applied version %" PRIi64 " < -1", - pRSmaStat->commitAppliedVer)) { - code = TSDB_CODE_APP_ERROR; - TSDB_CHECK_CODE(code, lino, _exit); + TD_SMA_LOOPS_CHECK(nLoops, 1000) } } // step 2: wait for all triggered fetch tasks to finish @@ -173,11 +161,7 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) { } else { smaDebug("vgId:%d, rsma commit%d, fetch tasks are not all finished yet", SMA_VID(pSma), isCommit); } - ++nLoops; - if (nLoops > 1000) { - sched_yield(); - nLoops = 0; - } + TD_SMA_LOOPS_CHECK(nLoops, 1000); } /** @@ -189,49 +173,26 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) { (void *)taosGetSelfPthreadId()); nLoops = 0; while (atomic_load_64(&pRSmaStat->nBufItems) > 0) { - ++nLoops; - if (nLoops > 1000) { - sched_yield(); - nLoops = 0; - } + TD_SMA_LOOPS_CHECK(nLoops, 1000); } if (!isCommit) goto _exit; - smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); + // code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); TSDB_CHECK_CODE(code, lino, _exit); smaInfo("vgId:%d, rsma commit, operator state committed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); -#if 0 // consuming task of qTaskInfo clone - // step 4: swap queue/qall and iQueue/iQall - // lock - taosWLockLatch(SMA_ENV_LOCK(pEnv)); - - void *pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); - - while (pIter) { - SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; - TSWAP(pInfo->iQall, pInfo->qall); - TSWAP(pInfo->iQueue, pInfo->queue); - TSWAP(pInfo->iTaskInfo[0], pInfo->taskInfo[0]); - TSWAP(pInfo->iTaskInfo[1], pInfo->taskInfo[1]); - pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter); - } - - // unlock - taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); -#endif + smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); // all rsma results are written completely STsdb *pTsdb = NULL; if ((pTsdb = VND_RSMA1(pSma->pVnode))) { - code = tsdbPrepareCommit(pTsdb); + code = tsdbPreCommit(pTsdb); TSDB_CHECK_CODE(code, lino, _exit); } if ((pTsdb = VND_RSMA2(pSma->pVnode))) { - code = tsdbPrepareCommit(pTsdb); + code = tsdbPreCommit(pTsdb); TSDB_CHECK_CODE(code, lino, _exit); } @@ -248,6 +209,7 @@ _exit: * @param pSma * @return int32_t */ +extern int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info); static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo) { int32_t code = 0; int32_t lino = 0; @@ -258,13 +220,10 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo) { goto _exit; } - code = tdRSmaFSCommit(pSma); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbCommit(VND_RSMA1(pVnode), pInfo); + code = tsdbCommitBegin(VND_RSMA1(pVnode), pInfo); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbCommit(VND_RSMA2(pVnode), pInfo); + code = tsdbCommitBegin(VND_RSMA2(pVnode), pInfo); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -310,20 +269,6 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { continue; } -#if 0 - if (pRSmaInfo->taskInfo[0]) { - if (pRSmaInfo->iTaskInfo[0]) { - SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pRSmaInfo->iTaskInfo[0]; - tdFreeRSmaInfo(pSma, pRSmaInfo, false); - pRSmaInfo->iTaskInfo[0] = NULL; - } - } else { - TSWAP(pRSmaInfo->taskInfo[0], pRSmaInfo->iTaskInfo[0]); - } - - taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter)); - smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), *pSuid); -#endif } // unlock diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 02766c8076b435e3cdac12885210f533c12e4c6e..04a254fc7a2202827a99f67f82099e4404df9d5d 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -30,7 +30,6 @@ static int32_t tdRsmaStartExecutor(const SSma *pSma); static int32_t tdRsmaStopExecutor(const SSma *pSma); static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); static void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); -static void *tdFreeTSmaStat(STSmaStat *pStat); static void tdDestroyRSmaStat(void *pRSmaStat); /** @@ -63,19 +62,15 @@ int32_t smaInit() { int32_t type = (8 == POINTER_BYTES) ? TSDB_DATA_TYPE_UBIGINT : TSDB_DATA_TYPE_UINT; smaMgmt.refHash = taosHashInit(64, taosGetDefaultHashFunction(type), true, HASH_ENTRY_LOCK); - if (!smaMgmt.refHash) { - taosCloseRef(smaMgmt.rsetId); - atomic_store_8(&smaMgmt.inited, 0); - smaError("failed to init sma tmr hanle since %s", terrstr()); - return TSDB_CODE_FAILED; - } - // init fetch timer handle smaMgmt.tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA"); - if (!smaMgmt.tmrHandle) { + + if (!smaMgmt.refHash || !smaMgmt.tmrHandle) { taosCloseRef(smaMgmt.rsetId); - taosHashCleanup(smaMgmt.refHash); - smaMgmt.refHash = NULL; + if (smaMgmt.refHash) { + taosHashCleanup(smaMgmt.refHash); + smaMgmt.refHash = NULL; + } atomic_store_8(&smaMgmt.inited, 0); smaError("failed to init sma tmr handle since %s", terrstr()); return TSDB_CODE_FAILED; @@ -143,10 +138,6 @@ static int32_t tdNewSmaEnv(SSma *pSma, int8_t smaType, SSmaEnv **ppEnv) { } static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, SSmaEnv **ppEnv) { - if (!ppEnv) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } if (!(*ppEnv)) { if (tdNewSmaEnv(pSma, smaType, ppEnv) != TSDB_CODE_SUCCESS) { @@ -196,10 +187,6 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS int32_t code = 0; int32_t lino = 0; - if (ASSERTS(pSmaStat != NULL, "pSmaStat is NULL")) { - terrno = TSDB_CODE_RSMA_INVALID_ENV; - TSDB_CHECK_CODE(code, lino, _exit); - } if (*pSmaStat) { // no lock return code; // success, return directly @@ -255,15 +242,13 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS taosInitRWLatch(RSMA_FS_LOCK(pRSmaStat)); } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { // TODO - } else { - ASSERTS(0, "unknown smaType:%" PRIi8, smaType); - code = TSDB_CODE_APP_ERROR; - TSDB_CHECK_CODE(code, lino, _exit); } } _exit: if (code) { smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); + } else { + smaDebug("vgId:%d, %s succeed, type:%" PRIi8, SMA_VID(pSma), __func__, smaType); } return code; } @@ -277,12 +262,6 @@ static void tdDestroyTSmaStat(STSmaStat *pStat) { } } -static void *tdFreeTSmaStat(STSmaStat *pStat) { - tdDestroyTSmaStat(pStat); - taosMemoryFreeClear(pStat); - return NULL; -} - static void tdDestroyRSmaStat(void *pRSmaStat) { if (pRSmaStat) { SRSmaStat *pStat = (SRSmaStat *)pRSmaStat; @@ -300,11 +279,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { } else { smaDebug("vgId:%d, rsma fetch tasks are not all finished yet", SMA_VID(pSma)); } - ++nLoops; - if (nLoops > 1000) { - sched_yield(); - nLoops = 0; - } + TD_SMA_LOOPS_CHECK(nLoops, 1000); } // step 3: @@ -313,10 +288,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { // step 4: destroy the rsma info and associated fetch tasks taosHashCleanup(RSMA_INFO_HASH(pStat)); - // step 5: - tdRSmaFSClose(RSMA_FS(pStat)); - - // step 6: free pStat + // step 5: free pStat tsem_destroy(&(pStat->notEmpty)); taosMemoryFreeClear(pStat); } @@ -354,10 +326,7 @@ static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { smaDebug("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " succeed", vid, refId, smaMgmt.rsetId); } } else { - ASSERTS(0, "unknown smaType:%" PRIi8, smaType); - terrno = TSDB_CODE_APP_ERROR; - smaError("%s failed at line %d since %s", __func__, __LINE__, terrstr()); - return -1; + smaError("%s failed at line %d since Unknown type", __func__, __LINE__); } } return 0; @@ -375,11 +344,6 @@ int32_t tdLockSma(SSma *pSma) { } int32_t tdUnLockSma(SSma *pSma) { - if (ASSERTS(SMA_LOCKED(pSma), "pSma %p is not locked:%d", pSma, pSma->locked)) { - terrno = TSDB_CODE_APP_ERROR; - smaError("vgId:%d, failed to unlock since %s", SMA_VID(pSma), tstrerror(terrno)); - return -1; - } pSma->locked = false; int code = taosThreadMutexUnlock(&pSma->mutex); diff --git a/source/dnode/vnode/src/sma/smaFS.c b/source/dnode/vnode/src/sma/smaFS.c deleted file mode 100644 index 1211ef940585a44effb47aa7f09ee69d36abf5e1..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/sma/smaFS.c +++ /dev/null @@ -1,649 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "sma.h" - -// ================================================================================================= - -// static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **output); -static int32_t tdQTaskInfCmprFn1(const void *p1, const void *p2); - -static FORCE_INLINE int32_t tPutQTaskF(uint8_t *p, SQTaskFile *pFile) { - int32_t n = 0; - - n += tPutI8(p ? p + n : p, pFile->level); - n += tPutI64v(p ? p + n : p, pFile->size); - n += tPutI64v(p ? p + n : p, pFile->suid); - n += tPutI64v(p ? p + n : p, pFile->version); - n += tPutI64v(p ? p + n : p, pFile->mtime); - - return n; -} - -static int32_t tdRSmaFSToBinary(uint8_t *p, SRSmaFS *pFS) { - int32_t n = 0; - uint32_t size = taosArrayGetSize(pFS->aQTaskInf); - - // version - n += tPutI8(p ? p + n : p, 0); - - // SArray - n += tPutU32v(p ? p + n : p, size); - for (uint32_t i = 0; i < size; ++i) { - n += tPutQTaskF(p ? p + n : p, taosArrayGet(pFS->aQTaskInf, i)); - } - - return n; -} - -int32_t tdRSmaGetQTaskF(uint8_t *p, SQTaskFile *pFile) { - int32_t n = 0; - - n += tGetI8(p + n, &pFile->level); - n += tGetI64v(p + n, &pFile->size); - n += tGetI64v(p + n, &pFile->suid); - n += tGetI64v(p + n, &pFile->version); - n += tGetI64v(p + n, &pFile->mtime); - - return n; -} - -static int32_t tsdbBinaryToFS(uint8_t *pData, int64_t nData, SRSmaFS *pFS) { - int32_t code = 0; - int32_t n = 0; - int8_t version = 0; - - // version - n += tGetI8(pData + n, &version); - - // SArray - taosArrayClear(pFS->aQTaskInf); - uint32_t size = 0; - n += tGetU32v(pData + n, &size); - for (uint32_t i = 0; i < size; ++i) { - SQTaskFile qTaskF = {0}; - - int32_t nt = tdRSmaGetQTaskF(pData + n, &qTaskF); - if (nt < 0) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } - - n += nt; - if (taosArrayPush(pFS->aQTaskInf, &qTaskF) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - - if (ASSERTS(n + sizeof(TSCKSUM) == nData, "n:%d + sizeof(TSCKSUM):%d != nData:%d", n, (int32_t)sizeof(TSCKSUM), - nData)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } - -_exit: - return code; -} - -static int32_t tdRSmaSaveFSToFile(SRSmaFS *pFS, const char *fname) { - int32_t code = 0; - int32_t lino = 0; - - // encode to binary - int32_t size = tdRSmaFSToBinary(NULL, pFS) + sizeof(TSCKSUM); - uint8_t *pData = taosMemoryMalloc(size); - if (pData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - tdRSmaFSToBinary(pData, pFS); - taosCalcChecksumAppend(0, pData, size); - - // save to file - TdFilePtr pFD = taosCreateFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t n = taosWriteFile(pFD, pData, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (taosFsyncFile(pFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - taosCloseFile(&pFD); - -_exit: - if (pData) taosMemoryFree(pData); - if (code) { - smaError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); - } - return code; -} - -static int32_t tdRSmaFSCreate(SRSmaFS *pFS, int32_t size) { - int32_t code = 0; - - pFS->aQTaskInf = taosArrayInit(size, sizeof(SQTaskFile)); - if (pFS->aQTaskInf == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - -_exit: - return code; -} - -static void tdRSmaGetCurrentFName(SSma *pSma, char *current, char *current_t) { - SVnode *pVnode = pSma->pVnode; - if (pVnode->pTfs) { - if (current) { - snprintf(current, TSDB_FILENAME_LEN - 1, "%s%svnode%svnode%d%srsma%sPRESENT", tfsGetPrimaryPath(pVnode->pTfs), - TD_DIRSEP, TD_DIRSEP, TD_VID(pVnode), TD_DIRSEP, TD_DIRSEP); - } - if (current_t) { - snprintf(current_t, TSDB_FILENAME_LEN - 1, "%s%svnode%svnode%d%srsma%sPRESENT.t", tfsGetPrimaryPath(pVnode->pTfs), - TD_DIRSEP, TD_DIRSEP, TD_VID(pVnode), TD_DIRSEP, TD_DIRSEP); - } - } else { -#if 0 - if (current) { - snprintf(current, TSDB_FILENAME_LEN - 1, "%s%sPRESENT", pTsdb->path, TD_DIRSEP); - } - if (current_t) { - snprintf(current_t, TSDB_FILENAME_LEN - 1, "%s%sPRESENT.t", pTsdb->path, TD_DIRSEP); - } -#endif - } -} - -static int32_t tdRSmaLoadFSFromFile(const char *fname, SRSmaFS *pFS) { - int32_t code = 0; - int32_t lino = 0; - uint8_t *pData = NULL; - - // load binary - TdFilePtr pFD = taosOpenFile(fname, TD_FILE_READ); - if (pFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t size; - if (taosFStatFile(pFD, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - pData = taosMemoryMalloc(size); - if (pData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (taosReadFile(pFD, pData, size) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (!taosCheckChecksumWhole(pData, size)) { - code = TSDB_CODE_FILE_CORRUPTED; - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - taosCloseFile(&pFD); - - // decode binary - code = tsdbBinaryToFS(pData, size, pFS); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (pData) taosMemoryFree(pData); - if (code) { - smaError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); - } - return code; -} - -static int32_t tdQTaskInfCmprFn1(const void *p1, const void *p2) { - const SQTaskFile *q1 = (const SQTaskFile *)p1; - const SQTaskFile *q2 = (const SQTaskFile *)p2; - - if (q1->suid < q2->suid) { - return -1; - } else if (q1->suid > q2->suid) { - return 1; - } - - if (q1->level < q2->level) { - return -1; - } else if (q1->level > q2->level) { - return 1; - } - - if (q1->version < q2->version) { - return -2; - } else if (q1->version > q2->version) { - return 1; - } - - return 0; -} - -static int32_t tdRSmaFSApplyChange(SSma *pSma, SRSmaFS *pFSNew) { - int32_t code = 0; - int32_t lino = 0; - int32_t nRef = 0; - SVnode *pVnode = pSma->pVnode; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - SRSmaFS *pFSOld = RSMA_FS(pStat); - int64_t version = pStat->commitAppliedVer; - char fname[TSDB_FILENAME_LEN] = {0}; - - // SQTaskFile - int32_t nNew = taosArrayGetSize(pFSNew->aQTaskInf); - int32_t iNew = 0; - while (iNew < nNew) { - SQTaskFile *pQTaskFNew = TARRAY_GET_ELEM(pFSNew->aQTaskInf, iNew++); - - int32_t idx = taosArraySearchIdx(pFSOld->aQTaskInf, pQTaskFNew, tdQTaskInfCmprFn1, TD_GE); - - if (idx < 0) { - idx = taosArrayGetSize(pFSOld->aQTaskInf); - pQTaskFNew->nRef = 1; - } else { - SQTaskFile *pTaskF = TARRAY_GET_ELEM(pFSOld->aQTaskInf, idx); - int32_t c1 = tdQTaskInfCmprFn1(pQTaskFNew, pTaskF); - if (c1 == 0) { - // utilize the item in pFSOld->qQTaskInf, instead of pFSNew - continue; - } else if (c1 < 0) { - // NOTHING TODO - } else { - code = TSDB_CODE_RSMA_FS_UPDATE; - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - if (taosArrayInsert(pFSOld->aQTaskInf, idx, pQTaskFNew) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - // remove previous version - while (--idx >= 0) { - SQTaskFile *preTaskF = TARRAY_GET_ELEM(pFSOld->aQTaskInf, idx); - int32_t c2 = tdQTaskInfCmprFn1(preTaskF, pQTaskFNew); - if (c2 == 0) { - code = TSDB_CODE_RSMA_FS_UPDATE; - TSDB_CHECK_CODE(code, lino, _exit); - } else if (c2 != -2) { - break; - } - - nRef = atomic_sub_fetch_32(&preTaskF->nRef, 1); - if (nRef <= 0) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), preTaskF->suid, preTaskF->level, preTaskF->version, - tfsGetPrimaryPath(pVnode->pTfs), fname); - (void)taosRemoveFile(fname); - taosArrayRemove(pFSOld->aQTaskInf, idx); - } - } - } - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tdRSmaFSScanAndTryFix(SSma *pSma) { - int32_t code = 0; -#if 0 - int32_t lino = 0; - SVnode *pVnode = pSma->pVnode; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - SRSmaFS *pFS = RSMA_FS(pStat); - char fname[TSDB_FILENAME_LEN] = {0}; - char fnameVer[TSDB_FILENAME_LEN] = {0}; - - // SArray - int32_t size = taosArrayGetSize(pFS->aQTaskInf); - for (int32_t i = 0; i < size; ++i) { - SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, i); - - // main.tdb ========= - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, pTaskF->version, - tfsGetPrimaryPath(pVnode->pTfs), fnameVer); - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, -1, tfsGetPrimaryPath(pVnode->pTfs), fname); - - if (taosCheckExistFile(fnameVer)) { - if (taosRenameFile(fnameVer, fname) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - smaDebug("vgId:%d, %s:%d succeed to to rename %s to %s", TD_VID(pVnode), __func__, lino, fnameVer, fname); - } else if (taosCheckExistFile(fname)) { - if (taosRemoveFile(fname) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - smaDebug("vgId:%d, %s:%d succeed to to remove %s", TD_VID(pVnode), __func__, lino, fname); - } - } - - { - // remove those invalid files (todo) - // main.tdb-journal.5 // TDB should handle its clear for kill -9 - } - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); - } -#endif - return code; -} - -// EXPOSED APIS ==================================================================================== - -int32_t tdRSmaFSOpen(SSma *pSma, int64_t version, int8_t rollback) { - int32_t code = 0; - int32_t lino = 0; - SVnode *pVnode = pSma->pVnode; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - - // open handle - code = tdRSmaFSCreate(RSMA_FS(pStat), 0); - TSDB_CHECK_CODE(code, lino, _exit); - - // open impl - char current[TSDB_FILENAME_LEN] = {0}; - char current_t[TSDB_FILENAME_LEN] = {0}; - tdRSmaGetCurrentFName(pSma, current, current_t); - - if (taosCheckExistFile(current)) { - code = tdRSmaLoadFSFromFile(current, RSMA_FS(pStat)); - TSDB_CHECK_CODE(code, lino, _exit); - - if (taosCheckExistFile(current_t)) { - if (rollback) { - code = tdRSmaFSRollback(pSma); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - code = tdRSmaFSCommit(pSma); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - } else { - // 1st time open with empty current/qTaskInfoFile - code = tdRSmaSaveFSToFile(RSMA_FS(pStat), current); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // scan and try fix(remove main.db/main.db.xxx and use the one with version) - code = tdRSmaFSScanAndTryFix(pSma); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -void tdRSmaFSClose(SRSmaFS *pFS) { pFS->aQTaskInf = taosArrayDestroy(pFS->aQTaskInf); } - -int32_t tdRSmaFSPrepareCommit(SSma *pSma, SRSmaFS *pFSNew) { - int32_t code = 0; - int32_t lino = 0; - char tfname[TSDB_FILENAME_LEN]; - - tdRSmaGetCurrentFName(pSma, NULL, tfname); - - // generate PRESENT.t - code = tdRSmaSaveFSToFile(pFSNew, tfname); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tdRSmaFSCommit(SSma *pSma) { - int32_t code = 0; - int32_t lino = 0; - SRSmaFS fs = {0}; - - char current[TSDB_FILENAME_LEN] = {0}; - char current_t[TSDB_FILENAME_LEN] = {0}; - tdRSmaGetCurrentFName(pSma, current, current_t); - - if (!taosCheckExistFile(current_t)) { - goto _exit; - } - - // rename the file - if (taosRenameFile(current_t, current) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // load the new FS - code = tdRSmaFSCreate(&fs, 1); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tdRSmaLoadFSFromFile(current, &fs); - TSDB_CHECK_CODE(code, lino, _exit); - - // apply file change - code = tdRSmaFSApplyChange(pSma, &fs); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - tdRSmaFSClose(&fs); - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tdRSmaFSFinishCommit(SSma *pSma) { - int32_t code = 0; - int32_t lino = 0; - SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); - - taosWLockLatch(RSMA_FS_LOCK(pStat)); - code = tdRSmaFSCommit(pSma); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); - } else { - smaInfo("vgId:%d, rsmaFS finish commit", SMA_VID(pSma)); - } - return code; -} - -int32_t tdRSmaFSRollback(SSma *pSma) { - int32_t code = 0; - int32_t lino = 0; - - char current_t[TSDB_FILENAME_LEN] = {0}; - tdRSmaGetCurrentFName(pSma, NULL, current_t); - (void)taosRemoveFile(current_t); - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(errno)); - } - return code; -} - -int32_t tdRSmaFSUpsertQTaskFile(SSma *pSma, SRSmaFS *pFS, SQTaskFile *qTaskFile, int32_t nSize) { - int32_t code = 0; - - for (int32_t i = 0; i < nSize; ++i) { - SQTaskFile *qTaskF = qTaskFile + i; - - int32_t idx = taosArraySearchIdx(pFS->aQTaskInf, qTaskF, tdQTaskInfCmprFn1, TD_GE); - - if (idx < 0) { - idx = taosArrayGetSize(pFS->aQTaskInf); - } else { - SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, idx); - int32_t c = tdQTaskInfCmprFn1(pTaskF, qTaskF); - if (c == 0) { - if (pTaskF->size != qTaskF->size) { - code = TSDB_CODE_RSMA_FS_UPDATE; - smaError("vgId:%d, %s failed at line %d since %s, level:%" PRIi8 ", suid:%" PRIi64 ", version:%" PRIi64 - ", size:%" PRIi64 " != %" PRIi64, - SMA_VID(pSma), __func__, __LINE__, tstrerror(code), pTaskF->level, pTaskF->suid, pTaskF->version, - pTaskF->size, qTaskF->size); - goto _exit; - } - continue; - } - } - - if (!taosArrayInsert(pFS->aQTaskInf, idx, qTaskF)) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - -_exit: - return code; -} - -int32_t tdRSmaFSRef(SSma *pSma, SRSmaFS *pFS) { - int32_t code = 0; - int32_t lino = 0; - int32_t nRef = 0; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - SRSmaFS *qFS = RSMA_FS(pStat); - int32_t size = taosArrayGetSize(qFS->aQTaskInf); - - pFS->aQTaskInf = taosArrayInit_s(sizeof(SQTaskFile), size); - if (pFS->aQTaskInf == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - for (int32_t i = 0; i < size; ++i) { - SQTaskFile *qTaskF = (SQTaskFile *)taosArrayGet(qFS->aQTaskInf, i); - nRef = atomic_fetch_add_32(&qTaskF->nRef, 1); - if (nRef <= 0) { - code = TSDB_CODE_RSMA_FS_REF; - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - memcpy(pFS->aQTaskInf->pData, qFS->aQTaskInf->pData, size * sizeof(SQTaskFile)); - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s, nRef %d", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code), - nRef); - } - return code; -} - -void tdRSmaFSUnRef(SSma *pSma, SRSmaFS *pFS) { - int32_t nRef = 0; - char fname[TSDB_FILENAME_LEN]; - SVnode *pVnode = pSma->pVnode; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - int32_t size = taosArrayGetSize(pFS->aQTaskInf); - - for (int32_t i = 0; i < size; ++i) { - SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, i); - - nRef = atomic_sub_fetch_32(&pTaskF->nRef, 1); - if (nRef == 0) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, pTaskF->version, - tfsGetPrimaryPath(pVnode->pTfs), fname); - if (taosRemoveFile(fname) < 0) { - smaWarn("vgId:%d, failed to remove %s since %s", TD_VID(pVnode), fname, tstrerror(TAOS_SYSTEM_ERROR(errno))); - } else { - smaDebug("vgId:%d, success to remove %s", TD_VID(pVnode), fname); - } - } else if (nRef < 0) { - smaWarn("vgId:%d, abnormal unref %s since %s", TD_VID(pVnode), fname, tstrerror(TSDB_CODE_RSMA_FS_REF)); - } - } - - taosArrayDestroy(pFS->aQTaskInf); -} - -int32_t tdRSmaFSTakeSnapshot(SSma *pSma, SRSmaFS *pFS) { - int32_t code = 0; - int32_t lino = 0; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - - taosRLockLatch(RSMA_FS_LOCK(pStat)); - code = tdRSmaFSRef(pSma, pFS); - TSDB_CHECK_CODE(code, lino, _exit); -_exit: - taosRUnLockLatch(RSMA_FS_LOCK(pStat)); - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tdRSmaFSCopy(SSma *pSma, SRSmaFS *pFS) { - int32_t code = 0; - int32_t lino = 0; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - SRSmaFS *qFS = RSMA_FS(pStat); - int32_t size = taosArrayGetSize(qFS->aQTaskInf); - - code = tdRSmaFSCreate(pFS, size); - TSDB_CHECK_CODE(code, lino, _exit); - taosArrayAddBatch(pFS->aQTaskInf, qFS->aQTaskInf->pData, size); - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 00000cb12914d7a0fbda6b2218c09c1726e51e17..4dc3e45ffe93ce391317d2f6b2cf3fff13a9c874 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -101,10 +101,6 @@ int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int ty terrno = 0; pKeepCfg->precision = pCfg->precision; switch (type) { - case TSDB_TYPE_TSMA: - ASSERTS(0, "undefined smaType:%d", (int32_t)type); - terrno = TSDB_CODE_APP_ERROR; - break; case TSDB_TYPE_RSMA_L0: SMA_SET_KEEP_CFG(pVnode, 0); break; @@ -115,7 +111,6 @@ int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int ty SMA_SET_KEEP_CFG(pVnode, 2); break; default: - ASSERTS(0, "unknown smaType:%d", (int32_t)type); terrno = TSDB_CODE_APP_ERROR; break; } @@ -189,8 +184,7 @@ int32_t smaClose(SSma *pSma) { */ int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback) { if (!VND_IS_RSMA(pSma->pVnode)) { - terrno = TSDB_CODE_RSMA_INVALID_ENV; - return TSDB_CODE_FAILED; + return TSDB_CODE_RSMA_INVALID_ENV; } return tdRSmaProcessRestoreImpl(pSma, type, committedVer, rollback); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ccc00ce25e1ad03543a6332324b57c480ceb95c4..1e7de3c5263049a02984b47d171104bf43ba496c 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -14,6 +14,7 @@ */ #include "sma.h" +#include "tq.h" #define RSMA_QTASKEXEC_SMOOTH_SIZE (100) // cnt #define RSMA_SUBMIT_BATCH_SIZE (1024) // cnt @@ -30,6 +31,8 @@ SSmaMgmt smaMgmt = { typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; +extern int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now); + static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static void tdUidStoreDestory(STbUidStore *pStore); static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, bool isAdd); @@ -44,7 +47,6 @@ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo); static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid); static void tdRSmaFetchTrigger(void *param, void *tmrId); -static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo); static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables); static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int8_t type, int64_t qTaskFileVer); @@ -64,10 +66,7 @@ static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t l if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { smaDebug("vgId:%d, free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); qDestroyTask(otaskHandle); - } else { - smaDebug("vgId:%d, not free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); } - // TODO: clear files related to qTaskInfo? } /** @@ -95,16 +94,6 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { if (isDeepFree && pInfo->taskInfo[i]) { tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1); - } else { - smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma), - pInfo->suid, i + 1); - } - - if (pInfo->iTaskInfo[i]) { - tdRSmaQTaskInfoFree(&pInfo->iTaskInfo[i], SMA_VID(pSma), i + 1); - } else { - smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty iTaskInfo", - SMA_VID(pSma), pInfo->suid, i + 1); } } if (isDeepFree) { @@ -112,14 +101,14 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { } if (isDeepFree) { - if (pInfo->queue) taosCloseQueue(pInfo->queue); - if (pInfo->qall) taosFreeQall(pInfo->qall); - if (pInfo->iQueue) taosCloseQueue(pInfo->iQueue); - if (pInfo->iQall) taosFreeQall(pInfo->iQall); - pInfo->queue = NULL; - pInfo->qall = NULL; - pInfo->iQueue = NULL; - pInfo->iQall = NULL; + if (pInfo->queue) { + taosCloseQueue(pInfo->queue); + pInfo->queue = NULL; + } + if (pInfo->qall) { + taosFreeQall(pInfo->qall); + pInfo->qall = NULL; + } } taosMemoryFree(pInfo); @@ -129,11 +118,6 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { } static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) { - if (ASSERTS(*pStore == NULL, "*pStore:%p != NULL", *pStore)) { - terrno = TSDB_CODE_APP_ERROR; - return TSDB_CODE_FAILED; - } - *pStore = taosMemoryCalloc(1, sizeof(STbUidStore)); if (*pStore == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -260,29 +244,30 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat void *pStreamState = NULL; // set the backend of stream state - tdRSmaQTaskInfoGetFullPathEx(TD_VID(pVnode), pRSmaInfo->suid, idx + 1, tfsGetPrimaryPath(pVnode->pTfs), taskInfDir); + tdRSmaQTaskInfoGetFullPath(pVnode, pRSmaInfo->suid, idx + 1, pVnode->pTfs, taskInfDir); + if (!taosCheckExistFile(taskInfDir)) { char *s = taosStrdup(taskInfDir); - if (taosMulMkDir(taosDirName(s)) != 0) { + if (taosMulMkDir(s) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); taosMemoryFree(s); return TSDB_CODE_FAILED; } taosMemoryFree(s); } - pStreamState = streamStateOpen(taskInfDir, NULL, true, -1, -1); + + SStreamTask task = {.id.taskId = 0, .id.streamId = 0}; // TODO: assign value + task.pMeta = pVnode->pTq->pStreamMeta; + pStreamState = streamStateOpen(taskInfDir, &task, true, -1, -1); if (!pStreamState) { terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN; return TSDB_CODE_FAILED; } - SReadHandle handle = { - .meta = pVnode->pMeta, - .vnode = pVnode, - .initTqReader = 1, - .pStateBackend = pStreamState, - }; - pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode)); + SReadHandle handle = {.vnode = pVnode, .initTqReader = 1, .pStateBackend = pStreamState}; + initStorageAPI(&handle.api); + + pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode), 0); if (!pRSmaInfo->taskInfo[idx]) { terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; return TSDB_CODE_FAILED; @@ -303,11 +288,6 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat pItem->level = idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2; - if (ASSERTS(pItem->level > 0, "pItem level:%" PRIi8 " should > 0", pItem->level)) { - terrno = TSDB_CODE_APP_ERROR; - return TSDB_CODE_FAILED; - } - SRSmaRef rsmaRef = {.refId = pStat->refId, .suid = pRSmaInfo->suid}; taosHashPut(smaMgmt.refHash, &pItem, POINTER_BYTES, &rsmaRef, sizeof(rsmaRef)); @@ -369,25 +349,10 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con pRSmaInfo->pTSchema = pTSchema; pRSmaInfo->suid = suid; T_REF_INIT_VAL(pRSmaInfo, 1); - if (!(pRSmaInfo->queue = taosOpenQueue())) { - goto _err; - } - - if (!(pRSmaInfo->qall = taosAllocateQall())) { - goto _err; - } - if (!(pRSmaInfo->iQueue = taosOpenQueue())) { - goto _err; - } - if (!(pRSmaInfo->iQall = taosAllocateQall())) { - goto _err; - } - - if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) { - goto _err; - } - if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 1) < 0) { + if (!(pRSmaInfo->queue = taosOpenQueue()) || !(pRSmaInfo->qall = taosAllocateQall()) || + tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0 || + tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 1) < 0) { goto _err; } @@ -565,15 +530,12 @@ void *tdUidStoreFree(STbUidStore *pStore) { * @return int32_t */ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { - if (!pReq) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - SSubmitReq2 *pSubmitReq = (SSubmitReq2 *)pReq; - // spin lock for race condition during insert data - if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { - return TSDB_CODE_FAILED; + if (pReq) { + SSubmitReq2 *pSubmitReq = (SSubmitReq2 *)pReq; + // spin lock for race condition during insert data + if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { + return TSDB_CODE_FAILED; + } } return TSDB_CODE_SUCCESS; @@ -595,7 +557,6 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq2 *pMsg, STbUidStore *pStore) { return 0; } -#if 0 /** * @brief retention of rsma1/rsma2 * @@ -611,56 +572,39 @@ int32_t smaDoRetention(SSma *pSma, int64_t now) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { - code = tsdbDoRetention(pSma->pRSmaTsdb[i], now); - if (code) goto _end; + // code = tsdbDoRetention(pSma->pRSmaTsdb[i], now); + // if (code) goto _end; } } _end: return code; } -#endif - -static void tdBlockDataDestroy(SArray *pBlockArr) { - for (int32_t i = 0; i < taosArrayGetSize(pBlockArr); ++i) { - blockDataDestroy(taosArrayGetP(pBlockArr, i)); - } - taosArrayDestroy(pBlockArr); -} static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid) { + int32_t code = 0; + int32_t lino = 0; + SSDataBlock *output = NULL; + SArray *pResList = taosArrayInit(1, POINTER_BYTES); if (pResList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } while (1) { uint64_t ts; bool hasMore = false; - int32_t code = qExecTaskOpt(taskInfo, pResList, &ts, &hasMore, NULL); - if (code < 0) { - if (code == TSDB_CODE_QRY_IN_EXEC) { - break; - } else { - smaError("vgId:%d, qExecTask for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, - pItem->level, terrstr(code)); - goto _err; - } + code = qExecTaskOpt(taskInfo, pResList, &ts, &hasMore, NULL); + if (code == TSDB_CODE_QRY_IN_EXEC) { + code = 0; + break; } + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayGetSize(pResList) == 0) { - if (terrno == 0) { - // smaDebug("vgId:%d, no rsma level %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level); - } else { - smaDebug("vgId:%d, no rsma level %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, terrstr()); - goto _err; - } - break; - } else { - smaDebug("vgId:%d, rsma level %" PRIi8 " data fetched", SMA_VID(pSma), pItem->level); } #if 0 char flag[10] = {0}; @@ -668,28 +612,24 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma blockDebugShowDataBlocks(pResList, flag); #endif for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) { - SSDataBlock *output = taosArrayGetP(pResList, i); - smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%" PRId64, output->info.id.uid, - output->info.id.groupId, output->info.rows); + output = taosArrayGetP(pResList, i); + smaDebug("vgId:%d, result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%" PRIi64, SMA_VID(pSma), + output->info.id.uid, output->info.id.groupId, output->info.rows); STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); SSubmitReq2 *pReq = NULL; // TODO: the schema update should be handled later(TD-17965) if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) { - smaError("vgId:%d, build submit req for rsma table suid:%" PRIu64 ", uid:%" PRIu64 ", level %" PRIi8 - " failed since %s", - SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr()); - goto _err; + code = terrno ? terrno : TSDB_CODE_RSMA_RESULT; + TSDB_CHECK_CODE(code, lino, _exit); } if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) { + code = terrno ? terrno : TSDB_CODE_RSMA_RESULT; tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); - smaError("vgId:%d, process submit req for rsma suid:%" PRIu64 ", uid:%" PRIu64 " level %" PRIi8 - " failed since %s", - SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr()); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64, @@ -701,15 +641,18 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma } } } - - taosArrayDestroy(pResList); - qCleanExecTaskBlockBuf(taskInfo); - return TSDB_CODE_SUCCESS; - -_err: +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s, suid:%" PRIi64 ", level:%" PRIi8 ", uid:%" PRIi64 + ", ver:%" PRIi64, + SMA_VID(pSma), __func__, lino, tstrerror(code), suid, pItem->level, output ? output->info.id.uid : -1, + output ? output->info.version : -1); + } else { + smaDebug("vgId:%d, %s succeed, suid:%" PRIi64 ", level:%" PRIi8, SMA_VID(pSma), __func__, suid, pItem->level); + } taosArrayDestroy(pResList); qCleanExecTaskBlockBuf(taskInfo); - return TSDB_CODE_FAILED; + return code; } /** @@ -803,7 +746,8 @@ static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) { static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int32_t inputType, SRSmaInfo *pInfo, ERsmaExecType type, int8_t level) { int32_t idx = level - 1; - void *qTaskInfo = (type == RSMA_EXEC_COMMIT) ? RSMA_INFO_IQTASK(pInfo, idx) : RSMA_INFO_QTASK(pInfo, idx); + void *qTaskInfo = RSMA_INFO_QTASK(pInfo, idx); + if (!qTaskInfo) { smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, pInfo->suid); @@ -836,112 +780,6 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, return TSDB_CODE_SUCCESS; } -static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t srcTaskInfo, SRSmaParam *param, - tb_uid_t suid, int8_t idx) { - int32_t code = 0; - int32_t lino = 0; - SVnode *pVnode = pSma->pVnode; - char *pOutput = NULL; - int32_t len = 0; - - if (!srcTaskInfo) { - code = TSDB_CODE_INVALID_PTR; - smaWarn("vgId:%d, rsma clone, table %" PRIi64 ", no need since srcTaskInfo is NULL", TD_VID(pVnode), suid); - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len); - TSDB_CHECK_CODE(code, lino, _exit); - - SReadHandle handle = { - .meta = pVnode->pMeta, - .vnode = pVnode, - .initTqReader = 1, - }; - - if (ASSERTS(!dstTaskInfo, "dstTaskInfo:%p is not NULL", dstTaskInfo)) { - code = TSDB_CODE_APP_ERROR; - TSDB_CHECK_CODE(code, lino, _exit); - } - - dstTaskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode)); - if (!dstTaskInfo) { - code = TSDB_CODE_RSMA_QTASKINFO_CREATE; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = qDeserializeTaskStatus(dstTaskInfo, pOutput, len); - TSDB_CHECK_CODE(code, lino, _exit); - - smaDebug("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " succeed", TD_VID(pVnode), suid); - -_exit: - taosMemoryFreeClear(pOutput); - if (code) { - tdRSmaQTaskInfoFree(dstTaskInfo, TD_VID(pVnode), idx + 1); - smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, - terrstr()); - } - return code; -} - -/** - * @brief Clone qTaskInfo of SRSmaInfo - * - * @param pSma - * @param pInfo - * @return int32_t - */ -static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo) { - int32_t code = 0; - int32_t lino = 0; - SRSmaParam *param = NULL; - SMetaReader mr = {0}; - - if (!pInfo) { - return TSDB_CODE_SUCCESS; - } - - metaReaderInit(&mr, SMA_META(pSma), 0); - smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid); - if (metaGetTableEntryByUidCache(&mr, pInfo->suid) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (mr.me.type != TSDB_SUPER_TABLE) { - code = TSDB_CODE_RSMA_INVALID_SCHEMA; - TSDB_CHECK_CODE(code, lino, _exit); - } - if (mr.me.uid != pInfo->suid) { - code = TSDB_CODE_RSMA_INVALID_SCHEMA; - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (TABLE_IS_ROLLUP(mr.me.flags)) { - param = &mr.me.stbEntry.rsmaParam; - for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { - if (!pInfo->iTaskInfo[i]) { - continue; - } - code = tdCloneQTaskInfo(pSma, pInfo->taskInfo[i], pInfo->iTaskInfo[i], param, pInfo->suid, i); - TSDB_CHECK_CODE(code, lino, _exit); - } - smaDebug("vgId:%d, rsma clone env success for %" PRIi64, SMA_VID(pSma), pInfo->suid); - } else { - code = TSDB_CODE_RSMA_INVALID_SCHEMA; - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s, suid:%" PRIi64 ", flags:%" PRIi8 ",type:%" PRIi8 ", uid:%" PRIi64, - SMA_VID(pSma), __func__, lino, tstrerror(code), pInfo->suid, mr.me.flags, mr.me.type, mr.me.uid); - } - metaReaderClear(&mr); - return code; -} - /** * @brief During async commit, the SRSmaInfo object would be COW from iRSmaInfoHash and write lock should be applied. * @@ -976,12 +814,7 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return NULL; } - if (!pRSmaInfo->taskInfo[0]) { - if ((terrno = tdRSmaInfoClone(pSma, pRSmaInfo)) < 0) { - taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); - return NULL; - } - } + tdRefRSmaInfo(pSma, pRSmaInfo); taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); if (ASSERTS(pRSmaInfo->suid == suid, "suid:%" PRIi64 " != %" PRIi64, pRSmaInfo->suid, suid)) { @@ -1122,7 +955,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { } int64_t nRsmaTables = 0; - metaReaderInit(&mr, SMA_META(pSma), 0); + metaReaderDoInit(&mr, SMA_META(pSma), 0); if (!(uidStore.tbUids = taosArrayInit(1024, sizeof(tb_uid_t)))) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); @@ -1131,7 +964,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { for (int64_t i = 0; i < arrSize; ++i) { suid = *(tb_uid_t *)taosArrayGet(suidList, i); smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid); - if (metaGetTableEntryByUidCache(&mr, suid) < 0) { + if (metaReaderGetTableEntryByUidCache(&mr, suid) < 0) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } @@ -1193,58 +1026,43 @@ _exit: * N.B. the data would be restored from the unified WAL replay procedure */ int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer, int8_t rollback) { + int32_t code = 0; + int64_t nTables = 0; + // step 1: init env if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - - // step 2: open SRSmaFS for qTaskFiles - if (tdRSmaFSOpen(pSma, qtaskFileVer, rollback) < 0) { + code = TSDB_CODE_TDB_INIT_FAILED; goto _err; } - // step 3: iterate all stables to restore the rsma env - int64_t nTables = 0; - if (tdRSmaRestoreQTaskInfoInit(pSma, &nTables) < 0) { + // step 2: iterate all stables to restore the rsma env + if ((code = tdRSmaRestoreQTaskInfoInit(pSma, &nTables)) < 0) { goto _err; } - if (nTables <= 0) { - smaDebug("vgId:%d, no need to restore rsma task %" PRIi8 " since no tables", SMA_VID(pSma), type); - return TSDB_CODE_SUCCESS; - } - smaInfo("vgId:%d, restore rsma task %" PRIi8 " from qtaskf %" PRIi64 " succeed", SMA_VID(pSma), type, qtaskFileVer); - return TSDB_CODE_SUCCESS; _err: - smaError("vgId:%d, restore rsma task %" PRIi8 "from qtaskf %" PRIi64 " failed since %s", SMA_VID(pSma), type, - qtaskFileVer, terrstr()); - return TSDB_CODE_FAILED; -} + if (code) { + smaError("vgId:%d, restore rsma task %" PRIi8 "from qtaskf %" PRIi64 " failed since %s", SMA_VID(pSma), type, + qtaskFileVer, tstrerror(code)); + } else { + smaInfo("vgId:%d, restore rsma task %" PRIi8 " from qtaskf %" PRIi64 " succeed, nTables:%" PRIi64, SMA_VID(pSma), + type, qtaskFileVer, nTables); + } + return code; +} +#if 0 int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { - int32_t code = 0; - int32_t lino = 0; - SSma *pSma = pRSmaStat->pSma; - SVnode *pVnode = pSma->pVnode; - SArray *qTaskFArray = NULL; - int64_t version = pRSmaStat->commitAppliedVer; - TdFilePtr pOutFD = NULL; - TdFilePtr pInFD = NULL; - char fname[TSDB_FILENAME_LEN]; - char fnameVer[TSDB_FILENAME_LEN]; - SRSmaFS fs = {0}; + int32_t code = 0; + int32_t lino = 0; + SSma *pSma = pRSmaStat->pSma; + SVnode *pVnode = pSma->pVnode; + SRSmaFS fs = {0}; if (taosHashGetSize(pInfoHash) <= 0) { return TSDB_CODE_SUCCESS; } - qTaskFArray = taosArrayInit(taosHashGetSize(pInfoHash) << 1, sizeof(SQTaskFile)); - if (!qTaskFArray) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - void *infoHash = NULL; while ((infoHash = taosHashIterate(pInfoHash, infoHash))) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; @@ -1262,76 +1080,19 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { } smaDebug("vgId:%d, rsma persist, stream state commit success, table %" PRIi64 ", level %d", TD_VID(pVnode), pRSmaInfo->suid, i + 1); - - // qTaskInfo file - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pRSmaInfo->suid, i + 1, -1, tfsGetPrimaryPath(pVnode->pTfs), fname); - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pRSmaInfo->suid, i + 1, version, tfsGetPrimaryPath(pVnode->pTfs), - fnameVer); - if (taosCheckExistFile(fnameVer)) { - smaWarn("vgId:%d, rsma persist, duplicate file %s exist", TD_VID(pVnode), fnameVer); - } - - pOutFD = taosCreateFile(fnameVer, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - pInFD = taosOpenFile(fname, TD_FILE_READ); - if (pInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t size = 0; - uint32_t mtime = 0; - if (taosFStatFile(pInFD, &size, &mtime) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t offset = 0; - if (taosFSendFile(pOutFD, pInFD, &offset, size) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - smaError("vgId:%d, rsma persist, send qtaskinfo file %s to %s failed since %s", TD_VID(pVnode), fname, - fnameVer, tstrerror(code)); - TSDB_CHECK_CODE(code, lino, _exit); - } - taosCloseFile(&pOutFD); - taosCloseFile(&pInFD); - - SQTaskFile qTaskF = { - .nRef = 1, .level = i + 1, .suid = pRSmaInfo->suid, .version = version, .size = size, .mtime = mtime}; - - taosArrayPush(qTaskFArray, &qTaskF); } } } - // prepare - code = tdRSmaFSCopy(pSma, &fs); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tdRSmaFSUpsertQTaskFile(pSma, &fs, qTaskFArray->pData, taosArrayGetSize(qTaskFArray)); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tdRSmaFSPrepareCommit(pSma, &fs); - TSDB_CHECK_CODE(code, lino, _exit); - _exit: - - taosArrayDestroy(fs.aQTaskInf); - taosArrayDestroy(qTaskFArray); - if (code) { - if (pOutFD) taosCloseFile(&pOutFD); - if (pInFD) taosCloseFile(&pInFD); smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); } terrno = code; return code; } - +#endif /** * @brief trigger to get rsma result in async mode * @@ -1352,8 +1113,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { } if (!(pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaRef->refId))) { - smaDebug("rsma fetch task not start since rsma stat already destroyed, rsetId:%d refId:%" PRIi64 ")", - smaMgmt.rsetId, pRSmaRef->refId); // pRSmaRef freed in taosHashRemove + smaWarn("rsma fetch task not start since rsma stat already destroyed, rsetId:%d refId:%" PRIi64 ")", smaMgmt.rsetId, + pRSmaRef->refId); // pRSmaRef freed in taosHashRemove taosHashRemove(smaMgmt.refHash, ¶m, POINTER_BYTES); return; } diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c index c00e96a06664db0a60184fdb09e16ee0b68c3d45..e01a33936b5673464ac7d7bdb15019aabdcd1e0b 100644 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -15,9 +15,6 @@ #include "sma.h" -static int32_t rsmaSnapReadQTaskInfo(SRSmaSnapReader* pReader, uint8_t** ppData); -static int32_t rsmaSnapWriteQTaskInfo(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); - // SRSmaSnapReader ======================================== struct SRSmaSnapReader { SSma* pSma; @@ -28,11 +25,6 @@ struct SRSmaSnapReader { // for data file int8_t rsmaDataDone[TSDB_RETENTION_L2]; STsdbSnapReader* pDataReader[TSDB_RETENTION_L2]; - - // for qtaskinfo file - int8_t qTaskDone; - int32_t fsIter; - SQTaskFReader* pQTaskFReader; }; int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapReader** ppReader) { @@ -62,22 +54,6 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapRead } } - // open qtaskinfo - taosRLockLatch(RSMA_FS_LOCK(pStat)); - code = tdRSmaFSRef(pSma, &pReader->fs); - taosRUnLockLatch(RSMA_FS_LOCK(pStat)); - TSDB_CHECK_CODE(code, lino, _exit); - - if (taosArrayGetSize(pReader->fs.aQTaskInf) > 0) { - pReader->pQTaskFReader = taosMemoryCalloc(1, sizeof(SQTaskFReader)); - if (!pReader->pQTaskFReader) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - pReader->pQTaskFReader->pSma = pSma; - pReader->pQTaskFReader->version = pReader->ever; - } - *ppReader = pReader; _exit: if (code) { @@ -88,114 +64,6 @@ _exit: return code; } -static int32_t rsmaSnapReadQTaskInfo(SRSmaSnapReader* pReader, uint8_t** ppBuf) { - int32_t code = 0; - int32_t lino = 0; - SVnode* pVnode = pReader->pSma->pVnode; - SQTaskFReader* qReader = pReader->pQTaskFReader; - SRSmaFS* pFS = &pReader->fs; - int64_t n = 0; - uint8_t* pBuf = NULL; - int64_t version = pReader->ever; - char fname[TSDB_FILENAME_LEN]; - - if (!qReader) { - *ppBuf = NULL; - smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, not needed since qTaskReader is NULL", TD_VID(pVnode)); - goto _exit; - } - - if (pReader->fsIter >= taosArrayGetSize(pFS->aQTaskInf)) { - *ppBuf = NULL; - smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, fsIter reach end", TD_VID(pVnode)); - goto _exit; - } - - while (pReader->fsIter < taosArrayGetSize(pFS->aQTaskInf)) { - SQTaskFile* qTaskF = taosArrayGet(pFS->aQTaskInf, pReader->fsIter++); - if (qTaskF->version != version) { - continue; - } - - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), qTaskF->suid, qTaskF->level, version, tfsGetPrimaryPath(pVnode->pTfs), - fname); - if (!taosCheckExistFile(fname)) { - smaError("vgId:%d, vnode snapshot rsma reader for qtaskinfo, table %" PRIi64 ", level %" PRIi8 - ", version %" PRIi64 " failed since %s not exist", - TD_VID(pVnode), qTaskF->suid, qTaskF->level, version, fname); - code = TSDB_CODE_RSMA_FS_SYNC; - TSDB_CHECK_CODE(code, lino, _exit); - } - - TdFilePtr fp = taosOpenFile(fname, TD_FILE_READ); - if (!fp) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - qReader->pReadH = fp; - qReader->level = qTaskF->level; - qReader->suid = qTaskF->suid; - } - - if (!qReader->pReadH) { - *ppBuf = NULL; - smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, not needed since readh is NULL", TD_VID(pVnode)); - goto _exit; - } - - int64_t size = 0; - if (taosFStatFile(qReader->pReadH, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // seek - if (taosLSeekFile(qReader->pReadH, 0, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (*ppBuf) { - *ppBuf = taosMemoryRealloc(*ppBuf, sizeof(SSnapDataHdr) + size); - } else { - *ppBuf = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); - } - if (!(*ppBuf)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - // read - n = taosReadFile(qReader->pReadH, POINTER_SHIFT(*ppBuf, sizeof(SSnapDataHdr)), size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } else if (n != size) { - code = TSDB_CODE_FILE_CORRUPTED; - TSDB_CHECK_CODE(code, lino, _exit); - } - - smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo, version:%" PRIi64 ", size:%" PRIi64, TD_VID(pVnode), version, - size); - - SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppBuf); - pHdr->type = SNAP_DATA_QTASK; - pHdr->flag = qReader->level; - pHdr->index = qReader->suid; - pHdr->size = size; - -_exit: - if (qReader) taosCloseFile(&qReader->pReadH); - - if (code) { - *ppBuf = NULL; - smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); - } else { - smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo succeed", TD_VID(pVnode)); - } - return code; -} - int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; int32_t lino = 0; @@ -223,18 +91,6 @@ int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { } } - // read qtaskinfo file - if (!pReader->qTaskDone) { - smaInfo("vgId:%d, vnode snapshot rsma qtaskinfo not done", SMA_VID(pReader->pSma)); - code = rsmaSnapReadQTaskInfo(pReader, ppData); - TSDB_CHECK_CODE(code, lino, _exit); - if (*ppData) { - goto _exit; - } else { - pReader->qTaskDone = 1; - } - } - _exit: if (code) { smaError("vgId:%d, vnode snapshot rsma read failed since %s", SMA_VID(pReader->pSma), tstrerror(code)); @@ -249,9 +105,6 @@ int32_t rsmaSnapReaderClose(SRSmaSnapReader** ppReader) { int32_t code = 0; SRSmaSnapReader* pReader = *ppReader; - tdRSmaFSUnRef(pReader->pSma, &pReader->fs); - taosMemoryFreeClear(pReader->pQTaskFReader); - for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pReader->pDataReader[i]) { tsdbSnapReaderClose(&pReader->pDataReader[i]); @@ -299,10 +152,6 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWrit } } - // qtaskinfo - code = tdRSmaFSCopy(pSma, &pWriter->fs); - TSDB_CHECK_CODE(code, lino, _exit); - // snapWriter *ppWriter = pWriter; _exit: @@ -316,22 +165,6 @@ _exit: return code; } -int32_t rsmaSnapWriterPrepareClose(SRSmaSnapWriter* pWriter) { - int32_t code = 0; - int32_t lino = 0; - - if (pWriter) { - code = tdRSmaFSPrepareCommit(pWriter->pSma, &pWriter->fs); - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pWriter->pSma), __func__, lino, tstrerror(code)); - } - return code; -} - int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) { int32_t code = 0; int32_t lino = 0; @@ -340,7 +173,6 @@ int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) { SSmaEnv* pEnv = NULL; SRSmaStat* pStat = NULL; SRSmaSnapWriter* pWriter = *ppWriter; - const char* primaryPath = NULL; char fname[TSDB_FILENAME_LEN] = {0}; char fnameVer[TSDB_FILENAME_LEN] = {0}; TdFilePtr pOutFD = NULL; @@ -354,7 +186,6 @@ int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) { pVnode = pSma->pVnode; pEnv = SMA_RSMA_ENV(pSma); pStat = (SRSmaStat*)SMA_ENV_STAT(pEnv); - primaryPath = tfsGetPrimaryPath(pVnode->pTfs); // rsma1/rsma2 for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { @@ -364,61 +195,6 @@ int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) { } } - // qtaskinfo - if (rollback) { - tdRSmaFSRollback(pSma); - // remove qTaskFiles - } else { - // sendFile from fname.Ver to fname - SRSmaFS* pFS = &pWriter->fs; - int32_t size = taosArrayGetSize(pFS->aQTaskInf); - for (int32_t i = 0; i < size; ++i) { - SQTaskFile* pTaskF = TARRAY_GET_ELEM(pFS->aQTaskInf, i); - if (pTaskF->version == pWriter->ever) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, pTaskF->version, primaryPath, fnameVer); - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, -1, primaryPath, fname); - - pInFD = taosOpenFile(fnameVer, TD_FILE_READ); - if (pInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - pOutFD = taosCreateFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t size = 0; - if (taosFStatFile(pInFD, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t offset = 0; - if (taosFSendFile(pOutFD, pInFD, &offset, size) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - smaError("vgId:%d, vnode snapshot rsma writer, send qtaskinfo file %s to %s failed since %s", TD_VID(pVnode), - fnameVer, fname, tstrerror(code)); - TSDB_CHECK_CODE(code, lino, _exit); - } - taosCloseFile(&pOutFD); - taosCloseFile(&pInFD); - } - } - - // lock - taosWLockLatch(RSMA_FS_LOCK(pStat)); - code = tdRSmaFSCommit(pSma); - if (code) { - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); - goto _exit; - } - // unlock - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); - } - // rsma restore code = tdRSmaRestore(pWriter->pSma, RSMA_RESTORE_SYNC, pWriter->ever, rollback); TSDB_CHECK_CODE(code, lino, _exit); @@ -450,8 +226,6 @@ int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) } else if (pHdr->type == SNAP_DATA_RSMA2) { pHdr->type = SNAP_DATA_TSDB; code = tsdbSnapWrite(pWriter->pDataWriter[1], pHdr); - } else if (pHdr->type == SNAP_DATA_QTASK) { - code = rsmaSnapWriteQTaskInfo(pWriter, pData, nData); } else { code = TSDB_CODE_RSMA_FS_SYNC; } @@ -466,68 +240,3 @@ _exit: } return code; } - -static int32_t rsmaSnapWriteQTaskInfo(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - int32_t lino = 0; - SSma* pSma = pWriter->pSma; - SVnode* pVnode = pSma->pVnode; - char fname[TSDB_FILENAME_LEN]; - TdFilePtr fp = NULL; - SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; - - fname[0] = '\0'; - - if (pHdr->size != (nData - sizeof(SSnapDataHdr))) { - code = TSDB_CODE_RSMA_FS_SYNC; - TSDB_CHECK_CODE(code, lino, _exit); - } - - SQTaskFile qTaskFile = { - .nRef = 1, .level = pHdr->flag, .suid = pHdr->index, .version = pWriter->ever, .size = pHdr->size}; - - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pHdr->index, pHdr->flag, qTaskFile.version, - tfsGetPrimaryPath(pVnode->pTfs), fname); - - fp = taosCreateFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (!fp) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t contLen = taosWriteFile(fp, pHdr->data, pHdr->size); - if (contLen != pHdr->size) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - uint32_t mtime = 0; - if (taosFStatFile(fp, NULL, &mtime) != 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - qTaskFile.mtime = mtime; - } - - if (taosFsyncFile(fp) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - taosCloseFile(&fp); - - code = tdRSmaFSUpsertQTaskFile(pSma, &pWriter->fs, &qTaskFile, 1); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - if (fp) { - (void)taosRemoveFile(fname); - } - smaError("vgId:%d, %s failed at line %d since %s, file:%s", TD_VID(pVnode), __func__, lino, tstrerror(code), fname); - } else { - smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo %s succeed", TD_VID(pVnode), fname); - } - - return code; -} diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 3542ea9ffb2e767200c2ba4cf5e8aa959d931c1d..0c37008344054d9af74894cc4825c53d079e2b3e 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -29,27 +29,21 @@ int32_t tdProcessTSmaInsert(SSma *pSma, int64_t indexUid, const char *msg) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) { - smaWarn("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + smaError("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(code)); } return code; } int32_t tdProcessTSmaCreate(SSma *pSma, int64_t version, const char *msg) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = tdProcessTSmaCreateImpl(pSma, version, msg); - if ((code = tdProcessTSmaCreateImpl(pSma, version, msg)) < 0) { - smaWarn("vgId:%d, create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } return code; } int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days)) < 0) { - smaWarn("vgId:%d, get tsma days failed since %s", pCfg->vgId, tstrerror(terrno)); - } - smaDebug("vgId:%d, get tsma days %d", pCfg->vgId, *days); + int32_t code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days); + return code; } @@ -63,19 +57,22 @@ int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t * * @return int32_t */ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { + int32_t code = 0; + int32_t lino = 0; SDecoder coder = {0}; tDecoderInit(&coder, pCont, contLen); STSma tsma = {0}; if (tDecodeSVCreateTSmaReq(&coder, &tsma) < 0) { - terrno = TSDB_CODE_MSG_DECODE_ERROR; - goto _err; + code = TSDB_CODE_MSG_DECODE_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); } + STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg; int64_t sInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND); if (sInterval <= 0) { *days = pTsdbCfg->days; - return 0; + goto _exit; } int64_t records = pTsdbCfg->days * 60 / sInterval; if (records >= SMA_STORAGE_SPLIT_FACTOR) { @@ -94,11 +91,14 @@ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t c *days = pTsdbCfg->days; } } +_exit: + if (code) { + smaWarn("vgId:%d, failed at line %d to get tsma days %d since %s", pCfg->vgId, lino, *days, tstrerror(code)); + } else { + smaDebug("vgId:%d, succeed to get tsma days %d", pCfg->vgId, *days); + } tDecoderClear(&coder); - return 0; -_err: - tDecoderClear(&coder); - return -1; + return code; } /** @@ -157,6 +157,8 @@ _exit: int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *pTSchema, SSchemaWrapper *pTagSchemaWrapper, bool createTb, int64_t suid, const char *stbFullName, SBatchDeleteReq *pDeleteReq, void **ppData, int32_t *pLen) { + int32_t code = 0; + int32_t lino = 0; void *pBuf = NULL; int32_t len = 0; SSubmitReq2 *pReq = NULL; @@ -166,21 +168,14 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * int32_t sz = taosArrayGetSize(pBlocks); - if (!(tagArray = taosArrayInit(1, sizeof(STagVal)))) { - goto _end; - } - - if (!(createTbArray = taosArrayInit(sz, POINTER_BYTES))) { - goto _end; - } + tagArray = taosArrayInit(1, sizeof(STagVal)); + createTbArray = taosArrayInit(sz, POINTER_BYTES); + pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)); + pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)); - if (!(pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - - if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { - goto _end; + if(!tagArray || !createTbArray || !pReq || !pReq->aSubmitTbData) { + code = terrno == TSDB_CODE_SUCCESS ? TSDB_CODE_OUT_OF_MEMORY : terrno; + TSDB_CHECK_CODE(code, lino, _exit); } // create table req @@ -194,7 +189,8 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * } if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) { - goto _end; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); }; // don't move to the end of loop as to destroy in the end of func when error occur @@ -223,8 +219,8 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * STag *pTag = NULL; tTagNew(tagArray, 1, false, &pTag); if (pTag == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } pCreateTbReq->ctb.pTag = (uint8_t *)pTag; @@ -259,7 +255,8 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * SSubmitTbData tbData = {0}; if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow *)))) { - goto _end; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } tbData.suid = suid; tbData.uid = 0; // uid is assigned by vnode @@ -272,7 +269,8 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { taosArrayDestroy(tbData.aRowP); - goto _end; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } for (int32_t j = 0; j < rows; ++j) { @@ -298,9 +296,9 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * } } SRow *pRow = NULL; - if ((terrno = tRowBuild(pVals, (STSchema *)pTSchema, &pRow)) < 0) { + if ((code = tRowBuild(pVals, (STSchema *)pTSchema, &pRow)) < 0) { tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); - goto _end; + TSDB_CHECK_CODE(code, lino, _exit); } taosArrayPush(tbData.aRowP, &pRow); } @@ -309,25 +307,27 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * } // encode - tEncodeSize(tEncodeSubmitReq, pReq, len, terrno); - if (TSDB_CODE_SUCCESS == terrno) { + tEncodeSize(tEncodeSubmitReq, pReq, len, code); + if (TSDB_CODE_SUCCESS == code) { SEncoder encoder; len += sizeof(SSubmitReq2Msg); - pBuf = rpcMallocCont(len); - if (NULL == pBuf) { - goto _end; + if (!(pBuf = rpcMallocCont(len))) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } + ((SSubmitReq2Msg *)pBuf)->header.vgId = TD_VID(pVnode); ((SSubmitReq2Msg *)pBuf)->header.contLen = htonl(len); ((SSubmitReq2Msg *)pBuf)->version = htobe64(1); tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg)); if (tEncodeSubmitReq(&encoder, pReq) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - /*vError("failed to encode submit req since %s", terrstr());*/ + tEncoderClear(&encoder); + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } tEncoderClear(&encoder); } -_end: +_exit: taosArrayDestroy(createTbArray); taosArrayDestroy(tagArray); taosArrayDestroy(pVals); @@ -336,14 +336,52 @@ _end: taosMemoryFree(pReq); } - if (terrno != 0) { + if (code) { rpcFreeCont(pBuf); taosArrayDestroy(pDeleteReq->deleteReqs); - return TSDB_CODE_FAILED; + smaWarn("vgId:%d, failed at line %d since %s", TD_VID(pVnode), lino, tstrerror(code)); + } else { + if (ppData) *ppData = pBuf; + if (pLen) *pLen = len; + } + return code; +} + +static int32_t tsmaProcessDelReq(SSma *pSma, int64_t indexUid, SBatchDeleteReq *pDelReq) { + int32_t code = 0; + int32_t lino = 0; + + if (taosArrayGetSize(pDelReq->deleteReqs) > 0) { + int32_t len = 0; + tEncodeSize(tEncodeSBatchDeleteReq, pDelReq, len, code); + TSDB_CHECK_CODE(code, lino, _exit); + + void *pBuf = rpcMallocCont(len + sizeof(SMsgHead)); + if (!pBuf) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + } + + SEncoder encoder; + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len); + tEncodeSBatchDeleteReq(&encoder, pDelReq); + tEncoderClear(&encoder); + + ((SMsgHead *)pBuf)->vgId = TD_VID(pSma->pVnode); + + SRpcMsg delMsg = {.msgType = TDMT_VND_BATCH_DEL, .pCont = pBuf, .contLen = len + sizeof(SMsgHead)}; + code = tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &delMsg); + TSDB_CHECK_CODE(code, lino, _exit); } - if (ppData) *ppData = pBuf; - if (pLen) *pLen = len; - return TSDB_CODE_SUCCESS; + +_exit: + taosArrayDestroy(pDelReq->deleteReqs); + if (code) { + smaError("vgId:%d, failed at line %d to process delete req for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), lino, + indexUid, tstrerror(code)); + } + + return code; } /** @@ -354,23 +392,18 @@ _end: * @return int32_t */ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { + int32_t code = 0; + int32_t lino = 0; const SArray *pDataBlocks = (const SArray *)msg; - // TODO: destroy SSDataBlocks(msg) - if (!pDataBlocks) { - terrno = TSDB_CODE_TSMA_INVALID_PTR; - smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is NULL", SMA_VID(pSma)); - return TSDB_CODE_FAILED; - } if (taosArrayGetSize(pDataBlocks) <= 0) { - terrno = TSDB_CODE_TSMA_INVALID_PARA; - smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is empty", SMA_VID(pSma)); - return TSDB_CODE_FAILED; + code = TSDB_CODE_TSMA_INVALID_PARA; + TSDB_CHECK_CODE(code, lino, _exit); } if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != 0) { - terrno = TSDB_CODE_TSMA_INIT_FAILED; - return TSDB_CODE_FAILED; + code = TSDB_CODE_TSMA_INIT_FAILED; + TSDB_CHECK_CODE(code, lino, _exit); } SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); @@ -378,49 +411,45 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char STSmaStat *pTsmaStat = NULL; if (!pEnv || !(pStat = SMA_ENV_STAT(pEnv))) { - terrno = TSDB_CODE_TSMA_INVALID_ENV; - return TSDB_CODE_FAILED; + code = TSDB_CODE_TSMA_INVALID_ENV; + TSDB_CHECK_CODE(code, lino, _exit); } pTsmaStat = SMA_STAT_TSMA(pStat); if (!pTsmaStat->pTSma) { + terrno = 0; STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); if (!pTSma) { - smaError("vgId:%d, failed to get STSma while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), - indexUid, tstrerror(terrno)); - goto _err; + code = terrno ? terrno : TSDB_CODE_TSMA_INVALID_PTR; + TSDB_CHECK_CODE(code, lino, _exit); } pTsmaStat->pTSma = pTSma; pTsmaStat->pTSchema = metaGetTbTSchema(SMA_META(pSma), pTSma->dstTbUid, -1, 1); if (!pTsmaStat->pTSchema) { - smaError("vgId:%d, failed to get STSchema while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), - indexUid, tstrerror(terrno)); - goto _err; + code = terrno ? terrno : TSDB_CODE_TSMA_INVALID_PTR; + TSDB_CHECK_CODE(code, lino, _exit); } } - if (pTsmaStat->pTSma->indexUid != indexUid) { - terrno = TSDB_CODE_APP_ERROR; - smaError("vgId:%d, tsma insert for smaIndex %" PRIi64 "(!=%" PRIi64 ") failed since %s", SMA_VID(pSma), indexUid, - pTsmaStat->pTSma->indexUid, tstrerror(terrno)); - goto _err; + if (ASSERTS(pTsmaStat->pTSma->indexUid == indexUid, "indexUid:%" PRIi64 " != %" PRIi64, pTsmaStat->pTSma->indexUid, + indexUid)) { + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); } SBatchDeleteReq deleteReq = {0}; void *pSubmitReq = NULL; int32_t contLen = 0; - if (smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, - pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, - &contLen) < 0) { - smaError("vgId:%d, failed to gen submit msg while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), - indexUid, tstrerror(terrno)); - goto _err; + code = smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, + pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, &contLen); + TSDB_CHECK_CODE(code, lino, _exit); + + if ((terrno = tsmaProcessDelReq(pSma, indexUid, &deleteReq)) != 0) { + goto _exit; } - // TODO deleteReq - taosArrayDestroy(deleteReq.deleteReqs); #if 0 if (!strncasecmp("td.tsma.rst.tb", pTsmaStat->pTSma->dstTbName, 14)) { terrno = TSDB_CODE_APP_ERROR; @@ -436,13 +465,13 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char .contLen = contLen, }; - if (tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg) < 0) { - smaError("vgId:%d, failed to put SubmitReq msg while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), - indexUid, tstrerror(terrno)); - goto _err; - } + code = tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg); + TSDB_CHECK_CODE(code, lino, _exit); - return TSDB_CODE_SUCCESS; -_err: - return TSDB_CODE_FAILED; +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s, smaIndex:%" PRIi64, SMA_VID(pSma), __func__, lino, + tstrerror(code), indexUid); + } + return code; } diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 7c538280e52ed127ff2815623a2afa00c126de7f..e45cbac3292a39f08074a75df8ab5c113deb6ac3 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -14,107 +14,34 @@ */ #include "sma.h" +#include "vnd.h" -#define TD_QTASKINFO_FNAME_PREFIX "main.tdb" - -void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t suid, int8_t level, int64_t version, char *outputName) { - tdRSmaGetFileName(vgId, NULL, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, suid, level, version, outputName); -} - -void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t suid, int8_t level, int64_t version, const char *path, - char *outputName) { - tdRSmaGetFileName(vgId, path, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, suid, level, version, outputName); -} - -void tdRSmaQTaskInfoGetFullPath(int32_t vgId, int8_t level, const char *path, char *outputName) { - tdRSmaGetDirName(vgId, path, VNODE_RSMA_DIR, true, outputName); - int32_t rsmaLen = strlen(outputName); - snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi8, level); -} - -void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, const char *path, char *outputName) { - tdRSmaGetDirName(vgId, path, VNODE_RSMA_DIR, true, outputName); +void tdRSmaQTaskInfoGetFullPath(SVnode *pVnode, tb_uid_t suid, int8_t level, STfs *pTfs, char *outputName) { + tdRSmaGetDirName(pVnode, pTfs, true, outputName); int32_t rsmaLen = strlen(outputName); snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi8 "%s%" PRIi64, level, TD_DIRSEP, suid); } -void tdRSmaGetFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t suid, - int8_t level, int64_t version, char *outputName) { - if (level >= 0 && suid > 0) { - if (version >= 0) { - if (pdname) { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s.%" PRIi64, pdname, - TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname, - version); - } else { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s.%" PRIi64, TD_DIRSEP, - vgId, TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname, version); - } - } else { - if (pdname) { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s", pdname, - TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname); - } else { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s", TD_DIRSEP, vgId, - TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname); - } - } - } else { - if (version >= 0) { - if (pdname) { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s%" PRIi64, pdname, TD_DIRSEP, TD_DIRSEP, - vgId, TD_DIRSEP, dname, TD_DIRSEP, vgId, fname, version); - } else { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s%" PRIi64, TD_DIRSEP, vgId, TD_DIRSEP, dname, - TD_DIRSEP, vgId, fname, version); - } - } else { - if (pdname) { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, - TD_DIRSEP, dname, TD_DIRSEP, vgId, fname); - } else { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s", TD_DIRSEP, vgId, TD_DIRSEP, dname, - TD_DIRSEP, vgId, fname); - } - } - } -} +void tdRSmaGetDirName(SVnode *pVnode, STfs *pTfs, bool endWithSep, char *outputName) { + int32_t offset = 0; -void tdRSmaGetDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName) { - if (pdname) { - if (endWithSep) { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP, - dname, TD_DIRSEP); - } else { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP, - dname); - } - } else { - if (endWithSep) { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%s", TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP); - } else { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s", TD_DIRSEP, vgId, TD_DIRSEP, dname); - } - } + // vnode + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pTfs, outputName, TSDB_FILENAME_LEN); + offset = strlen(outputName); + + // rsma + snprintf(outputName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s%s", TD_DIRSEP, VNODE_RSMA_DIR, + (endWithSep ? TD_DIRSEP : "")); } // smaXXXUtil ================ -void *tdAcquireSmaRef(int32_t rsetId, int64_t refId) { - void *pResult = taosAcquireRef(rsetId, refId); - if (!pResult) { - smaWarn("rsma acquire ref for rsetId:%d refId:%" PRIi64 " failed since %s", rsetId, refId, terrstr()); - } else { - smaTrace("rsma acquire ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId); - } - return pResult; -} +void *tdAcquireSmaRef(int32_t rsetId, int64_t refId) { return taosAcquireRef(rsetId, refId); } int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId) { if (taosReleaseRef(rsetId, refId) < 0) { smaWarn("rsma release ref for rsetId:%d refId:%" PRIi64 " failed since %s", rsetId, refId, terrstr()); return TSDB_CODE_FAILED; } - smaTrace("rsma release ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId); return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 2398a8b5623e2b5d1eff2866f4f06e23a6bed178..de34a968367b7d882cd11d2e612c60d9cfd4f2ab 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -18,13 +18,11 @@ // 0: not init // 1: already inited // 2: wait to be inited or cleaup -#define WAL_READ_TASKS_ID (-1) - static int32_t tqInitialize(STQ* pTq); static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_STATUS_EXEC == pHandle->status; } -static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_EXEC;} -static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_IDLE;} +static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_EXEC; } +static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_IDLE; } int32_t tqInit() { int8_t old; @@ -62,30 +60,32 @@ void tqCleanUp() { } } -static void destroyTqHandle(void* data) { +void tqDestroyTqHandle(void* data) { STqHandle* pData = (STqHandle*)data; qDestroyTask(pData->execHandle.task); if (pData->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { taosMemoryFreeClear(pData->execHandle.execCol.qmsg); } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__DB) { - tqCloseReader(pData->execHandle.pTqReader); + tqReaderClose(pData->execHandle.pTqReader); walCloseReader(pData->pWalReader); taosHashCleanup(pData->execHandle.execDb.pFilterOutTbUid); } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { walCloseReader(pData->pWalReader); - tqCloseReader(pData->execHandle.pTqReader); + tqReaderClose(pData->execHandle.pTqReader); + taosMemoryFreeClear(pData->execHandle.execTb.qmsg); + nodesDestroyNode(pData->execHandle.execTb.node); } - if(pData->msg != NULL) { + if (pData->msg != NULL) { rpcFreeCont(pData->msg->pCont); taosMemoryFree(pData->msg); pData->msg = NULL; } } -static bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) { +static bool tqOffsetEqual(const STqOffset* pLeft, const STqOffset* pRight) { return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG && - pLeft->val.version <= pRight->val.version; + pLeft->val.version == pRight->val.version; } STQ* tqOpen(const char* path, SVnode* pVnode) { @@ -100,7 +100,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { pTq->walLogLastVer = pVnode->pWal->vers.lastVer; pTq->pHandle = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); - taosHashSetFreeFp(pTq->pHandle, destroyTqHandle); + taosHashSetFreeFp(pTq->pHandle, tqDestroyTqHandle); taosInitRWLatch(&pTq->lock); pTq->pPushMgr = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); @@ -146,6 +146,20 @@ void tqClose(STQ* pTq) { return; } + void* pIter = taosHashIterate(pTq->pPushMgr, NULL); + while (pIter) { + STqHandle* pHandle = *(STqHandle**)pIter; + int32_t vgId = TD_VID(pTq->pVnode); + + if(pHandle->msg != NULL) { + tqPushEmptyDataRsp(pHandle, vgId); + rpcFreeCont(pHandle->msg->pCont); + taosMemoryFree(pHandle->msg); + pHandle->msg = NULL; + } + pIter = taosHashIterate(pTq->pPushMgr, pIter); + } + tqOffsetClose(pTq->pOffsetStore); taosHashCleanup(pTq->pHandle); taosHashCleanup(pTq->pPushMgr); @@ -156,6 +170,29 @@ void tqClose(STQ* pTq) { taosMemoryFree(pTq); } +static bool hasStreamTaskInTimer(SStreamMeta* pMeta) { + bool inTimer = false; + + taosWLockLatch(&pMeta->lock); + + void* pIter = NULL; + while(1) { + pIter = taosHashIterate(pMeta->pTasks, pIter); + if (pIter == NULL) { + break; + } + + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->status.timerActive >= 1) { + inTimer = true; + } + } + + taosWUnLockLatch(&pMeta->lock); + + return inTimer; +} + void tqNotifyClose(STQ* pTq) { if (pTq != NULL) { taosWLockLatch(&pTq->pStreamMeta->lock); @@ -168,87 +205,121 @@ void tqNotifyClose(STQ* pTq) { } SStreamTask* pTask = *(SStreamTask**)pIter; - tqDebug("vgId:%d s-task:%s set dropping flag", pTq->pStreamMeta->vgId, pTask->id.idStr); + tqDebug("vgId:%d s-task:%s set closing flag", pTq->pStreamMeta->vgId, pTask->id.idStr); pTask->status.taskStatus = TASK_STATUS__STOP; int64_t st = taosGetTimestampMs(); qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS); + int64_t el = taosGetTimestampMs() - st; tqDebug("vgId:%d s-task:%s is closed in %" PRId64 " ms", pTq->pStreamMeta->vgId, pTask->id.idStr, el); } taosWUnLockLatch(&pTq->pStreamMeta->lock); - } -} -static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, - int64_t consumerId, int32_t type) { - int32_t len = 0; - int32_t code = 0; + tqDebug("vgId:%d start to check all tasks", pTq->pStreamMeta->vgId); - if (type == TMQ_MSG_TYPE__POLL_RSP) { - tEncodeSize(tEncodeMqDataRsp, pRsp, len, code); - } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) { - tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code); - } + int64_t st = taosGetTimestampMs(); - if (code < 0) { - return -1; - } + while(hasStreamTaskInTimer(pTq->pStreamMeta)) { + tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pTq->pStreamMeta->vgId); + taosMsleep(100); + } - int32_t tlen = sizeof(SMqRspHead) + len; - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - return -1; + int64_t el = taosGetTimestampMs() - st; + tqDebug("vgId:%d all stream tasks are not in timer, continue close, elapsed time:%"PRId64" ms", pTq->pStreamMeta->vgId, el); } +} - ((SMqRspHead*)buf)->mqMsgType = type; - ((SMqRspHead*)buf)->epoch = epoch; - ((SMqRspHead*)buf)->consumerId = consumerId; - - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - - SEncoder encoder = {0}; - tEncoderInit(&encoder, abuf, len); - - if (type == TMQ_MSG_TYPE__POLL_RSP) { - tEncodeMqDataRsp(&encoder, pRsp); - } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) { - tEncodeSTaosxRsp(&encoder, (STaosxRsp*)pRsp); +//static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, +// int64_t consumerId, int32_t type) { +// int32_t len = 0; +// int32_t code = 0; +// +// if (type == TMQ_MSG_TYPE__POLL_DATA_RSP) { +// tEncodeSize(tEncodeMqDataRsp, pRsp, len, code); +// } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { +// tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code); +// } +// +// if (code < 0) { +// return -1; +// } +// +// int32_t tlen = sizeof(SMqRspHead) + len; +// void* buf = rpcMallocCont(tlen); +// if (buf == NULL) { +// return -1; +// } +// +// ((SMqRspHead*)buf)->mqMsgType = type; +// ((SMqRspHead*)buf)->epoch = epoch; +// ((SMqRspHead*)buf)->consumerId = consumerId; +// +// void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); +// +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, abuf, len); +// +// if (type == TMQ_MSG_TYPE__POLL_DATA_RSP) { +// tEncodeMqDataRsp(&encoder, pRsp); +// } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { +// tEncodeSTaosxRsp(&encoder, (STaosxRsp*)pRsp); +// } +// +// tEncoderClear(&encoder); +// +// SRpcMsg rsp = { +// .info = *pRpcHandleInfo, +// .pCont = buf, +// .contLen = tlen, +// .code = 0, +// }; +// +// tmsgSendRsp(&rsp); +// return 0; +//} + +int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { + SMqPollReq req = {0}; + if (tDeserializeSMqPollReq(pHandle->msg->pCont, pHandle->msg->contLen, &req) < 0) { + tqError("tDeserializeSMqPollReq %d failed", pHandle->msg->contLen); + terrno = TSDB_CODE_INVALID_MSG; + return -1; } - tEncoderClear(&encoder); - - SRpcMsg rsp = { - .info = *pRpcHandleInfo, - .pCont = buf, - .contLen = tlen, - .code = 0, - }; - - tmsgSendRsp(&rsp); - return 0; -} - -int32_t tqPushDataRsp(STqHandle* pHandle, int32_t vgId) { SMqDataRsp dataRsp = {0}; - dataRsp.head.consumerId = pHandle->consumerId; - dataRsp.head.epoch = pHandle->epoch; - dataRsp.head.mqMsgType = TMQ_MSG_TYPE__POLL_RSP; - - int64_t sver = 0, ever = 0; - walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); - tqDoSendDataRsp(&pHandle->msg->info, &dataRsp, pHandle->epoch, pHandle->consumerId, TMQ_MSG_TYPE__POLL_RSP, sver, ever); - - char buf1[80] = {0}; - char buf2[80] = {0}; - tFormatOffset(buf1, tListLen(buf1), &dataRsp.reqOffset); - tFormatOffset(buf2, tListLen(buf2), &dataRsp.rspOffset); - tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) push rsp, block num: %d, req:%s, rsp:%s", - vgId, dataRsp.head.consumerId, dataRsp.head.epoch, dataRsp.blockNum, buf1, buf2); + tqInitDataRsp(&dataRsp, req.reqOffset); + dataRsp.blockNum = 0; + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.reqOffset); + tqInfo("tqPushEmptyDataRsp to consumer:0x%"PRIx64 " vgId:%d, offset:%s, reqId:0x%" PRIx64, req.consumerId, vgId, buf, req.reqId); + + tqSendDataRsp(pHandle, pHandle->msg, &req, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + tDeleteMqDataRsp(&dataRsp); return 0; } +//int32_t tqPushDataRsp(STqHandle* pHandle, int32_t vgId) { +// SMqDataRsp dataRsp = {0}; +// dataRsp.head.consumerId = pHandle->consumerId; +// dataRsp.head.epoch = pHandle->epoch; +// dataRsp.head.mqMsgType = TMQ_MSG_TYPE__POLL_RSP; +// +// int64_t sver = 0, ever = 0; +// walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); +// tqDoSendDataRsp(&pHandle->msg->info, &dataRsp, pHandle->epoch, pHandle->consumerId, TMQ_MSG_TYPE__POLL_RSP, sver, +// ever); +// +// char buf1[TSDB_OFFSET_LEN] = {0}; +// char buf2[TSDB_OFFSET_LEN] = {0}; +// tFormatOffset(buf1, tListLen(buf1), &dataRsp.reqOffset); +// tFormatOffset(buf2, tListLen(buf2), &dataRsp.rspOffset); +// tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) push rsp, block num: %d, req:%s, rsp:%s", vgId, +// dataRsp.head.consumerId, dataRsp.head.epoch, dataRsp.blockNum, buf1, buf2); +// return 0; +//} + int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type, int32_t vgId) { int64_t sver = 0, ever = 0; @@ -256,13 +327,13 @@ int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* tqDoSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type, sver, ever); - char buf1[80] = {0}; - char buf2[80] = {0}; - tFormatOffset(buf1, 80, &pRsp->reqOffset); - tFormatOffset(buf2, 80, &pRsp->rspOffset); + char buf1[TSDB_OFFSET_LEN] = {0}; + char buf2[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf1, TSDB_OFFSET_LEN, &pRsp->reqOffset); + tFormatOffset(buf2, TSDB_OFFSET_LEN, &pRsp->rspOffset); - tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64, - vgId, pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId); + tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64, vgId, + pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId); return 0; } @@ -287,17 +358,14 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t } else if (pOffset->val.type == TMQ_OFFSET__LOG) { tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, pOffset->subKey, vgId, pOffset->val.version); - if (pOffset->val.version + 1 == sversion) { - pOffset->val.version += 1; - } } else { tqError("invalid commit offset type:%d", pOffset->val.type); return -1; } STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); - if (pSavedOffset != NULL && tqOffsetLessOrEqual(pOffset, pSavedOffset)) { - tqDebug("not update the offset, vgId:%d sub:%s since committed:%" PRId64 " less than/equal to existed:%" PRId64, + if (pSavedOffset != NULL && tqOffsetEqual(pOffset, pSavedOffset)) { + tqInfo("not update the offset, vgId:%d sub:%s since committed:%" PRId64 " less than/equal to existed:%" PRId64, vgId, pOffset->subKey, pOffset->val.version, pSavedOffset->val.version); return 0; // no need to update the offset value } @@ -307,93 +375,127 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t return -1; } - if (pOffset->val.type == TMQ_OFFSET__LOG) { - taosWLockLatch(&pTq->lock); - STqHandle* pHandle = taosHashGet(pTq->pHandle, pOffset->subKey, strlen(pOffset->subKey)); - if (pHandle && (walSetRefVer(pHandle->pRef, pOffset->val.version) < 0)) { - taosWUnLockLatch(&pTq->lock); - return -1; - } - taosWUnLockLatch(&pTq->lock); - } - return 0; } -int32_t tqProcessSeekReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - SMqVgOffset vgOffset = {0}; +int32_t tqProcessSeekReq(STQ* pTq, SRpcMsg* pMsg) { + SMqSeekReq req = {0}; int32_t vgId = TD_VID(pTq->pVnode); + SRpcMsg rsp = {.info = pMsg->info}; + int code = 0; - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - if (tDecodeMqVgOffset(&decoder, &vgOffset) < 0) { - return -1; - } - - tDecoderClear(&decoder); - - STqOffset* pOffset = &vgOffset.offset; - if (pOffset->val.type != TMQ_OFFSET__LOG) { - tqError("vgId:%d, subKey:%s invalid seek offset type:%d", vgId, pOffset->subKey, pOffset->val.type); - return -1; + if (tDeserializeSMqSeekReq(pMsg->pCont, pMsg->contLen, &req) < 0) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; } - STqHandle* pHandle = taosHashGet(pTq->pHandle, pOffset->subKey, strlen(pOffset->subKey)); + tqDebug("tmq seek: consumer:0x%" PRIx64 " vgId:%d, subkey %s", req.consumerId, vgId, req.subKey); + STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { - tqError("tmq seek: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", vgOffset.consumerId, vgId, - pOffset->subKey); - terrno = TSDB_CODE_INVALID_MSG; - return -1; + tqWarn("tmq seek: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", req.consumerId, vgId, req.subKey); + code = 0; + goto end; } // 2. check consumer-vg assignment status taosRLockLatch(&pTq->lock); - if (pHandle->consumerId != vgOffset.consumerId) { - tqDebug("ERROR tmq seek: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, - vgOffset.consumerId, vgId, pOffset->subKey, pHandle->consumerId); - terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + if (pHandle->consumerId != req.consumerId) { + tqError("ERROR tmq seek: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + req.consumerId, vgId, req.subKey, pHandle->consumerId); taosRUnLockLatch(&pTq->lock); - return -1; - } - taosRUnLockLatch(&pTq->lock); - - //3. check the offset info - STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); - if (pSavedOffset != NULL) { - if (pSavedOffset->val.type != TMQ_OFFSET__LOG) { - tqError("invalid saved offset type, vgId:%d sub:%s", vgId, pOffset->subKey); - return 0; // no need to update the offset value - } - - if (pSavedOffset->val.version == pOffset->val.version) { - tqDebug("vgId:%d subKey:%s no need to seek to %" PRId64 " prev offset:%" PRId64, vgId, pOffset->subKey, - pOffset->val.version, pSavedOffset->val.version); - return 0; - } - } - - int64_t sver = 0, ever = 0; - walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); - if (pOffset->val.version < sver) { - pOffset->val.version = sver; - } else if (pOffset->val.version > ever) { - pOffset->val.version = ever; - } - - // save the new offset value - if (pSavedOffset != NULL) { - tqDebug("vgId:%d sub:%s seek to:%" PRId64 " prev offset:%" PRId64, vgId, pOffset->subKey, pOffset->val.version, - pSavedOffset->val.version); - } else { - tqDebug("vgId:%d sub:%s seek to:%"PRId64" not saved yet", vgId, pOffset->subKey, pOffset->val.version); + code = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + goto end; } - if (tqOffsetWrite(pTq->pOffsetStore, pOffset) < 0) { - tqError("failed to save offset, vgId:%d sub:%s seek to %" PRId64, vgId, pOffset->subKey, pOffset->val.version); - return -1; - } + //if consumer register to push manager, push empty to consumer to change vg status from TMQ_VG_STATUS__WAIT to TMQ_VG_STATUS__IDLE, + //otherwise poll data failed after seek. + tqUnregisterPushHandle(pTq, pHandle); + taosRUnLockLatch(&pTq->lock); +end: + rsp.code = code; + tmsgSendRsp(&rsp); return 0; + +// SMqVgOffset vgOffset = {0}; +// int32_t vgId = TD_VID(pTq->pVnode); +// +// SDecoder decoder; +// tDecoderInit(&decoder, (uint8_t*)msg, msgLen); +// if (tDecodeMqVgOffset(&decoder, &vgOffset) < 0) { +// tqError("vgId:%d failed to decode seek msg", vgId); +// return -1; +// } +// +// tDecoderClear(&decoder); +// +// tqDebug("topic:%s, vgId:%d process offset seek by consumer:0x%" PRIx64 ", req offset:%" PRId64, +// vgOffset.offset.subKey, vgId, vgOffset.consumerId, vgOffset.offset.val.version); +// +// STqOffset* pOffset = &vgOffset.offset; +// if (pOffset->val.type != TMQ_OFFSET__LOG) { +// tqError("vgId:%d, subKey:%s invalid seek offset type:%d", vgId, pOffset->subKey, pOffset->val.type); +// return -1; +// } +// +// STqHandle* pHandle = taosHashGet(pTq->pHandle, pOffset->subKey, strlen(pOffset->subKey)); +// if (pHandle == NULL) { +// tqError("tmq seek: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", vgOffset.consumerId, vgId, pOffset->subKey); +// terrno = TSDB_CODE_INVALID_MSG; +// return -1; +// } +// +// // 2. check consumer-vg assignment status +// taosRLockLatch(&pTq->lock); +// if (pHandle->consumerId != vgOffset.consumerId) { +// tqDebug("ERROR tmq seek: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, +// vgOffset.consumerId, vgId, pOffset->subKey, pHandle->consumerId); +// terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; +// taosRUnLockLatch(&pTq->lock); +// return -1; +// } +// taosRUnLockLatch(&pTq->lock); +// +// // 3. check the offset info +// STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); +// if (pSavedOffset != NULL) { +// if (pSavedOffset->val.type != TMQ_OFFSET__LOG) { +// tqError("invalid saved offset type, vgId:%d sub:%s", vgId, pOffset->subKey); +// return 0; // no need to update the offset value +// } +// +// if (pSavedOffset->val.version == pOffset->val.version) { +// tqDebug("vgId:%d subKey:%s no need to seek to %" PRId64 " prev offset:%" PRId64, vgId, pOffset->subKey, +// pOffset->val.version, pSavedOffset->val.version); +// return 0; +// } +// } +// +// int64_t sver = 0, ever = 0; +// walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); +// if (pOffset->val.version < sver) { +// pOffset->val.version = sver; +// } else if (pOffset->val.version > ever) { +// pOffset->val.version = ever; +// } +// +// // save the new offset value +// if (pSavedOffset != NULL) { +// tqDebug("vgId:%d sub:%s seek to:%" PRId64 " prev offset:%" PRId64, vgId, pOffset->subKey, pOffset->val.version, +// pSavedOffset->val.version); +// } else { +// tqDebug("vgId:%d sub:%s seek to:%" PRId64 " not saved yet", vgId, pOffset->subKey, pOffset->val.version); +// } +// +// if (tqOffsetWrite(pTq->pOffsetStore, pOffset) < 0) { +// tqError("failed to save offset, vgId:%d sub:%s seek to %" PRId64, vgId, pOffset->subKey, pOffset->val.version); +// return -1; +// } +// +// tqDebug("topic:%s, vgId:%d consumer:0x%" PRIx64 " offset is update to:%" PRId64, vgOffset.offset.subKey, vgId, +// vgOffset.consumerId, vgOffset.offset.val.version); +// +// return 0; } int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { @@ -422,8 +524,39 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { return 0; } +int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg) { + int32_t vgId = TD_VID(pTq->pVnode); + taosWLockLatch(&pTq->lock); + if (taosHashGetSize(pTq->pPushMgr) > 0) { + void* pIter = taosHashIterate(pTq->pPushMgr, NULL); + + while (pIter) { + STqHandle* pHandle = *(STqHandle**)pIter; + tqInfo("vgId:%d start set submit for pHandle:%p, consumer:0x%" PRIx64, vgId, pHandle, pHandle->consumerId); + + if (ASSERT(pHandle->msg != NULL)) { + tqError("pHandle->msg should not be null"); + taosHashCancelIterate(pTq->pPushMgr, pIter); + break; + }else{ + SRpcMsg msg = {.msgType = TDMT_VND_TMQ_CONSUME, .pCont = pHandle->msg->pCont, .contLen = pHandle->msg->contLen, .info = pHandle->msg->info}; + tmsgPutToQueue(&pTq->pVnode->msgCb, QUERY_QUEUE, &msg); + taosMemoryFree(pHandle->msg); + pHandle->msg = NULL; + } + + pIter = taosHashIterate(pTq->pPushMgr, pIter); + } + + taosHashClear(pTq->pPushMgr); + } + taosWUnLockLatch(&pTq->lock); + return 0; +} + int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { SMqPollReq req = {0}; + int code = 0; if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen); terrno = TSDB_CODE_INVALID_MSG; @@ -434,51 +567,121 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int32_t reqEpoch = req.epoch; STqOffsetVal reqOffset = req.reqOffset; int32_t vgId = TD_VID(pTq->pVnode); + STqHandle* pHandle = NULL; - taosWLockLatch(&pTq->lock); - // 1. find handle - STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); - if (pHandle == NULL) { - tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); - terrno = TSDB_CODE_INVALID_MSG; - taosWUnLockLatch(&pTq->lock); - return -1; - } + while (1) { + taosWLockLatch(&pTq->lock); + // 1. find handle + pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + if (pHandle == NULL) { + do{ + if (tqMetaGetHandle(pTq, req.subKey) == 0){ + pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + if(pHandle != NULL){ + break; + } + } + tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_MSG; + taosWUnLockLatch(&pTq->lock); + return -1; + }while(0); + } - while (tqIsHandleExec(pHandle)) { - tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", consumerId, vgId, req.subKey); - taosMsleep(5); - } + // 2. check re-balance status + if (pHandle->consumerId != consumerId) { + tqError("ERROR tmq poll: consumer:0x%" PRIx64 + " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + taosWUnLockLatch(&pTq->lock); + return -1; + } - // 2. check re-balance status - if (pHandle->consumerId != consumerId) { - tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, - consumerId, vgId, req.subKey, pHandle->consumerId); - terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + bool exec = tqIsHandleExec(pHandle); + if (!exec) { + tqSetHandleExec(pHandle); + // qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); + tqDebug("tmq poll: consumer:0x%" PRIx64 " vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, + req.subKey, pHandle); + taosWUnLockLatch(&pTq->lock); + break; + } taosWUnLockLatch(&pTq->lock); - return -1; + + tqDebug("tmq poll: consumer:0x%" PRIx64 + "vgId:%d, topic:%s, subscription is executing, wait for 10ms and retry, pHandle:%p", + consumerId, vgId, req.subKey, pHandle); + taosMsleep(10); } - tqSetHandleExec(pHandle); - taosWUnLockLatch(&pTq->lock); // 3. update the epoch value - int32_t savedEpoch = pHandle->epoch; - if (savedEpoch < reqEpoch) { - tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, + if (pHandle->epoch < reqEpoch) { + tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, pHandle->epoch, reqEpoch); pHandle->epoch = reqEpoch; } - char buf[80]; - tFormatOffset(buf, 80, &reqOffset); + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &reqOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); - int code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); + code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); tqSetHandleIdle(pHandle); + + tqDebug("tmq poll: consumer:0x%" PRIx64 " vgId:%d, topic:%s, set handle idle, pHandle:%p", consumerId, vgId, + req.subKey, pHandle); return code; } +int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg) { + void* data = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + + SMqVgOffset vgOffset = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)data, len); + if (tDecodeMqVgOffset(&decoder, &vgOffset) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + tDecoderClear(&decoder); + + STqOffset* pOffset = &vgOffset.offset; + STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); + if (pSavedOffset == NULL) { + terrno = TSDB_CODE_TMQ_NO_COMMITTED; + return terrno; + } + vgOffset.offset = *pSavedOffset; + + int32_t code = 0; + tEncodeSize(tEncodeMqVgOffset, &vgOffset, len, code); + if (code < 0) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + void* buf = rpcMallocCont(len); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + SEncoder encoder; + tEncoderInit(&encoder, buf, len); + tEncodeMqVgOffset(&encoder, &vgOffset); + tEncoderClear(&encoder); + + SRpcMsg rsp = {.info = pMsg->info, .pCont = buf, .contLen = len, .code = 0}; + + tmsgSendRsp(&rsp); + + return 0; +} + int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { SMqPollReq req = {0}; if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { @@ -513,65 +716,69 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { int64_t sver = 0, ever = 0; walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); - int64_t currentVer = walReaderGetCurrentVer(pHandle->execHandle.pTqReader->pWalReader); - SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, &req); - - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey); - if (pOffset != NULL) { - if (pOffset->val.type != TMQ_OFFSET__LOG) { - tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s use snapshot, no valid wal info", consumerId, vgId, req.subKey); - terrno = TSDB_CODE_INVALID_PARA; - tDeleteMqDataRsp(&dataRsp); - return -1; - } + tqInitDataRsp(&dataRsp, req.reqOffset); - dataRsp.rspOffset.type = TMQ_OFFSET__LOG; - dataRsp.rspOffset.version = pOffset->val.version; - } else { - if (req.useSnapshot == true) { - tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s snapshot not support wal info", consumerId, vgId, req.subKey); - terrno = TSDB_CODE_INVALID_PARA; - tDeleteMqDataRsp(&dataRsp); - return -1; - } + if (req.useSnapshot == true) { + tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s snapshot not support wal info", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_PARA; + tDeleteMqDataRsp(&dataRsp); + return -1; + } - dataRsp.rspOffset.type = TMQ_OFFSET__LOG; + dataRsp.rspOffset.type = TMQ_OFFSET__LOG; - if (reqOffset.type == TMQ_OFFSET__LOG) { - dataRsp.rspOffset.version = currentVer; // return current consume offset value - } else if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { - dataRsp.rspOffset.version = sver; // not consume yet, set the earliest position - } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - dataRsp.rspOffset.version = ever; - } else { - tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s invalid offset type:%d", consumerId, vgId, req.subKey, - reqOffset.type); - terrno = TSDB_CODE_INVALID_PARA; - tDeleteMqDataRsp(&dataRsp); - return -1; + if (reqOffset.type == TMQ_OFFSET__LOG) { + dataRsp.rspOffset.version = reqOffset.version; + } else if(reqOffset.type < 0){ + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey); + if (pOffset != NULL) { + if (pOffset->val.type != TMQ_OFFSET__LOG) { + tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s, no valid wal info", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_PARA; + tDeleteMqDataRsp(&dataRsp); + return -1; + } + + dataRsp.rspOffset.version = pOffset->val.version; + tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from store:%"PRId64, consumerId, vgId, req.subKey, dataRsp.rspOffset.version); + }else{ + if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEST) { + dataRsp.rspOffset.version = sver; // not consume yet, set the earliest position + } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { + dataRsp.rspOffset.version = ever; + } + tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from init:%"PRId64, consumerId, vgId, req.subKey, dataRsp.rspOffset.version); } + } else { + tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s invalid offset type:%d", consumerId, vgId, req.subKey, + reqOffset.type); + terrno = TSDB_CODE_INVALID_PARA; + tDeleteMqDataRsp(&dataRsp); + return -1; } tqDoSendDataRsp(&pMsg->info, &dataRsp, req.epoch, req.consumerId, TMQ_MSG_TYPE__WALINFO_RSP, sver, ever); + tDeleteMqDataRsp(&dataRsp); return 0; } int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; - int32_t vgId = TD_VID(pTq->pVnode); + int32_t vgId = TD_VID(pTq->pVnode); - tqDebug("vgId:%d, tq process delete sub req %s", vgId, pReq->subKey); + tqInfo("vgId:%d, tq process delete sub req %s", vgId, pReq->subKey); int32_t code = 0; taosWLockLatch(&pTq->lock); STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { while (tqIsHandleExec(pHandle)) { - tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", vgId, pHandle->subKey); - taosMsleep(5); + tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 10ms and retry, pHandle:%p", vgId, + pHandle->subKey, pHandle); + taosMsleep(10); } + if (pHandle->pRef) { walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId); } @@ -628,197 +835,168 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t } int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - int ret = 0; + int ret = 0; SMqRebVgReq req = {0}; - tDecodeSMqRebVgReq(msg, &req); + SDecoder dc = {0}; - SVnode* pVnode = pTq->pVnode; - int32_t vgId = TD_VID(pVnode); + tDecoderInit(&dc, msg, msgLen); + + // decode req + if (tDecodeSMqRebVgReq(&dc, &req) < 0) { + terrno = TSDB_CODE_INVALID_MSG; + tDecoderClear(&dc); + return -1; + } - tqDebug("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pVnode->config.vgId, req.subKey, + tqInfo("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pTq->pVnode->config.vgId, req.subKey, req.oldConsumerId, req.newConsumerId); - taosWLockLatch(&pTq->lock); - STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + STqHandle* pHandle = NULL; + while(1){ + pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + if (pHandle || tqMetaGetHandle(pTq, req.subKey) < 0){ + break; + } + } + if (pHandle == NULL) { if (req.oldConsumerId != -1) { tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64, req.vgId, req.subKey, req.newConsumerId, req.oldConsumerId); } - if (req.newConsumerId == -1) { tqError("vgId:%d, tq invalid re-balance request, new consumerId %" PRId64 "", req.vgId, req.newConsumerId); goto end; } - - STqHandle tqHandle = {0}; - pHandle = &tqHandle; - - uint64_t oldConsumerId = pHandle->consumerId; - memcpy(pHandle->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN); - pHandle->consumerId = req.newConsumerId; - pHandle->epoch = -1; - - pHandle->execHandle.subType = req.subType; - pHandle->fetchMeta = req.withMeta; - - // TODO version should be assigned and refed during preprocess - SWalRef* pRef = walRefCommittedVer(pVnode->pWal); - if (pRef == NULL) { - ret = -1; + STqHandle handle = {0}; + ret = tqCreateHandle(pTq, &req, &handle); + if(ret < 0){ + tqDestroyTqHandle(&handle); goto end; } - - int64_t ver = pRef->refVer; - pHandle->pRef = pRef; - - SReadHandle handle = { - .meta = pVnode->pMeta, .vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver}; - pHandle->snapshotVer = ver; - - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - pHandle->execHandle.execCol.qmsg = req.qmsg; - req.qmsg = NULL; - - pHandle->execHandle.task = qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, vgId, - &pHandle->execHandle.numOfCols, req.newConsumerId); - void* scanner = NULL; - qExtractStreamScanner(pHandle->execHandle.task, &scanner); - pHandle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); - } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { - pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL); - pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); - - pHandle->execHandle.execDb.pFilterOutTbUid = - taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta, - (SSnapContext**)(&handle.sContext)); - - pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); - } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { - pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL); - pHandle->execHandle.execTb.suid = req.suid; - - SArray* tbUidList = taosArrayInit(0, sizeof(int64_t)); - vnodeGetCtbIdList(pVnode, req.suid, tbUidList); - tqDebug("vgId:%d, tq try to get all ctb, suid:%" PRId64, pVnode->config.vgId, req.suid); - for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { - int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); - } - pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); - tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); - taosArrayDestroy(tbUidList); - - buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, - (SSnapContext**)(&handle.sContext)); - pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); - } - - taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); - tqDebug("try to persist handle %s consumer:0x%" PRIx64 " , old consumer:0x%" PRIx64, req.subKey, - pHandle->consumerId, oldConsumerId); - ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); - goto end; + ret = tqMetaSaveHandle(pTq, req.subKey, &handle); } else { - while (tqIsHandleExec(pHandle)) { - tqDebug("sub req vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", vgId, pHandle->subKey); - taosMsleep(5); - } + taosWLockLatch(&pTq->lock); if (pHandle->consumerId == req.newConsumerId) { // do nothing - tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs", req.vgId, req.newConsumerId); - atomic_add_fetch_32(&pHandle->epoch, 1); - + tqInfo("vgId:%d no switch consumer:0x%" PRIx64 " remains, because redo wal log", req.vgId, req.newConsumerId); } else { - tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, - req.newConsumerId); + tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); - atomic_store_32(&pHandle->epoch, 0); - } - // kill executing task - qTaskInfo_t pTaskInfo = pHandle->execHandle.task; - if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); + // atomic_add_fetch_32(&pHandle->epoch, 1); + + // kill executing task + // if(tqIsHandleExec(pHandle)) { + // qTaskInfo_t pTaskInfo = pHandle->execHandle.task; + // if (pTaskInfo != NULL) { + // qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); + // } + + // if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + // qStreamCloseTsdbReader(pTaskInfo); + // } + // } + // remove if it has been register in the push manager, and return one empty block to consumer + tqUnregisterPushHandle(pTq, pHandle); + ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); } - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pTaskInfo); - } - // remove if it has been register in the push manager, and return one empty block to consumer - tqUnregisterPushHandle(pTq, pHandle); - ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); - goto end; + taosWUnLockLatch(&pTq->lock); } end: - taosWUnLockLatch(&pTq->lock); - taosMemoryFree(req.qmsg); + tDecoderClear(&dc); return ret; } -void freePtr(void *ptr) { - taosMemoryFree(*(void**)ptr); -} +void freePtr(void* ptr) { taosMemoryFree(*(void**)ptr); } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { int32_t vgId = TD_VID(pTq->pVnode); + pTask->id.idStr = createStreamTaskIdStr(pTask->id.streamId, pTask->id.taskId); pTask->refCnt = 1; pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->inputQueue = streamQueueOpen(512 << 10); - pTask->outputQueue = streamQueueOpen(512 << 10); + pTask->outputInfo.queue = streamQueueOpen(512 << 10); - if (pTask->inputQueue == NULL || pTask->outputQueue == NULL) { + if (pTask->inputQueue == NULL || pTask->outputInfo.queue == NULL) { + tqError("s-task:%s failed to prepare the input/output queue, initialize task failed", pTask->id.idStr); return -1; } + pTask->tsInfo.init = taosGetTimestampMs(); pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; - pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; + pTask->outputInfo.status = TASK_OUTPUT_STATUS__NORMAL; pTask->pMsgCb = &pTq->pVnode->msgCb; pTask->pMeta = pTq->pStreamMeta; + + // backup the initial status, and set it to be TASK_STATUS__INIT pTask->chkInfo.version = ver; pTask->chkInfo.currentVer = ver; - // expand executor - pTask->status.taskStatus = (pTask->fillHistory)? TASK_STATUS__WAIT_DOWNSTREAM:TASK_STATUS__NORMAL; + pTask->dataRange.range.maxVer = ver; + pTask->dataRange.range.minVer = ver; + + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + SStreamTask* pSateTask = pTask; + SStreamTask task = {0}; + if (pTask->info.fillHistory) { + task.id = pTask->streamTaskId; + task.pMeta = pTask->pMeta; + pSateTask = &task; + } - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); + pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pSateTask, false, -1, -1); if (pTask->pState == NULL) { return -1; } - SReadHandle handle = { - .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState}; + SReadHandle handle = {.vnode = pTq->pVnode, + .initTqReader = 1, + .pStateBackend = pTask->pState, + .fillHistory = pTask->info.fillHistory, + .winRange = pTask->dataRange.window}; + initStorageAPI(&handle.api); - pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId); + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId, pTask->id.taskId); if (pTask->exec.pExecutor == NULL) { return -1; } qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); - } else if (pTask->taskLevel == TASK_LEVEL__AGG) { - pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); + } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) { + SStreamTask* pSateTask = pTask; + SStreamTask task = {0}; + if (pTask->info.fillHistory) { + task.id = pTask->streamTaskId; + task.pMeta = pTask->pMeta; + pSateTask = &task; + } + pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pSateTask, false, -1, -1); if (pTask->pState == NULL) { return -1; } - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo); - SReadHandle mgHandle = {.vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState}; + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->pUpstreamEpInfoList); + SReadHandle handle = {.vnode = NULL, + .numOfVgroups = numOfVgroups, + .pStateBackend = pTask->pState, + .fillHistory = pTask->info.fillHistory, + .winRange = pTask->dataRange.window}; + initStorageAPI(&handle.api); - pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId); + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId, pTask->id.taskId); if (pTask->exec.pExecutor == NULL) { return -1; } - qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); } // sink - if (pTask->outputType == TASK_OUTPUT__SMA) { + if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { pTask->smaSink.vnode = pTq->pVnode; pTask->smaSink.smaSink = smaHandleRes; - } else if (pTask->outputType == TASK_OUTPUT__TABLE) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline; @@ -838,14 +1016,24 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { tSimpleHashSetFreeFp(pTask->tbSink.pTblInfo, freePtr); } - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + SWalFilterCond cond = {.deleteMsg = 1}; // delete msg also extract from wal files + pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond); } - streamSetupTrigger(pTask); + // reset the task status from unfinished transaction + if (pTask->status.taskStatus == TASK_STATUS__PAUSE) { + tqWarn("s-task:%s reset task status to be normal, kept in meta status: Paused", pTask->id.idStr); + pTask->status.taskStatus = TASK_STATUS__NORMAL; + } - tqInfo("vgId:%d expand stream task, s-task:%s, checkpoint ver:%" PRId64 " child id:%d, level:%d", vgId, pTask->id.idStr, - pTask->chkInfo.version, pTask->selfChildId, pTask->taskLevel); + taosThreadMutexInit(&pTask->lock, NULL); + streamSetupScheduleTrigger(pTask); + + tqInfo("vgId:%d expand stream task, s-task:%s, checkpoint ver:%" PRId64 + " child id:%d, level:%d, fill-history:%d, trigger:%" PRId64 " ms, disable pause", + vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->info.selfChildId, pTask->info.taskLevel, + pTask->info.fillHistory, pTask->triggerParam); // next valid version will add one pTask->chkInfo.version += 1; @@ -853,15 +1041,19 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { } int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { - char* msgStr = pMsg->pCont; - char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + char* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + SStreamTaskCheckReq req; SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); - tDecodeSStreamTaskCheckReq(&decoder, &req); + tDecodeStreamTaskCheckReq(&decoder, &req); tDecoderClear(&decoder); - int32_t taskId = req.downstreamTaskId; + + int32_t taskId = req.downstreamTaskId; + SStreamTaskCheckRsp rsp = { .reqId = req.reqId, .streamId = req.streamId, @@ -872,68 +1064,53 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { .upstreamTaskId = req.upstreamTaskId, }; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - - if (pTask) { + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.streamId, taskId); + if (pTask != NULL) { rsp.status = streamTaskCheckStatus(pTask); streamMetaReleaseTask(pTq->pStreamMeta, pTask); - tqDebug("tq recv task check req(reqId:0x%" PRIx64 - ") %d at node %d task status:%d, check req from task %d at node %d, rsp status %d", - rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, pTask->status.taskStatus, rsp.upstreamTaskId, - rsp.upstreamNodeId, rsp.status); + const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); + tqDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d", + pTask->id.idStr, pStatus, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); } else { rsp.status = 0; - tqDebug("tq recv task check(taskId:%d not built yet) req(reqId:0x%" PRIx64 - ") %d at node %d, check req from task %d at node %d, rsp status %d", - taskId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, - rsp.status); + tqDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64 + ") from task:0x%x (vgId:%d), rsp status %d", + req.streamId, taskId, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); } - SEncoder encoder; - int32_t code; - int32_t len; - tEncodeSize(tEncodeSStreamTaskCheckRsp, &rsp, len, code); - if (code < 0) { - tqError("unable to encode rsp %d", __LINE__); - return -1; - } - - void* buf = rpcMallocCont(sizeof(SMsgHead) + len); - ((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId); - - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncoderInit(&encoder, (uint8_t*)abuf, len); - tEncodeSStreamTaskCheckRsp(&encoder, &rsp); - tEncoderClear(&encoder); - - SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = pMsg->info}; - tmsgSendRsp(&rspMsg); - return 0; + return streamSendCheckRsp(pTq->pStreamMeta, &req, &rsp, &pMsg->info, taskId); } -int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { +int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t sversion, SRpcMsg* pMsg) { + char* pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + int32_t code; SStreamTaskCheckRsp rsp; SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - code = tDecodeSStreamTaskCheckRsp(&decoder, &rsp); + tDecoderInit(&decoder, (uint8_t*)pReq, len); + code = tDecodeStreamTaskCheckRsp(&decoder, &rsp); + if (code < 0) { tDecoderClear(&decoder); return -1; } tDecoderClear(&decoder); - tqDebug("tq recv task check rsp(reqId:0x%" PRIx64 ") %d at node %d check req from task %d at node %d, status %d", - rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", + rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, rsp.upstreamTaskId); + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, rsp.streamId, rsp.upstreamTaskId); if (pTask == NULL) { + tqError("tq failed to locate the stream task:0x%x (vgId:%d), it may have been destroyed", rsp.upstreamTaskId, + pTq->pStreamMeta->vgId); + terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; return -1; } - code = streamProcessTaskCheckRsp(pTask, &rsp, sversion); + code = streamProcessCheckRsp(pTask, &rsp); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return code; } @@ -949,6 +1126,9 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms // 1.deserialize msg and build task SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d failed to create stream task due to out of memory, alloc size:%d", vgId, + (int32_t)sizeof(SStreamTask)); return -1; } @@ -963,209 +1143,327 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms tDecoderClear(&decoder); + SStreamMeta* pStreamMeta = pTq->pStreamMeta; + // 2.save task, use the newest commit version as the initial start version of stream task. - taosWLockLatch(&pTq->pStreamMeta->lock); - code = streamMetaAddDeployedTask(pTq->pStreamMeta, sversion, pTask); + int32_t taskId = pTask->id.taskId; + bool added = false; + + taosWLockLatch(&pStreamMeta->lock); + code = streamMetaRegisterTask(pStreamMeta, sversion, pTask, &added); + int32_t numOfTasks = streamMetaGetNumOfTasks(pStreamMeta); + taosWUnLockLatch(&pStreamMeta->lock); + if (code < 0) { - tqError("vgId:%d failed to add s-task:%s, total:%d", vgId, pTask->id.idStr, - streamMetaGetNumOfTasks(pTq->pStreamMeta)); - taosWUnLockLatch(&pTq->pStreamMeta->lock); + tqError("vgId:%d failed to add s-task:0x%x, total:%d", vgId, pTask->id.taskId, numOfTasks); + tFreeStreamTask(pTask); return -1; } - taosWUnLockLatch(&pTq->pStreamMeta->lock); - - // 3.go through recover steps to fill history - if (pTask->fillHistory) { - streamTaskCheckDownstream(pTask, sversion); + // not added into meta store + if (added) { + tqDebug("vgId:%d s-task:0x%x is deployed and add into meta, numOfTasks:%d", vgId, taskId, numOfTasks); + SStreamTask* p = streamMetaAcquireTask(pStreamMeta, pTask->id.streamId, taskId); + if (p != NULL) { // reset the downstreamReady flag. + streamTaskCheckDownstreamTasks(p); + } + streamMetaReleaseTask(pStreamMeta, p); + } else { + tqWarn("vgId:%d failed to add s-task:0x%x, already exists in meta store", vgId, taskId); + tFreeStreamTask(pTask); } - tqDebug("vgId:%d s-task:%s is deployed and add meta from mnd, status:%d, total:%d", vgId, pTask->id.idStr, - pTask->status.taskStatus, streamMetaGetNumOfTasks(pTq->pStreamMeta)); return 0; } -int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { - int32_t code; - char* msg = pMsg->pCont; - int32_t msgLen = pMsg->contLen; +int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { + SStreamScanHistoryReq* pReq = (SStreamScanHistoryReq*)pMsg->pCont; + SStreamMeta* pMeta = pTq->pStreamMeta; - SStreamRecoverStep1Req* pReq = (SStreamRecoverStep1Req*)msg; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); + int32_t code = TSDB_CODE_SUCCESS; + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); if (pTask == NULL) { + tqError("vgId:%d failed to acquire stream task:0x%x during stream recover, task may have been destroyed", + pMeta->vgId, pReq->taskId); return -1; } - // check param - int64_t fillVer1 = pTask->chkInfo.version; - if (fillVer1 <= 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; - } - - // do recovery step 1 - tqDebug("s-task:%s start non-blocking recover stage(step 1) scan", pTask->id.idStr); - int64_t st = taosGetTimestampMs(); + // do recovery step1 + const char* id = pTask->id.idStr; + const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); + tqDebug("s-task:%s start scan-history stage(step 1), status:%s", id, pStatus); - streamSourceRecoverScanStep1(pTask); - if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { - tqDebug("s-task:%s is dropped, abort recover in step1", pTask->id.idStr); + if (pTask->tsInfo.step1Start == 0) { + ASSERT(pTask->status.pauseAllowed == false); + pTask->tsInfo.step1Start = taosGetTimestampMs(); + if (pTask->info.fillHistory == 1) { + streamTaskEnablePause(pTask); + } + } else { + tqDebug("s-task:%s resume from paused, start ts:%"PRId64, pTask->id.idStr, pTask->tsInfo.step1Start); + } - streamMetaReleaseTask(pTq->pStreamMeta, pTask); + // we have to continue retrying to successfully execute the scan history task. + int8_t schedStatus = atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE, + TASK_SCHED_STATUS__WAITING); + if (schedStatus != TASK_SCHED_STATUS__INACTIVE) { + tqError( + "s-task:%s failed to start scan-history in first stream time window since already started, unexpected " + "sched-status:%d", + id, schedStatus); return 0; } - double el = (taosGetTimestampMs() - st) / 1000.0; - tqDebug("s-task:%s non-blocking recover stage(step 1) ended, elapsed time:%.2fs", pTask->id.idStr, el); - - // build msg to launch next step - SStreamRecoverStep2Req req; - code = streamBuildSourceRecover2Req(pTask, &req); - if (code < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; + if (pTask->info.fillHistory == 1) { + ASSERT(pTask->status.pauseAllowed == true); } - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { + streamSourceScanHistoryData(pTask); + if (pTask->status.taskStatus == TASK_STATUS__PAUSE) { + double el = (taosGetTimestampMs() - pTask->tsInfo.step1Start) / 1000.0; + tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs, sched-status:%d", pTask->id.idStr, el, + TASK_SCHED_STATUS__INACTIVE); + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); return 0; } - // serialize msg - int32_t len = sizeof(SStreamRecoverStep1Req); + // the following procedure should be executed, no matter status is stop/pause or not + double el = (taosGetTimestampMs() - pTask->tsInfo.step1Start) / 1000.0; + tqDebug("s-task:%s scan-history stage(step 1) ended, elapsed time:%.2fs", id, el); - void* serializedReq = rpcMallocCont(len); - if (serializedReq == NULL) { - tqError("s-task:%s failed to prepare the step2 stage, out of memory", pTask->id.idStr); - return -1; - } + if (pTask->info.fillHistory) { + SVersionRange* pRange = NULL; + SStreamTask* pStreamTask = NULL; + bool done = false; + + // 1. get the related stream task + pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); + if (pStreamTask == NULL) { + // todo delete this task, if the related stream task is dropped + qError("failed to find s-task:0x%x, it may have been destroyed, drop fill-history task:%s", + pTask->streamTaskId.taskId, pTask->id.idStr); - memcpy(serializedReq, &req, len); + tqDebug("s-task:%s fill-history task set status to be dropping", id); - // dispatch msg - tqDebug("s-task:%s step 1 finished, send msg to start blocking recover stage(step 2)", pTask->id.idStr); + streamMetaUnregisterTask(pMeta, pTask->id.streamId, pTask->id.taskId); + streamMetaReleaseTask(pMeta, pTask); + return -1; + } + + ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE); + + // 2. it cannot be paused, when the stream task in TASK_STATUS__SCAN_HISTORY status. Let's wait for the + // stream task get ready for scan history data + while (pStreamTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) { + tqDebug( + "s-task:%s level:%d related stream task:%s(status:%s) not ready for halt, wait for it and recheck in 100ms", + id, pTask->info.taskLevel, pStreamTask->id.idStr, streamGetTaskStatusStr(pStreamTask->status.taskStatus)); + taosMsleep(100); + } + + // now we can stop the stream task execution + streamTaskHalt(pStreamTask); + + tqDebug("s-task:%s level:%d sched-status:%d is halt by fill-history task:%s", pStreamTask->id.idStr, + pStreamTask->info.taskLevel, pStreamTask->status.schedStatus, id); + + // if it's an source task, extract the last version in wal. + pRange = &pTask->dataRange.range; + int64_t latestVer = walReaderGetCurrentVer(pStreamTask->exec.pWalReader); + done = streamHistoryTaskSetVerRangeStep2(pTask, latestVer); + + if (done) { + pTask->tsInfo.step2Start = taosGetTimestampMs(); + streamTaskEndScanWAL(pTask); + } else { + STimeWindow* pWindow = &pTask->dataRange.window; + tqDebug("s-task:%s level:%d verRange:%" PRId64 " - %" PRId64 " window:%" PRId64 "-%" PRId64 + ", do secondary scan-history from WAL after halt the related stream task:%s", + id, pTask->info.taskLevel, pRange->minVer, pRange->maxVer, pWindow->skey, pWindow->ekey, + pStreamTask->id.idStr); + ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__WAITING); + + pTask->tsInfo.step2Start = taosGetTimestampMs(); + streamSetParamForStreamScannerStep2(pTask, pRange, pWindow); + + int64_t dstVer = pTask->dataRange.range.minVer - 1; + + pTask->chkInfo.currentVer = dstVer; + walReaderSetSkipToVersion(pTask->exec.pWalReader, dstVer); + tqDebug("s-task:%s wal reader start scan WAL verRange:%" PRId64 "-%" PRId64 ", set sched-status:%d", id, dstVer, + pTask->dataRange.range.maxVer, TASK_SCHED_STATUS__INACTIVE); + + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + + // set the fill-history task to be normal + if (pTask->info.fillHistory == 1) { + streamSetStatusNormal(pTask); + } + + tqStartStreamTasks(pTq); + } + + streamMetaReleaseTask(pMeta, pTask); + streamMetaReleaseTask(pMeta, pStreamTask); + } else { + // todo update the chkInfo version for current task. + // this task has an associated history stream task, so we need to scan wal from the end version of + // history scan. The current version of chkInfo.current is not updated during the history scan + STimeWindow* pWindow = &pTask->dataRange.window; + + if (pTask->historyTaskId.taskId == 0) { + *pWindow = (STimeWindow){INT64_MIN, INT64_MAX}; + tqDebug( + "s-task:%s scan-history in stream time window completed, no related fill-history task, reset the time " + "window:%" PRId64 " - %" PRId64, + id, pWindow->skey, pWindow->ekey); + qStreamInfoResetTimewindowFilter(pTask->exec.pExecutor); + } else { + // when related fill-history task exists, update the fill-history time window only when the + // state transfer is completed. + tqDebug( + "s-task:%s scan-history in stream time window completed, now start to handle data from WAL, start " + "ver:%" PRId64 ", window:%" PRId64 " - %" PRId64, + id, pTask->chkInfo.currentVer, pWindow->skey, pWindow->ekey); + } + + // notify the downstream agg tasks that upstream tasks are ready to processing the WAL data, update the + code = streamTaskScanHistoryDataComplete(pTask); + streamMetaReleaseTask(pMeta, pTask); + + // when all source task complete to scan history data in stream time window, they are allowed to handle stream data + // at the same time. + return code; + } - SRpcMsg rpcMsg = { - .code = 0, .contLen = len, .msgType = TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, .pCont = serializedReq}; - tmsgPutToQueue(&pTq->pVnode->msgCb, WRITE_QUEUE, &rpcMsg); return 0; } -int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - int32_t code = 0; +// notify the downstream tasks to transfer executor state after handle all history blocks. +int32_t tqProcessTaskTransferStateReq(STQ* pTq, SRpcMsg* pMsg) { + char* pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); - SStreamRecoverStep2Req* pReq = (SStreamRecoverStep2Req*)msg; + SStreamTransferReq req = {0}; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); - if (pTask == NULL) { - return -1; - } + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)pReq, len); + int32_t code = tDecodeStreamScanHistoryFinishReq(&decoder, &req); + tDecoderClear(&decoder); - // do recovery step 2 - int64_t st = taosGetTimestampMs(); - tqDebug("s-task:%s start step2 recover, ts:%"PRId64, pTask->id.idStr, st); + tqDebug("vgId:%d start to process transfer state msg, from s-task:0x%x", pTq->pStreamMeta->vgId, req.downstreamTaskId); - code = streamSourceRecoverScanStep2(pTask, sversion); - if (code < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.streamId, req.downstreamTaskId); + if (pTask == NULL) { + tqError("failed to find task:0x%x, it may have been dropped already. process transfer state failed", req.downstreamTaskId); return -1; } - if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { + int32_t remain = streamAlignTransferState(pTask); + if (remain > 0) { + tqDebug("s-task:%s receive upstream transfer state msg, remain:%d", pTask->id.idStr, remain); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } - // restore param - code = streamRestoreParam(pTask); - if (code < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; - } + // transfer the ownership of executor state + tqDebug("s-task:%s all upstream tasks send transfer msg, open transfer state flag", pTask->id.idStr); + ASSERT(pTask->streamTaskId.taskId != 0 && pTask->info.fillHistory == 1); - // set status normal - tqDebug("s-task:%s blocking stage completed, set the status to be normal", pTask->id.idStr); - code = streamSetStatusNormal(pTask); - if (code < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; - } + pTask->status.transferState = true; - double el = (taosGetTimestampMs() - st)/ 1000.0; - tqDebug("s-task:%s step2 recover finished, el:%.2fs", pTask->id.idStr, el); + streamSchedExec(pTask); + streamMetaReleaseTask(pTq->pStreamMeta, pTask); + return 0; +} - // dispatch recover finish req to all related downstream task - code = streamDispatchRecoverFinishReq(pTask); - if (code < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); +int32_t tqProcessTaskScanHistoryFinishReq(STQ* pTq, SRpcMsg* pMsg) { + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + // deserialize + SStreamScanHistoryFinishReq req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + tDecodeStreamScanHistoryFinishReq(&decoder, &req); + tDecoderClear(&decoder); + + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.streamId, req.downstreamTaskId); + if (pTask == NULL) { + tqError("vgId:%d process scan history finish msg, failed to find task:0x%x, it may be destroyed", + pTq->pStreamMeta->vgId, req.downstreamTaskId); return -1; } - atomic_store_8(&pTask->fillHistory, 0); - streamMetaSaveTask(pTq->pStreamMeta, pTask); + tqDebug("s-task:%s receive scan-history finish msg from task:0x%x", pTask->id.idStr, req.upstreamTaskId); + int32_t code = streamProcessScanHistoryFinishReq(pTask, &req, &pMsg->info); streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return 0; + return code; } -int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg) { +int32_t tqProcessTaskScanHistoryFinishRsp(STQ* pTq, SRpcMsg* pMsg) { char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); // deserialize - SStreamRecoverFinishReq req; + SStreamCompleteHistoryMsg req = {0}; SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - tDecodeSStreamRecoverFinishReq(&decoder, &req); + tDecodeCompleteHistoryDataMsg(&decoder, &req); tDecoderClear(&decoder); - // find task - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.taskId); + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.streamId, req.upstreamTaskId); if (pTask == NULL) { + tqError("vgId:%d process scan history finish rsp, failed to find task:0x%x, it may be destroyed", + pTq->pStreamMeta->vgId, req.upstreamTaskId); return -1; } - // do process request - if (streamProcessRecoverFinishReq(pTask, req.childId) < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; + + int32_t remain = atomic_sub_fetch_32(&pTask->notReadyTasks, 1); + if (remain > 0) { + tqDebug("s-task:%s scan-history finish rsp received from downstream task:0x%x, remain:%d not send finish rsp", + pTask->id.idStr, req.downstreamId, remain); + } else { + tqDebug( + "s-task:%s scan-history finish rsp received from downstream task:0x%x, all downstream tasks rsp scan-history " + "completed msg", pTask->id.idStr, req.downstreamId); + streamProcessScanHistoryFinishRsp(pTask); } streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } -int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg) { - // - return 0; -} - -int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { - bool failed = false; +int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, SStreamRefDataBlock** pRefBlock) { SDecoder* pCoder = &(SDecoder){0}; SDeleteRes* pRes = &(SDeleteRes){0}; + (*pRefBlock) = NULL; + pRes->uidList = taosArrayInit(0, sizeof(tb_uid_t)); if (pRes->uidList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - failed = true; + return TSDB_CODE_OUT_OF_MEMORY; } - tDecoderInit(pCoder, pReq, len); + tDecoderInit(pCoder, (uint8_t*)pData, len); tDecodeDeleteRes(pCoder, pRes); tDecoderClear(pCoder); - int32_t sz = taosArrayGetSize(pRes->uidList); - if (sz == 0 || pRes->affectedRows == 0) { + int32_t numOfTables = taosArrayGetSize(pRes->uidList); + if (numOfTables == 0 || pRes->affectedRows == 0) { taosArrayDestroy(pRes->uidList); - return 0; + return TSDB_CODE_SUCCESS; } + SSDataBlock* pDelBlock = createSpecialDataBlock(STREAM_DELETE_DATA); - blockDataEnsureCapacity(pDelBlock, sz); - pDelBlock->info.rows = sz; + blockDataEnsureCapacity(pDelBlock, numOfTables); + pDelBlock->info.rows = numOfTables; pDelBlock->info.version = ver; - for (int32_t i = 0; i < sz; i++) { + for (int32_t i = 0; i < numOfTables; i++) { // start key column SColumnInfoData* pStartCol = taosArrayGet(pDelBlock->pDataBlock, START_TS_COLUMN_INDEX); colDataSetVal(pStartCol, i, (const char*)&pRes->skey, false); // end key column @@ -1182,116 +1480,14 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { } taosArrayDestroy(pRes->uidList); - - int32_t* pRef = taosMemoryMalloc(sizeof(int32_t)); - *pRef = 1; - - taosWLockLatch(&pTq->pStreamMeta->lock); - - void* pIter = NULL; - while (1) { - pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); - if (pIter == NULL) { - break; - } - - SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->taskLevel != TASK_LEVEL__SOURCE) { - continue; - } - - qDebug("s-task:%s delete req enqueue, ver: %" PRId64, pTask->id.idStr, ver); - - if (!failed) { - SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); - pRefBlock->type = STREAM_INPUT__REF_DATA_BLOCK; - pRefBlock->pBlock = pDelBlock; - pRefBlock->dataRef = pRef; - atomic_add_fetch_32(pRefBlock->dataRef, 1); - - if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) { - atomic_sub_fetch_32(pRef, 1); - taosFreeQitem(pRefBlock); - continue; - } - - if (streamSchedExec(pTask) < 0) { - qError("s-task:%s stream task launch failed", pTask->id.idStr); - continue; - } - - } else { - streamTaskInputFail(pTask); - } - } - - taosWUnLockLatch(&pTq->pStreamMeta->lock); - - int32_t ref = atomic_sub_fetch_32(pRef, 1); - if (ref == 0) { - blockDataDestroy(pDelBlock); - taosMemoryFree(pRef); - } - -#if 0 - SStreamDataBlock* pStreamBlock = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); - pStreamBlock->type = STREAM_INPUT__DATA_BLOCK; - pStreamBlock->blocks = taosArrayInit(0, sizeof(SSDataBlock)); - SSDataBlock block = {0}; - assignOneDataBlock(&block, pDelBlock); - block.info.type = STREAM_DELETE_DATA; - taosArrayPush(pStreamBlock->blocks, &block); - - if (!failed) { - if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pStreamBlock) < 0) { - qError("stream task input del failed, task id %d", pTask->id.taskId); - continue; - } - - if (streamSchedExec(pTask) < 0) { - qError("stream task launch failed, task id %d", pTask->id.taskId); - continue; - } - } else { - streamTaskInputFail(pTask); - } + *pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); + if ((*pRefBlock) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; } - blockDataDestroy(pDelBlock); -#endif - return 0; -} - -int32_t tqProcessSubmitReqForSubscribe(STQ* pTq) { - int32_t vgId = TD_VID(pTq->pVnode); - - taosWLockLatch(&pTq->lock); - - if (taosHashGetSize(pTq->pPushMgr) > 0) { - void* pIter = taosHashIterate(pTq->pPushMgr, NULL); - - while (pIter) { - STqHandle* pHandle = *(STqHandle**)pIter; - tqDebug("vgId:%d start set submit for pHandle:%p, consumer:0x%" PRIx64, vgId, pHandle, pHandle->consumerId); - - if (ASSERT(pHandle->msg != NULL)) { - tqError("pHandle->msg should not be null"); - break; - }else{ - SRpcMsg msg = {.msgType = TDMT_VND_TMQ_CONSUME, .pCont = pHandle->msg->pCont, .contLen = pHandle->msg->contLen, .info = pHandle->msg->info}; - tmsgPutToQueue(&pTq->pVnode->msgCb, QUERY_QUEUE, &msg); - taosMemoryFree(pHandle->msg); - pHandle->msg = NULL; - } - pIter = taosHashIterate(pTq->pPushMgr, pIter); - } - - taosHashClear(pTq->pPushMgr); - } - - // unlock - taosWUnLockLatch(&pTq->lock); - return 0; + (*pRefBlock)->type = STREAM_INPUT__REF_DATA_BLOCK; + (*pRefBlock)->pBlock = pDelBlock; + return TSDB_CODE_SUCCESS; } int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { @@ -1300,43 +1496,55 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { int32_t taskId = pReq->taskId; int32_t vgId = TD_VID(pTq->pVnode); - if (taskId == WAL_READ_TASKS_ID) { // all tasks are extracted submit data from the wal + if (taskId == STREAM_TASK_STATUS_CHECK_ID) { + tqStreamTasksStatusCheck(pTq); + return 0; + } + + if (taskId == EXTRACT_DATA_FROM_WAL_ID) { // all tasks are extracted submit data from the wal tqStreamTasksScanWal(pTq); return 0; } - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, taskId); if (pTask != NULL) { - if (pTask->status.taskStatus == TASK_STATUS__NORMAL) { - tqDebug("vgId:%d s-task:%s start to process block from wal, last chk point:%" PRId64, vgId, - pTask->id.idStr, pTask->chkInfo.version); + // even in halt status, the data in inputQ must be processed + int8_t st = pTask->status.taskStatus; + if (st == TASK_STATUS__NORMAL || st == TASK_STATUS__SCAN_HISTORY) { + tqDebug("vgId:%d s-task:%s start to process block from inputQ, last chk point:%" PRId64, vgId, pTask->id.idStr, + pTask->chkInfo.version); streamProcessRunReq(pTask); } else { - tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr); + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, + pTask->id.idStr, streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus); } streamMetaReleaseTask(pTq->pStreamMeta, pTask); tqStartStreamTasks(pTq); return 0; - } else { - tqError("vgId:%d failed to found s-task, taskId:%d", vgId, taskId); + } else { // NOTE: pTask->status.schedStatus is not updated since it is not be handled by the run exec. + // todo add one function to handle this + tqError("vgId:%d failed to found s-task, taskId:0x%x may have been dropped", vgId, taskId); return -1; } } int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { - char* msgStr = pMsg->pCont; - char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - SStreamDispatchReq req; - SDecoder decoder; + char* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + SStreamDispatchReq req = {0}; + + SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecodeStreamDispatchReq(&decoder, &req); - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.taskId); + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.streamId, req.taskId); if (pTask) { SRpcMsg rsp = {.info = pMsg->info, .code = 0}; - streamProcessDispatchReq(pTask, &req, &rsp, exec); + streamProcessDispatchMsg(pTask, &req, &rsp, exec); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } else { @@ -1347,73 +1555,143 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t taskId = ntohl(pRsp->upstreamTaskId); - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - tqDebug("recv dispatch rsp, code:%x", pMsg->code); + + int32_t vgId = pTq->pStreamMeta->vgId; + int32_t taskId = htonl(pRsp->upstreamTaskId); + int64_t streamId = htobe64(pRsp->streamId); + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, streamId, taskId); + if (pTask) { streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } else { - return -1; + tqDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, taskId); + return TSDB_CODE_INVALID_MSG; } } int32_t tqProcessTaskDropReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; - streamMetaRemoveTask(pTq->pStreamMeta, pReq->taskId); + tqDebug("vgId:%d receive msg to drop stream task:0x%x", TD_VID(pTq->pVnode), pReq->taskId); + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, pReq->taskId); + if (pTask == NULL) { + tqError("vgId:%d failed to acquire s-task:0x%x when dropping it", pTq->pStreamMeta->vgId, pReq->taskId); + return 0; + } + + streamMetaUnregisterTask(pTq->pStreamMeta, pReq->streamId, pReq->taskId); + streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)msg; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); - if (pTask) { - tqDebug("vgId:%d s-task:%s set pause flag", pTq->pStreamMeta->vgId, pTask->id.idStr); - atomic_store_8(&pTask->status.keepTaskStatus, pTask->status.taskStatus); - atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__PAUSE); - streamMetaReleaseTask(pTq->pStreamMeta, pTask); + + SStreamMeta* pMeta = pTq->pStreamMeta; + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + if (pTask == NULL) { + tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, + pReq->taskId); + // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active + return TSDB_CODE_SUCCESS; } - return 0; + + tqDebug("s-task:%s receive pause msg from mnode", pTask->id.idStr); + streamTaskPause(pTask); + + SStreamTask* pHistoryTask = NULL; + if (pTask->historyTaskId.taskId != 0) { + pHistoryTask = streamMetaAcquireTask(pMeta, pTask->historyTaskId.streamId, pTask->historyTaskId.taskId); + if (pHistoryTask == NULL) { + tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%x, it may have been dropped already", + pMeta->vgId, pTask->historyTaskId.taskId); + streamMetaReleaseTask(pMeta, pTask); + + // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active + return TSDB_CODE_SUCCESS; + } + + tqDebug("s-task:%s fill-history task handle paused along with related stream task", pHistoryTask->id.idStr); + + streamTaskPause(pHistoryTask); + streamMetaReleaseTask(pMeta, pHistoryTask); + } + + streamMetaReleaseTask(pMeta, pTask); + return TSDB_CODE_SUCCESS; } -int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - SVResumeStreamTaskReq* pReq = (SVResumeStreamTaskReq*)msg; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); - if (pTask) { - atomic_store_8(&pTask->status.taskStatus, pTask->status.keepTaskStatus); +int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion, int8_t igUntreated) { + int32_t vgId = pTq->pStreamMeta->vgId; + if (pTask == NULL) { + return -1; + } + + // todo: handle the case: resume from halt to pause/ from halt to normal/ from pause to normal + streamTaskResume(pTask); + int32_t level = pTask->info.taskLevel; + int8_t status = pTask->status.taskStatus; + if (status == TASK_STATUS__NORMAL || status == TASK_STATUS__SCAN_HISTORY) { // no lock needs to secure the access of the version - if (pReq->igUntreated) { // discard all the data when the stream task is suspended. - pTask->chkInfo.currentVer = sversion; - tqDebug("vgId:%d s-task:%s resume to normal from the latest version:%" PRId64 ", vnode ver:%" PRId64, pTq->pStreamMeta->vgId, - pTask->id.idStr, pTask->chkInfo.currentVer, sversion); + if (igUntreated && level == TASK_LEVEL__SOURCE && !pTask->info.fillHistory) { + // discard all the data when the stream task is suspended. + walReaderSetSkipToVersion(pTask->exec.pWalReader, sversion); + tqDebug("vgId:%d s-task:%s resume to exec, prev paused version:%" PRId64 ", start from vnode ver:%" PRId64 + ", schedStatus:%d", + vgId, pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); } else { // from the previous paused version and go on - tqDebug("vgId:%d s-task:%s resume to normal from paused ver:%" PRId64 ", vnode ver:%" PRId64, pTq->pStreamMeta->vgId, - pTask->id.idStr, pTask->chkInfo.currentVer, sversion); + tqDebug("vgId:%d s-task:%s resume to exec, from paused ver:%" PRId64 ", vnode ver:%" PRId64 ", schedStatus:%d", + vgId, pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); } - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - tqStartStreamTasks(pTq); + if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) { + streamStartScanHistoryAsync(pTask, igUntreated); + } else if (level == TASK_LEVEL__SOURCE && (taosQueueItemSize(pTask->inputQueue->queue) == 0)) { + tqStartStreamTasks(pTq); + } else { + streamSchedExec(pTask); + } } + streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } +int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { + SVResumeStreamTaskReq* pReq = (SVResumeStreamTaskReq*)msg; + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, pReq->taskId); + int32_t code = tqProcessTaskResumeImpl(pTq, pTask, sversion, pReq->igUntreated); + if (code != 0) { + return code; + } + + SStreamTask* pHistoryTask = streamMetaAcquireTask(pTq->pStreamMeta, pTask->historyTaskId.streamId, pTask->historyTaskId.taskId); + if (pHistoryTask) { + code = tqProcessTaskResumeImpl(pTq, pHistoryTask, sversion, pReq->igUntreated); + } + + return code; +} + int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { - char* msgStr = pMsg->pCont; - char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + char* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + SDecoder decoder; + SStreamRetrieveReq req; - SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecodeStreamRetrieveReq(&decoder, &req); tDecoderClear(&decoder); - int32_t taskId = req.dstTaskId; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); + + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.streamId, req.dstTaskId); + if (pTask) { SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamProcessRetrieveReq(pTask, &req, &rsp); + streamMetaReleaseTask(pTq->pStreamMeta, pTask); tDeleteStreamRetrieveReq(&req); return 0; @@ -1446,28 +1724,31 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { tDecoderClear(&decoder); int32_t taskId = req.taskId; - - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - if (pTask) { + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.streamId, req.taskId); + if (pTask != NULL) { SRpcMsg rsp = {.info = pMsg->info, .code = 0}; - streamProcessDispatchReq(pTask, &req, &rsp, false); + streamProcessDispatchMsg(pTask, &req, &rsp, false); streamMetaReleaseTask(pTq->pStreamMeta, pTask); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); return 0; } else { + tDeleteStreamDispatchReq(&req); } code = TSDB_CODE_STREAM_TASK_NOT_EXIST; FAIL: - if (pMsg->info.handle == NULL) return -1; + if (pMsg->info.handle == NULL) { + tqError("s-task:0x%x vgId:%d msg handle is null, abort enqueue dispatch msg", pTq->pStreamMeta->vgId, taskId); + return -1; + } SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp)); if (pRspHead == NULL) { SRpcMsg rsp = {.code = TSDB_CODE_OUT_OF_MEMORY, .info = pMsg->info}; - tqDebug("send dispatch error rsp, code: %x", code); + tqError("s-task:0x%x send dispatch error rsp, code:%s", taskId, tstrerror(code)); tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); @@ -1483,9 +1764,10 @@ FAIL: pRsp->downstreamTaskId = htonl(req.taskId); pRsp->inputStatus = TASK_OUTPUT_STATUS__NORMAL; - SRpcMsg rsp = { - .code = code, .info = pMsg->info, .contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp), .pCont = pRspHead}; - tqDebug("send dispatch error rsp, code: %x", code); + int32_t len = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); + SRpcMsg rsp = { .code = code, .info = pMsg->info, .contLen = len, .pCont = pRspHead}; + tqError("s-task:0x%x send dispatch error rsp, code:%s", taskId, tstrerror(code)); + tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); @@ -1494,43 +1776,3 @@ FAIL: int32_t tqCheckLogInWal(STQ* pTq, int64_t sversion) { return sversion <= pTq->walLogLastVer; } -int32_t tqStartStreamTasks(STQ* pTq) { - int32_t vgId = TD_VID(pTq->pVnode); - SStreamMeta* pMeta = pTq->pStreamMeta; - - taosWLockLatch(&pMeta->lock); - - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - if (numOfTasks == 0) { - tqInfo("vgId:%d no stream tasks exist", vgId); - taosWUnLockLatch(&pTq->pStreamMeta->lock); - return 0; - } - - pMeta->walScanCounter += 1; - - if (pMeta->walScanCounter > 1) { - tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->walScanCounter); - taosWUnLockLatch(&pTq->pStreamMeta->lock); - return 0; - } - - SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); - if (pRunReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); - taosWUnLockLatch(&pTq->pStreamMeta->lock); - return -1; - } - - tqDebug("vgId:%d create msg to start wal scan to launch stream tasks, numOfTasks:%d", vgId, numOfTasks); - pRunReq->head.vgId = vgId; - pRunReq->streamId = 0; - pRunReq->taskId = WAL_READ_TASKS_ID; - - SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; - tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); - taosWUnLockLatch(&pTq->pStreamMeta->lock); - - return 0; -} diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 5654147b6df6d1518285e0c028e23db20997ff57..85151c6e1944a61d1bdd7f43b6f82bca2963f0eb 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -37,6 +37,9 @@ int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) { } } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (tEncodeI64(pEncoder, pHandle->execHandle.execTb.suid) < 0) return -1; + if (pHandle->execHandle.execTb.qmsg != NULL){ + if (tEncodeCStr(pEncoder, pHandle->execHandle.execTb.qmsg) < 0) return -1; + } } tEndEncode(pEncoder); return pEncoder->pos; @@ -64,6 +67,9 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) { } } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (tDecodeI64(pDecoder, &pHandle->execHandle.execTb.suid) < 0) return -1; + if (!tDecodeIsEnd(pDecoder)){ + if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.execTb.qmsg) < 0) return -1; + } } tEndDecode(pDecoder); return 0; @@ -82,9 +88,9 @@ int32_t tqMetaOpen(STQ* pTq) { return -1; } - if (tqMetaRestoreHandle(pTq) < 0) { - return -1; - } +// if (tqMetaRestoreHandle(pTq) < 0) { +// return -1; +// } if (tqMetaRestoreCheckInfo(pTq) < 0) { return -1; @@ -268,6 +274,120 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { return 0; } +static int buildHandle(STQ* pTq, STqHandle* handle){ + SVnode* pVnode = pTq->pVnode; + int32_t vgId = TD_VID(pVnode); + + handle->pRef = walOpenRef(pVnode->pWal); + if (handle->pRef == NULL) { + return -1; + } + walSetRefVer(handle->pRef, handle->snapshotVer); + + SReadHandle reader = { + .vnode = pVnode, + .initTableReader = true, + .initTqReader = true, + .version = handle->snapshotVer, + }; + + initStorageAPI(&reader.api); + + if (handle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + handle->execHandle.task = + qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId, &handle->execHandle.numOfCols, handle->consumerId); + if (handle->execHandle.task == NULL) { + tqError("cannot create exec task for %s", handle->subKey); + return -1; + } + void* scanner = NULL; + qExtractStreamScanner(handle->execHandle.task, &scanner); + if (scanner == NULL) { + tqError("cannot extract stream scanner for %s", handle->subKey); + return -1; + } + handle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); + if (handle->execHandle.pTqReader == NULL) { + tqError("cannot extract exec reader for %s", handle->subKey); + return -1; + } + } else if (handle->execHandle.subType == TOPIC_SUB_TYPE__DB) { + handle->pWalReader = walOpenReader(pVnode->pWal, NULL); + handle->execHandle.pTqReader = tqReaderOpen(pVnode); + + buildSnapContext(reader.vnode, reader.version, 0, handle->execHandle.subType, handle->fetchMeta, + (SSnapContext**)(&reader.sContext)); + handle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, handle->consumerId); + } else if (handle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { + handle->pWalReader = walOpenReader(pVnode->pWal, NULL); + + if(handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) { + if (nodesStringToNode(handle->execHandle.execTb.qmsg, &handle->execHandle.execTb.node) != 0) { + tqError("nodesStringToNode error in sub stable, since %s", terrstr()); + return -1; + } + } + buildSnapContext(reader.vnode, reader.version, handle->execHandle.execTb.suid, handle->execHandle.subType, + handle->fetchMeta, (SSnapContext**)(&reader.sContext)); + handle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, handle->consumerId); + + SArray* tbUidList = NULL; + int ret = qGetTableList(handle->execHandle.execTb.suid, pVnode, handle->execHandle.execTb.node, &tbUidList, handle->execHandle.task); + if(ret != TDB_CODE_SUCCESS) { + tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle->subKey, handle->consumerId); + taosArrayDestroy(tbUidList); + return -1; + } + tqInfo("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, handle->execHandle.execTb.suid); + handle->execHandle.pTqReader = tqReaderOpen(pVnode); + tqReaderSetTbUidList(handle->execHandle.pTqReader, tbUidList, NULL); + taosArrayDestroy(tbUidList); + } + return 0; +} + +static int restoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle){ + int32_t vgId = TD_VID(pTq->pVnode); + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)pVal, vLen); + tDecodeSTqHandle(&decoder, handle); + tDecoderClear(&decoder); + + if(buildHandle(pTq, handle) < 0){ + return -1; + } + tqInfo("restoreHandle %s consumer 0x%" PRIx64 " vgId:%d", handle->subKey, handle->consumerId, vgId); + return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle)); +} + +int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){ + int32_t vgId = TD_VID(pTq->pVnode); + + memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN); + handle->consumerId = req->newConsumerId; + handle->epoch = -1; + + handle->execHandle.subType = req->subType; + handle->fetchMeta = req->withMeta; + if(req->subType == TOPIC_SUB_TYPE__COLUMN){ + handle->execHandle.execCol.qmsg = taosStrdup(req->qmsg); + }else if(req->subType == TOPIC_SUB_TYPE__DB){ + handle->execHandle.execDb.pFilterOutTbUid = + taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + }else if(req->subType == TOPIC_SUB_TYPE__TABLE){ + handle->execHandle.execTb.suid = req->suid; + handle->execHandle.execTb.qmsg = taosStrdup(req->qmsg); + } + + handle->snapshotVer = walGetCommittedVer(pTq->pVnode->pWal); + + if(buildHandle(pTq, handle) < 0){ + return -1; + } + tqInfo("tqCreateHandle %s consumer 0x%" PRIx64 " vgId:%d", handle->subKey, handle->consumerId, vgId); + return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle)); +} + int32_t tqMetaRestoreHandle(STQ* pTq) { int code = 0; TBC* pCur = NULL; @@ -275,91 +395,40 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { return -1; } - int32_t vgId = TD_VID(pTq->pVnode); void* pKey = NULL; int kLen = 0; void* pVal = NULL; int vLen = 0; - SDecoder decoder; tdbTbcMoveToFirst(pCur); while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { STqHandle handle = {0}; - tDecoderInit(&decoder, (uint8_t*)pVal, vLen); - tDecodeSTqHandle(&decoder, &handle); - tDecoderClear(&decoder); - - handle.pRef = walOpenRef(pTq->pVnode->pWal); - if (handle.pRef == NULL) { - code = -1; - goto end; - } - walSetRefVer(handle.pRef, handle.snapshotVer); - - SReadHandle reader = { - .meta = pTq->pVnode->pMeta, - .vnode = pTq->pVnode, - .initTableReader = true, - .initTqReader = true, - .version = handle.snapshotVer, - }; - - if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - handle.execHandle.task = - qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, 0); - if (handle.execHandle.task == NULL) { - tqError("cannot create exec task for %s", handle.subKey); - code = -1; - goto end; - } - void* scanner = NULL; - qExtractStreamScanner(handle.execHandle.task, &scanner); - if (scanner == NULL) { - tqError("cannot extract stream scanner for %s", handle.subKey); - code = -1; - goto end; - } - handle.execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); - if (handle.execHandle.pTqReader == NULL) { - tqError("cannot extract exec reader for %s", handle.subKey); - code = -1; - goto end; - } - } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__DB) { - handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode); - - buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta, - (SSnapContext**)(&reader.sContext)); - handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); - } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) { - handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - - SArray* tbUidList = taosArrayInit(0, sizeof(int64_t)); - vnodeGetCtbIdList(pTq->pVnode, handle.execHandle.execTb.suid, tbUidList); - tqDebug("vgId:%d, tq try to get all ctb, suid:%" PRId64, pTq->pVnode->config.vgId, handle.execHandle.execTb.suid); - for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { - int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); - } - handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode); - tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList); - taosArrayDestroy(tbUidList); - - buildSnapContext(reader.meta, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, - handle.fetchMeta, (SSnapContext**)(&reader.sContext)); - handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); + code = restoreHandle(pTq, pVal, vLen, &handle); + if (code < 0) { + tqDestroyTqHandle(&handle); + break; } - tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, vgId); - taosWLockLatch(&pTq->lock); - taosHashPut(pTq->pHandle, pKey, kLen, &handle, sizeof(STqHandle)); - taosWUnLockLatch(&pTq->lock); } -end: tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); return code; } + +int32_t tqMetaGetHandle(STQ* pTq, const char* key) { + void* pVal = NULL; + int vLen = 0; + + if (tdbTbGet(pTq->pExecStore, key, (int)strlen(key), &pVal, &vLen) < 0) { + return -1; + } + STqHandle handle = {0}; + int code = restoreHandle(pTq, pVal, vLen, &handle); + if (code < 0){ + tqDestroyTqHandle(&handle); + } + tdbFree(pVal); + return code; +} diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 377a5d18875c77b38d4ca6036a96cabfafa0ea3f..11bb737225d39ccd7c9f7ba0be2c6f0abc0eda42 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -78,7 +78,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { // todo remove this if (offset.val.type == TMQ_OFFSET__LOG) { - taosWLockLatch(&pStore->pTq->lock); STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey)); if (pHandle) { if (walSetRefVer(pHandle->pRef, offset.val.version) < 0) { @@ -86,7 +85,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { // offset.val.version); } } - taosWUnLockLatch(&pStore->pTq->lock); } taosMemoryFree(pMemBuf); @@ -106,7 +104,7 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) { pStore->needCommit = 0; pTq->pOffsetStore = pStore; - pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); + pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); if (pStore->pHash == NULL) { taosMemoryFree(pStore); return NULL; diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index e1e9bec34818636da45df11cdc9341bbe9bc852a..070a8ecf6fa95a3cd4303f3b4606d81c2940775c 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -16,14 +16,30 @@ #include "tq.h" #include "vnd.h" -int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { +int32_t tqProcessSubmitReqForSubscribe(STQ* pTq) { + if (taosHashGetSize(pTq->pPushMgr) <= 0) { + return 0; + } + SRpcMsg msg = {.msgType = TDMT_VND_TMQ_CONSUME_PUSH}; + msg.pCont = rpcMallocCont(sizeof(SMsgHead)); + msg.contLen = sizeof(SMsgHead); + SMsgHead *pHead = msg.pCont; + pHead->vgId = TD_VID(pTq->pVnode); + pHead->contLen = msg.contLen; + tmsgPutToQueue(&pTq->pVnode->msgCb, QUERY_QUEUE, &msg); + return 0; +} +int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { if (msgType == TDMT_VND_SUBMIT) { tqProcessSubmitReqForSubscribe(pTq); } + taosRLockLatch(&pTq->pStreamMeta->lock); int32_t numOfTasks = streamMetaGetNumOfTasks(pTq->pStreamMeta); - tqDebug("handle submit, restore:%d, size:%d", pTq->pVnode->restored, numOfTasks); + taosRUnLockLatch(&pTq->pStreamMeta->lock); + + tqTrace("handle submit, restore:%d, size:%d", pTq->pVnode->restored, numOfTasks); // push data for stream processing: // 1. the vnode has already been restored. @@ -34,13 +50,9 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v return 0; } - if (msgType == TDMT_VND_SUBMIT) { + if (msgType == TDMT_VND_SUBMIT || msgType == TDMT_VND_DELETE) { tqStartStreamTasks(pTq); } - - if (msgType == TDMT_VND_DELETE) { - tqProcessDelReq(pTq, POINTER_SHIFT(msg, sizeof(SMsgHead)), msgLen - sizeof(SMsgHead), ver); - } } return 0; @@ -55,7 +67,9 @@ int32_t tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg) { memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg)); pHandle->msg->pCont = rpcMallocCont(pMsg->contLen); } else { - tqPushDataRsp(pHandle, vgId); +// tqPushDataRsp(pHandle, vgId); + tqPushEmptyDataRsp(pHandle, vgId); + void* tmp = pHandle->msg->pCont; memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg)); pHandle->msg->pCont = tmp; @@ -64,20 +78,24 @@ int32_t tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg) { memcpy(pHandle->msg->pCont, pMsg->pCont, pMsg->contLen); pHandle->msg->contLen = pMsg->contLen; int32_t ret = taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey), &pHandle, POINTER_BYTES); - tqDebug("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64 ", register to pHandle:%p, pCont:%p, len:%d", vgId, ret, + tqInfo("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64 ", register to pHandle:%p, pCont:%p, len:%d", vgId, ret, pHandle->consumerId, pHandle, pHandle->msg->pCont, pHandle->msg->contLen); return 0; } -int32_t tqUnregisterPushHandle(STQ* pTq, void *handle) { +int tqUnregisterPushHandle(STQ* pTq, void *handle) { STqHandle *pHandle = (STqHandle*)handle; int32_t vgId = TD_VID(pTq->pVnode); + if(taosHashGetSize(pTq->pPushMgr) <= 0) { + return 0; + } int32_t ret = taosHashRemove(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey)); - tqError("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); + tqInfo("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); if(pHandle->msg != NULL) { - tqPushDataRsp(pHandle, vgId); +// tqPushDataRsp(pHandle, vgId); + tqPushEmptyDataRsp(pHandle, vgId); rpcFreeCont(pHandle->msg->pCont); taosMemoryFree(pHandle->msg); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index f7c37f229d74667f2201ade4c4b937f65972f6dc..9b8f1781cb47182156d7834adb1cbb2c5fbf2050 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -114,7 +114,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) { } SMetaReader mr = {0}; - metaReaderInit(&mr, pHandle->execHandle.pTqReader->pVnodeMeta, 0); + metaReaderDoInit(&mr, pHandle->execHandle.pTqReader->pVnodeMeta, 0); if (metaGetTableEntryByName(&mr, req.tbName) < 0) { metaReaderClear(&mr); @@ -196,7 +196,7 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return, reqId:0x%" PRIx64, pHandle->consumerId, pHandle->epoch, vgId, offset, reqId); - *fetchOffset = offset - 1; + *fetchOffset = offset; code = -1; goto END; } @@ -216,9 +216,9 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea code = 0; goto END; } else { - if (pHandle->fetchMeta) { + if (pHandle->fetchMeta != WITH_DATA) { SWalCont* pHead = &((*ppCkHead)->head); - if (IS_META_MSG(pHead->msgType)) { + if (IS_META_MSG(pHead->msgType) && !(pHead->msgType == TDMT_VND_DELETE && pHandle->fetchMeta == ONLY_META)) { code = walFetchBody(pHandle->pWalReader, ppCkHead); if (code < 0) { *fetchOffset = offset; @@ -273,7 +273,7 @@ STqReader* tqReaderOpen(SVnode* pVnode) { return pReader; } -void tqCloseReader(STqReader* pReader) { +void tqReaderClose(STqReader* pReader) { // close wal reader if (pReader->pWalReader) { walCloseReader(pReader->pWalReader); @@ -294,7 +294,7 @@ void tqCloseReader(STqReader* pReader) { taosMemoryFree(pReader); } -int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) { +int32_t tqReaderSeek(STqReader* pReader, int64_t ver, const char* id) { if (walReaderSeekVer(pReader->pWalReader, ver) < 0) { return -1; } @@ -302,30 +302,54 @@ int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) { return 0; } -int32_t extractSubmitMsgFromWal(SWalReader* pReader, SPackedData* pPackedData) { - if (walNextValidMsg(pReader) < 0) { - return -1; +int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, const char* id) { + int32_t code = walNextValidMsg(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; } - void* pBody = POINTER_SHIFT(pReader->pHead->head.body, sizeof(SSubmitReq2Msg)); - int32_t len = pReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); int64_t ver = pReader->pHead->head.version; + if (ver > maxVer) { + tqDebug("maxVer in WAL:%"PRId64" reached current:%"PRId64", do not scan wal anymore, %s", maxVer, ver, id); + return TSDB_CODE_SUCCESS; + } - void* data = taosMemoryMalloc(len); - if (data == NULL) { - // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry - terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", 0); - return -1; + if (pReader->pHead->head.msgType == TDMT_VND_SUBMIT) { + void* pBody = POINTER_SHIFT(pReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + int32_t len = pReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); + + void* data = taosMemoryMalloc(len); + if (data == NULL) { + // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", 0); + return -1; + } + + memcpy(data, pBody, len); + SPackedData data1 = (SPackedData){.ver = ver, .msgLen = len, .msgStr = data}; + + *pItem = (SStreamQueueItem*)streamDataSubmitNew(&data1, STREAM_INPUT__DATA_SUBMIT); + if (*pItem == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("%s failed to create data submit for stream since out of memory", id); + return terrno; + } + } else if (pReader->pHead->head.msgType == TDMT_VND_DELETE) { + void* pBody = POINTER_SHIFT(pReader->pHead->head.body, sizeof(SMsgHead)); + int32_t len = pReader->pHead->head.bodyLen - sizeof(SMsgHead); + + extractDelDataBlock(pBody, len, ver, (SStreamRefDataBlock**)pItem); + tqDebug("s-task:%s delete msg extract from WAL, len:%d, ver:%"PRId64, id, len, ver); + } else { + ASSERT(0); } - memcpy(data, pBody, len); - *pPackedData = (SPackedData){.ver = ver, .msgLen = len, .msgStr = data}; return 0; } // todo ignore the error in wal? -int32_t tqNextBlockInWal(STqReader* pReader) { +bool tqNextBlockInWal(STqReader* pReader, const char* id) { SWalReader* pWalReader = pReader->pWalReader; while (1) { @@ -334,7 +358,7 @@ int32_t tqNextBlockInWal(STqReader* pReader) { // try next message in wal file // todo always retry to avoid read failure caused by wal file deletion if (walNextValidMsg(pWalReader) < 0) { - return FETCH_TYPE__NONE; + return false; } void* pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); @@ -360,42 +384,46 @@ int32_t tqNextBlockInWal(STqReader* pReader) { if (tDecodeSubmitReq(&decoder, &pReader->submit) < 0) { tDecoderClear(&decoder); tqError("decode wal file error, msgLen:%d, ver:%" PRId64, bodyLen, ver); - return FETCH_TYPE__NONE; + return false; } tDecoderClear(&decoder); pReader->nextBlk = 0; } - size_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData); + int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < numOfBlocks) { - tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg.msgStr, pReader->msg.msgLen, - pReader->msg.ver, pReader->nextBlk); + tqTrace("tq reader next data block %d/%d, len:%d %" PRId64 " %d", pReader->nextBlk, + numOfBlocks, pReader->msg.msgLen, pReader->msg.ver, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); if (pReader->tbIdHash == NULL) { - int32_t code = tqRetrieveDataBlock(pReader, NULL); - if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) { - return FETCH_TYPE__DATA; + SSDataBlock* pRes = NULL; + int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); + if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { + return true; } } void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); if (ret != NULL) { - tqDebug("tq reader return submit block, uid:%" PRId64 ", ver:%" PRId64, pSubmitTbData->uid, pReader->msg.ver); + tqTrace("tq reader return submit block, uid:%" PRId64 ", ver:%" PRId64, pSubmitTbData->uid, pReader->msg.ver); - int32_t code = tqRetrieveDataBlock(pReader, NULL); - if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) { - return FETCH_TYPE__DATA; + SSDataBlock* pRes = NULL; + int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); + if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { + return true; } } else { pReader->nextBlk += 1; - tqDebug("tq reader discard submit block, uid:%" PRId64 ", continue", pSubmitTbData->uid); + tqTrace("tq reader discard submit block, uid:%" PRId64 ", continue", pSubmitTbData->uid); } } + qTrace("stream scan return empty, all %d submit blocks consumed, %s", numOfBlocks, id); tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->msg.msgStr = NULL; } } @@ -419,6 +447,14 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i return 0; } +SWalReader* tqGetWalReader(STqReader* pReader) { + return pReader->pWalReader; +} + +SSDataBlock* tqGetResultBlock (STqReader* pReader) { + return pReader->pResBlock; +} + bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { if (pReader->msg.msgStr == NULL) { return false; @@ -426,8 +462,8 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < numOfBlocks) { - tqDebug("tq reader next data block, len:%d ver:%" PRId64 " index:%d/%d, %s", pReader->msg.msgLen, - pReader->msg.ver, pReader->nextBlk, numOfBlocks, idstr); + tqDebug("try next data block, len:%d ver:%" PRId64 " index:%d/%d, %s", pReader->msg.msgLen, pReader->msg.ver, + pReader->nextBlk, numOfBlocks, idstr); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); if (pReader->tbIdHash == NULL) { @@ -436,10 +472,11 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); if (ret != NULL) { - tqDebug("tq reader block found, ver:%" PRId64 ", uid:%" PRId64, pReader->msg.ver, pSubmitTbData->uid); + tqDebug("block found, ver:%" PRId64 ", uid:%" PRId64", %s", pReader->msg.ver, pSubmitTbData->uid, idstr); return true; } else { - tqDebug("tq reader discard submit block, uid:%" PRId64 ", continue", pSubmitTbData->uid); + tqDebug("discard submit block, uid:%" PRId64 ", total queried tables:%d continue %s", pSubmitTbData->uid, + taosHashGetSize(pReader->tbIdHash), idstr); } pReader->nextBlk++; @@ -503,15 +540,84 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap return 0; } -int32_t tqRetrieveDataBlock(STqReader* pReader, const char* idstr) { - tqDebug("tq reader retrieve data block %p, index:%d/%d, %s", pReader->msg.msgStr, pReader->nextBlk, - (int32_t)taosArrayGetSize(pReader->submit.aSubmitTbData), idstr); +static int32_t buildResSDataBlock(SSDataBlock* pBlock, SSchemaWrapper* pSchema, const SArray* pColIdList) { + if (blockDataGetNumOfCols(pBlock) > 0) { + return TSDB_CODE_SUCCESS; + } + + int32_t numOfCols = taosArrayGetSize(pColIdList); + if (numOfCols == 0) { // all columns are required + for (int32_t i = 0; i < pSchema->nCols; ++i) { + SSchema* pColSchema = &pSchema->pSchema[i]; + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + blockDataFreeRes(pBlock); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + } else { + if (numOfCols > pSchema->nCols) { + numOfCols = pSchema->nCols; + } + + int32_t i = 0; + int32_t j = 0; + while (i < pSchema->nCols && j < numOfCols) { + SSchema* pColSchema = &pSchema->pSchema[i]; + col_id_t colIdSchema = pColSchema->colId; + + col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pColIdList, j); + if (colIdSchema < colIdNeed) { + i++; + } else if (colIdSchema > colIdNeed) { + j++; + } else { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + return -1; + } + i++; + j++; + } + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SColVal* pColVal) { + int32_t code = TSDB_CODE_SUCCESS; + + if (IS_STR_DATA_TYPE(pColVal->type)) { + char val[65535 + 2] = {0}; + if (pColVal->value.pData != NULL) { + memcpy(varDataVal(val), pColVal->value.pData, pColVal->value.nData); + varDataSetLen(val, pColVal->value.nData); + code = colDataSetVal(pColumnInfoData, rowIndex, val, !COL_VAL_IS_VALUE(pColVal)); + } else { + colDataSetNULL(pColumnInfoData, rowIndex); + } + } else { + code = colDataSetVal(pColumnInfoData, rowIndex, (void*)&pColVal->value.val, !COL_VAL_IS_VALUE(pColVal)); + } + + return code; +} + +int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* id) { + tqTrace("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++); SSDataBlock* pBlock = pReader->pResBlock; + *pRes = pBlock; + blockDataCleanup(pBlock); + int32_t vgId = pReader->pWalReader->pWal->cfg.vgId; int32_t sversion = pSubmitTbData->sver; int64_t suid = pSubmitTbData->suid; int64_t uid = pSubmitTbData->uid; @@ -528,7 +634,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, const char* idstr) { if (pReader->pSchemaWrapper == NULL) { tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", uid:%" PRId64 "version %d, possibly dropped table", - pReader->pWalReader->pWal->cfg.vgId, suid, uid, pReader->cachedSchemaVer); + vgId, suid, uid, pReader->cachedSchemaVer); pReader->cachedSchemaSuid = 0; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; @@ -538,53 +644,12 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, const char* idstr) { pReader->cachedSchemaSuid = suid; pReader->cachedSchemaVer = sversion; - SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - if (blockDataGetNumOfCols(pBlock) > 0) { - blockDataDestroy(pReader->pResBlock); - pReader->pResBlock = createDataBlock(); - pBlock = pReader->pResBlock; - - pBlock->info.id.uid = uid; - pBlock->info.version = pReader->msg.ver; - } - - int32_t numOfCols = taosArrayGetSize(pReader->pColIdList); - if (numOfCols == 0) { // all columns are required - for (int32_t i = 0; i < pSchemaWrapper->nCols; ++i) { - SSchema* pColSchema = &pSchemaWrapper->pSchema[i]; - SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); - - int32_t code = blockDataAppendColInfo(pBlock, &colInfo); - if (code != TSDB_CODE_SUCCESS) { - blockDataFreeRes(pBlock); - return -1; - } - } - } else { - if (numOfCols > pSchemaWrapper->nCols) { - numOfCols = pSchemaWrapper->nCols; - } - - int32_t i = 0; - int32_t j = 0; - while (i < pSchemaWrapper->nCols && j < numOfCols) { - SSchema* pColSchema = &pSchemaWrapper->pSchema[i]; - col_id_t colIdSchema = pColSchema->colId; - - col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, j); - if (colIdSchema < colIdNeed) { - i++; - } else if (colIdSchema > colIdNeed) { - j++; - } else { - SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); - int32_t code = blockDataAppendColInfo(pBlock, &colInfo); - if (code != TSDB_CODE_SUCCESS) { - return -1; - } - i++; - j++; - } + ASSERT(pReader->cachedSchemaVer == pReader->pSchemaWrapper->version); + if (blockDataGetNumOfCols(pBlock) == 0) { + int32_t code = buildResSDataBlock(pReader->pResBlock, pReader->pSchemaWrapper, pReader->pColIdList); + if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d failed to build data block, code:%s", vgId, tstrerror(code)); + return code; } } } @@ -632,30 +697,15 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, const char* idstr) { } else if (pCol->cid == pColData->info.colId) { for (int32_t i = 0; i < pCol->nVal; i++) { tColDataGetValue(pCol, i, &colVal); - if (IS_STR_DATA_TYPE(colVal.type)) { - if (colVal.value.pData != NULL) { - char val[65535 + 2] = {0}; - memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); - varDataSetLen(val, colVal.value.nData); - if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { - return -1; - } - } else { - colDataSetNULL(pColData, i); - } - } else { - if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { - return -1; - } + int32_t code = doSetVal(pColData, i, &colVal); + if (code != TSDB_CODE_SUCCESS) { + return code; } } sourceIdx++; targetIdx++; } else { - for (int32_t i = 0; i < pCol->nVal; i++) { - colDataSetNULL(pColData, i); - } - + colDataSetNNULL(pColData, 0, pCol->nVal); targetIdx++; } } @@ -674,26 +724,15 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, const char* idstr) { SColVal colVal; tRowGet(pRow, pTSchema, sourceIdx, &colVal); if (colVal.cid < pColData->info.colId) { -// tqDebug("colIndex:%d column id:%d in row, ignore, the required colId:%d, total cols in schema:%d", -// sourceIdx, colVal.cid, pColData->info.colId, pTSchema->numOfCols); + // tqDebug("colIndex:%d column id:%d in row, ignore, the required colId:%d, total cols in + // schema:%d", + // sourceIdx, colVal.cid, pColData->info.colId, pTSchema->numOfCols); sourceIdx++; continue; } else if (colVal.cid == pColData->info.colId) { - if (IS_STR_DATA_TYPE(colVal.type)) { - if (colVal.value.pData != NULL) { - char val[65535 + 2] = {0}; - memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); - varDataSetLen(val, colVal.value.nData); - if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { - return -1; - } - } else { - colDataSetNULL(pColData, i); - } - } else { - if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { - return -1; - } + int32_t code = doSetVal(pColData, i, &colVal); + if (code != TSDB_CODE_SUCCESS) { + return code; } sourceIdx++; @@ -712,13 +751,17 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, const char* idstr) { return 0; } +// todo refactor: int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) { tqDebug("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); pReader->nextBlk++; - if (pSubmitTbDataRet) *pSubmitTbDataRet = pSubmitTbData; + if (pSubmitTbDataRet) { + *pSubmitTbDataRet = pSubmitTbData; + } + int32_t sversion = pSubmitTbData->sver; int64_t suid = pSubmitTbData->suid; int64_t uid = pSubmitTbData->uid; @@ -831,14 +874,14 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas char val[65535 + 2]; memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); varDataSetLen(val, colVal.value.nData); - if (colDataAppend(pColData, curRow - lastRow, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + if (colDataSetVal(pColData, curRow - lastRow, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { goto FAIL; } } else { colDataSetNULL(pColData, curRow - lastRow); } } else { - if (colDataAppend(pColData, curRow - lastRow, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + if (colDataSetVal(pColData, curRow - lastRow, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { goto FAIL; } } @@ -928,14 +971,14 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas char val[65535 + 2]; memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); varDataSetLen(val, colVal.value.nData); - if (colDataAppend(pColData, curRow - lastRow, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + if (colDataSetVal(pColData, curRow - lastRow, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { goto FAIL; } } else { colDataSetNULL(pColData, curRow - lastRow); } } else { - if (colDataAppend(pColData, curRow - lastRow, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + if (colDataSetVal(pColData, curRow - lastRow, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { goto FAIL; } } @@ -962,7 +1005,7 @@ FAIL: void tqReaderSetColIdList(STqReader* pReader, SArray* pColIdList) { pReader->pColIdList = pColIdList; } -int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) { +int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList, const char* id) { if (pReader->tbIdHash) { taosHashClear(pReader->tbIdHash); } else { @@ -979,6 +1022,7 @@ int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) { taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0); } + tqDebug("s-task:%s %d tables are set to be queried target table", id, (int32_t) taosArrayGetSize(tbUidList)); return 0; } @@ -1000,6 +1044,14 @@ int tqReaderAddTbUidList(STqReader* pReader, const SArray* pTableUidList) { return 0; } +bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid) { + return taosHashGet(pReader->tbIdHash, &uid, sizeof(uint64_t)); +} + +bool tqCurrentBlockConsumed(const STqReader* pReader) { + return pReader->msg.msgStr == NULL; +} + int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); @@ -1015,7 +1067,6 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { int32_t vgId = TD_VID(pTq->pVnode); // update the table list for each consumer handle - taosWLockLatch(&pTq->lock); while (1) { pIter = taosHashIterate(pTq->pHandle, pIter); if (pIter == NULL) { @@ -1039,40 +1090,20 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (isAdd) { - SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); - SMetaReader mr = {0}; - metaReaderInit(&mr, pTq->pVnode->pMeta, 0); - for (int32_t i = 0; i < taosArrayGetSize(tbUidList); ++i) { - uint64_t* id = (uint64_t*)taosArrayGet(tbUidList, i); - - int32_t code = metaGetTableEntryByUidCache(&mr, *id); - if (code != TSDB_CODE_SUCCESS) { - tqError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); - continue; - } - - tDecoderClear(&mr.coder); - if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pTqHandle->execHandle.execTb.suid) { - tqDebug("table uid %" PRId64 " does not add to tq handle", *id); - continue; - } - - tqDebug("table uid %" PRId64 " add to tq handle", *id); - taosArrayPush(qa, id); + SArray* list = NULL; + int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode, pTqHandle->execHandle.execTb.node, &list, pTqHandle->execHandle.task); + if(ret != TDB_CODE_SUCCESS) { + tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey, pTqHandle->consumerId); + taosArrayDestroy(list); + return ret; } - - metaReaderClear(&mr); - if (taosArrayGetSize(qa) > 0) { - tqReaderAddTbUidList(pTqHandle->execHandle.pTqReader, qa); - } - - taosArrayDestroy(qa); + tqReaderSetTbUidList(pTqHandle->execHandle.pTqReader, list, NULL); + taosArrayDestroy(list); } else { tqReaderRemoveTbUidList(pTqHandle->execHandle.pTqReader, tbUidList); } } } - taosWUnLockLatch(&pTq->lock); // update the table list handle for each stream scanner/wal reader taosWLockLatch(&pTq->pStreamMeta->lock); @@ -1083,7 +1114,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { int32_t code = qUpdateTableListForStreamScanner(pTask->exec.pExecutor, tbUidList, isAdd); if (code != 0) { tqError("vgId:%d, s-task:%s update qualified table error for stream task", vgId, pTask->id.idStr); diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index ea7e9ee715c029f78a9132639aaeb2295d0e5183..3d9a91899cb2f91e811fb66c15610a9d0717ddca 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -15,7 +15,8 @@ #include "tq.h" -static int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle); +static int32_t createStreamTaskRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle); +static int32_t doSetOffsetForWalReader(SStreamTask *pTask, int32_t vgId); // this function should be executed by stream threads. // extract submit block from WAL, and add them into the input queue for the sources tasks. @@ -30,16 +31,14 @@ int32_t tqStreamTasksScanWal(STQ* pTq) { // check all restore tasks bool shouldIdle = true; - createStreamRunReq(pTq->pStreamMeta, &shouldIdle); + createStreamTaskRunReq(pTq->pStreamMeta, &shouldIdle); int32_t times = 0; if (shouldIdle) { taosWLockLatch(&pMeta->lock); - pMeta->walScanCounter -= 1; - times = pMeta->walScanCounter; - + times = (--pMeta->walScanCounter); ASSERT(pMeta->walScanCounter >= 0); if (pMeta->walScanCounter <= 0) { @@ -57,9 +56,173 @@ int32_t tqStreamTasksScanWal(STQ* pTq) { return 0; } -int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { +int32_t tqStreamTasksStatusCheck(STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + SStreamMeta* pMeta = pTq->pStreamMeta; + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + tqDebug("vgId:%d start to check all (%d) stream tasks downstream status", vgId, numOfTasks); + if (numOfTasks == 0) { + return TSDB_CODE_SUCCESS; + } + + SArray* pTaskList = NULL; + taosWLockLatch(&pMeta->lock); + pTaskList = taosArrayDup(pMeta->pTaskList, NULL); + taosWUnLockLatch(&pMeta->lock); + + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + if (pTask == NULL) { + continue; + } + + if (pTask->info.fillHistory == 1) { + tqDebug("s-task:%s fill-history task, wait for related stream task:0x%x to launch it", pTask->id.idStr, + pTask->streamTaskId.taskId); + continue; + } + + streamTaskDoCheckDownstreamTasks(pTask); + streamMetaReleaseTask(pMeta, pTask); + } + + taosArrayDestroy(pTaskList); + return 0; +} + +int32_t tqCheckStreamStatus(STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + SStreamMeta* pMeta = pTq->pStreamMeta; + + taosWLockLatch(&pMeta->lock); + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + if (numOfTasks == 0) { + tqInfo("vgId:%d no stream tasks exist", vgId); + taosWUnLockLatch(&pMeta->lock); + return 0; + } + + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); + taosWUnLockLatch(&pMeta->lock); + return -1; + } + + tqDebug("vgId:%d check for stream tasks status, numOfTasks:%d", vgId, numOfTasks); + pRunReq->head.vgId = vgId; + pRunReq->streamId = 0; + pRunReq->taskId = STREAM_TASK_STATUS_CHECK_ID; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); + taosWUnLockLatch(&pMeta->lock); + + return 0; +} + +int32_t tqStartStreamTasks(STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + SStreamMeta* pMeta = pTq->pStreamMeta; + + taosWLockLatch(&pMeta->lock); + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + if (numOfTasks == 0) { + tqInfo("vgId:%d no stream tasks exist", vgId); + taosWUnLockLatch(&pMeta->lock); + return 0; + } + + pMeta->walScanCounter += 1; + + if (pMeta->walScanCounter > 1) { + tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->walScanCounter); + taosWUnLockLatch(&pMeta->lock); + return 0; + } + + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); + taosWUnLockLatch(&pMeta->lock); + return -1; + } + + tqDebug("vgId:%d create msg to start wal scan to launch stream tasks, numOfTasks:%d", vgId, numOfTasks); + pRunReq->head.vgId = vgId; + pRunReq->streamId = 0; + pRunReq->taskId = EXTRACT_DATA_FROM_WAL_ID; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); + taosWUnLockLatch(&pMeta->lock); + + return 0; +} + +int32_t doSetOffsetForWalReader(SStreamTask *pTask, int32_t vgId) { + // seek the stored version and extract data from WAL + int64_t firstVer = walReaderGetValidFirstVer(pTask->exec.pWalReader); + if (pTask->chkInfo.currentVer < firstVer) { + tqWarn("vgId:%d s-task:%s ver:%"PRId64" earlier than the first ver of wal range %" PRId64 ", forward to %" PRId64, vgId, + pTask->id.idStr, pTask->chkInfo.currentVer, firstVer, firstVer); + + pTask->chkInfo.currentVer = firstVer; + + // todo need retry if failed + int32_t code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // append the data for the stream + tqDebug("vgId:%d s-task:%s wal reader seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer); + } else { + int64_t currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader); + if (currentVer == -1) { // we only seek the read for the first time + int32_t code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer); + if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit + return code; + } + + // append the data for the stream + tqDebug("vgId:%d s-task:%s wal reader initial seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer); + } + } + + int64_t skipToVer = walReaderGetSkipToVersion(pTask->exec.pWalReader); + if (skipToVer != 0 && skipToVer > pTask->chkInfo.currentVer) { + int32_t code = walReaderSeekVer(pTask->exec.pWalReader, skipToVer); + if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit + return code; + } + + tqDebug("vgId:%d s-task:%s wal reader jump to ver:%" PRId64, vgId, pTask->id.idStr, skipToVer); + } + + return TSDB_CODE_SUCCESS; +} + +static void checkForFillHistoryVerRange(SStreamTask* pTask, int64_t ver) { + if ((pTask->info.fillHistory == 1) && ver > pTask->dataRange.range.maxVer) { + qWarn("s-task:%s fill-history scan WAL, currentVer:%" PRId64 " reach the maximum ver:%" PRId64 + ", not scan wal anymore, set the transfer state flag", + pTask->id.idStr, ver, pTask->dataRange.range.maxVer); + pTask->status.transferState = true; + + /*int32_t code = */streamSchedExec(pTask); + } +} + +int32_t createStreamTaskRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { *pScanIdle = true; - bool noNewDataInWal = true; + bool noDataInWal = true; int32_t vgId = pStreamMeta->vgId; int32_t numOfTasks = taosArrayGetSize(pStreamMeta->pTaskList); @@ -67,6 +230,7 @@ int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { return TSDB_CODE_SUCCESS; } + // clone the task list, to avoid the task update during scan wal files SArray* pTaskList = NULL; taosWLockLatch(&pStreamMeta->lock); pTaskList = taosArrayDup(pStreamMeta->pTaskList, NULL); @@ -78,28 +242,37 @@ int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { numOfTasks = taosArrayGetSize(pTaskList); for (int32_t i = 0; i < numOfTasks; ++i) { - int32_t* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, *pTaskId); + SStreamId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, pTaskId->streamId, pTaskId->taskId); if (pTask == NULL) { continue; } int32_t status = pTask->status.taskStatus; - if (pTask->taskLevel != TASK_LEVEL__SOURCE) { - tqDebug("s-task:%s level:%d not source task, no need to start", pTask->id.idStr, pTask->taskLevel); + + // non-source or fill-history tasks don't need to response the WAL scan action. + if ((pTask->info.taskLevel != TASK_LEVEL__SOURCE) || (pTask->status.downstreamReady == 0)) { streamMetaReleaseTask(pStreamMeta, pTask); continue; } - if (streamTaskShouldStop(&pTask->status) || status == TASK_STATUS__RECOVER_PREPARE || - status == TASK_STATUS__WAIT_DOWNSTREAM || streamTaskShouldPause(&pTask->status)) { - tqDebug("s-task:%s not ready for new submit block from wal, status:%d", pTask->id.idStr, status); + if (status != TASK_STATUS__NORMAL) { + tqDebug("s-task:%s not ready for new submit block from wal, status:%s", pTask->id.idStr, streamGetTaskStatusStr(status)); + streamMetaReleaseTask(pStreamMeta, pTask); + continue; + } + + if ((pTask->info.fillHistory == 1) && pTask->status.transferState) { + ASSERT(status == TASK_STATUS__NORMAL); + // the maximum version of data in the WAL has reached already, the step2 is done + tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr, + pTask->dataRange.range.maxVer); streamMetaReleaseTask(pStreamMeta, pTask); continue; } if (tInputQueueIsFull(pTask)) { - tqDebug("s-task:%s input queue is full, do nothing", pTask->id.idStr); + tqTrace("s-task:%s input queue is full, do nothing", pTask->id.idStr); streamMetaReleaseTask(pStreamMeta, pTask); continue; } @@ -107,73 +280,55 @@ int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { *pScanIdle = false; // seek the stored version and extract data from WAL - int64_t firstVer = walReaderGetValidFirstVer(pTask->exec.pWalReader); - if (pTask->chkInfo.currentVer < firstVer) { - pTask->chkInfo.currentVer = firstVer; - tqWarn("vgId:%d s-task:%s ver earlier than the first ver of wal range %" PRId64 ", forward to %" PRId64, vgId, - pTask->id.idStr, firstVer, pTask->chkInfo.currentVer); - - // todo need retry if failed - int32_t code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer); - if (code != TSDB_CODE_SUCCESS) { - streamMetaReleaseTask(pStreamMeta, pTask); - continue; - } - - - - // append the data for the stream - tqDebug("vgId:%d s-task:%s wal reader seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer); - } else { - int64_t currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader); - if (currentVer != pTask->chkInfo.currentVer) { - int32_t code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer); - if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit - streamMetaReleaseTask(pStreamMeta, pTask); - continue; - } - - // append the data for the stream - tqDebug("vgId:%d s-task:%s wal reader seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer); - } - } - - SPackedData packData = {0}; - int32_t code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData); - if (code != TSDB_CODE_SUCCESS) { // failed, continue + int32_t code = doSetOffsetForWalReader(pTask, vgId); + if (code != TSDB_CODE_SUCCESS) { streamMetaReleaseTask(pStreamMeta, pTask); continue; } - SStreamDataSubmit2* p = streamDataSubmitNew(packData, STREAM_INPUT__DATA_SUBMIT); - if (p == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("%s failed to create data submit for stream since out of memory", pTask->id.idStr); + int32_t numOfItemsInQ = taosQueueItemSize(pTask->inputQueue->queue); + int64_t maxVer = (pTask->info.fillHistory == 1)? pTask->dataRange.range.maxVer:INT64_MAX; + + SStreamQueueItem* pItem = NULL; + code = extractMsgFromWal(pTask->exec.pWalReader, (void**) &pItem, maxVer, pTask->id.idStr); + + if ((code != TSDB_CODE_SUCCESS || pItem == NULL) && (numOfItemsInQ == 0)) { // failed, continue + checkForFillHistoryVerRange(pTask, walReaderGetCurrentVer(pTask->exec.pWalReader)); streamMetaReleaseTask(pStreamMeta, pTask); continue; } - noNewDataInWal = false; + if (pItem != NULL) { + noDataInWal = false; + code = tAppendDataToInputQueue(pTask, pItem); + if (code == TSDB_CODE_SUCCESS) { + int64_t ver = walReaderGetCurrentVer(pTask->exec.pWalReader); + pTask->chkInfo.currentVer = ver; + checkForFillHistoryVerRange(pTask, ver); + tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr, ver); + } else { + tqError("s-task:%s append input queue failed, too many in inputQ, ver:%" PRId64, pTask->id.idStr, + pTask->chkInfo.currentVer); + } + } - code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver); - if (code == TSDB_CODE_SUCCESS) { - pTask->chkInfo.currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader); - tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr, - pTask->chkInfo.currentVer); - } else { - tqError("s-task:%s append input queue failed, ver:%" PRId64, pTask->id.idStr, pTask->chkInfo.currentVer); + if ((code == TSDB_CODE_SUCCESS) || (numOfItemsInQ > 0)) { + code = streamSchedExec(pTask); + if (code != TSDB_CODE_SUCCESS) { + streamMetaReleaseTask(pStreamMeta, pTask); + return -1; + } } - streamDataSubmitDestroy(p); - taosFreeQitem(p); streamMetaReleaseTask(pStreamMeta, pTask); } // all wal are checked, and no new data available in wal. - if (noNewDataInWal) { + if (noDataInWal) { *pScanIdle = true; } taosArrayDestroy(pTaskList); return 0; } + diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 52f92cd2296d98087992a145e08391cfeb250164..cbe3ffee9e9fa5f62bad48fc0df07bfedf2ea610 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -48,10 +48,10 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, int32_t n) { SMetaReader mr = {0}; - metaReaderInit(&mr, pTq->pVnode->pMeta, 0); + metaReaderDoInit(&mr, pTq->pVnode->pMeta, 0); // TODO add reference to gurantee success - if (metaGetTableEntryByUidCache(&mr, uid) < 0) { + if (metaReaderGetTableEntryByUidCache(&mr, uid) < 0) { metaReaderClear(&mr); return -1; } @@ -82,9 +82,12 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; qStreamSetOpen(task); + tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); - if (qExecTask(task, &pDataBlock, &ts) != TSDB_CODE_SUCCESS) { - tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, terrstr()); + code = qExecTask(task, &pDataBlock, &ts); + if (code != TSDB_CODE_SUCCESS) { + tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, tstrerror(code)); + terrno = code; return -1; } @@ -127,8 +130,10 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; tqDebug("tmqsnap task start to execute"); - if (qExecTask(task, &pDataBlock, &ts) < 0) { - tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr()); + int code = qExecTask(task, &pDataBlock, &ts); + if (code != 0) { + tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, tstrerror(code)); + terrno = code; return -1; } @@ -210,19 +215,15 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR taosArrayClear(pSchemas); SSubmitTbData* pSubmitTbDataRet = NULL; if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet) < 0) { - if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; + if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_table; } if (pRsp->withTbName) { int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { - taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); - taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); - pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); - pSchemas = taosArrayInit(0, sizeof(void*)); - continue; + goto loop_table; } } - if (pHandle->fetchMeta && pSubmitTbDataRet->pCreateTbReq != NULL) { + if (pHandle->fetchMeta != WITH_DATA && pSubmitTbDataRet->pCreateTbReq != NULL) { if (pRsp->createTableNum == 0) { pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); @@ -232,7 +233,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR uint32_t len = 0; tEncodeSize(tEncodeSVCreateTbReq, pSubmitTbDataRet->pCreateTbReq, len, code); if (TSDB_CODE_SUCCESS != code) { - continue; + goto loop_table; } void* createReq = taosMemoryCalloc(1, len); SEncoder encoder = {0}; @@ -241,7 +242,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (code < 0) { tEncoderClear(&encoder); taosMemoryFree(createReq); - continue; + goto loop_table; } taosArrayPush(pRsp->createTableLen, &len); @@ -250,6 +251,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR tEncoderClear(&encoder); } + if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL){ + goto loop_table; + } for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { SSDataBlock* pBlock = taosArrayGet(pBlocks, i); tqAddBlockDataToRsp(pBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlock->pDataBlock), @@ -260,6 +264,12 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR taosArrayPush(pRsp->blockSchema, &pSW); pRsp->blockNum++; } + continue; + loop_table: + taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); + taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); + pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); + pSchemas = taosArrayInit(0, sizeof(void*)); } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { STqReader* pReader = pExec->pTqReader; @@ -269,19 +279,15 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR taosArrayClear(pSchemas); SSubmitTbData* pSubmitTbDataRet = NULL; if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet) < 0) { - if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; + if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_db; } if (pRsp->withTbName) { int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { - taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); - taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); - pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); - pSchemas = taosArrayInit(0, sizeof(void*)); - continue; + goto loop_db; } } - if (pHandle->fetchMeta && pSubmitTbDataRet->pCreateTbReq != NULL) { + if (pHandle->fetchMeta != WITH_DATA && pSubmitTbDataRet->pCreateTbReq != NULL) { if (pRsp->createTableNum == 0) { pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); @@ -291,7 +297,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR uint32_t len = 0; tEncodeSize(tEncodeSVCreateTbReq, pSubmitTbDataRet->pCreateTbReq, len, code); if (TSDB_CODE_SUCCESS != code) { - continue; + goto loop_db; } void* createReq = taosMemoryCalloc(1, len); SEncoder encoder = {0}; @@ -300,7 +306,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (code < 0) { tEncoderClear(&encoder); taosMemoryFree(createReq); - continue; + goto loop_db; } taosArrayPush(pRsp->createTableLen, &len); @@ -309,6 +315,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR tEncoderClear(&encoder); } + if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL){ + goto loop_db; + } for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { SSDataBlock* pBlock = taosArrayGet(pBlocks, i); tqAddBlockDataToRsp(pBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlock->pDataBlock), @@ -319,6 +328,12 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR taosArrayPush(pRsp->blockSchema, &pSW); pRsp->blockNum++; } + continue; + loop_db: + taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); + taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); + pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); + pSchemas = taosArrayInit(0, sizeof(void*)); } } taosArrayDestroy(pBlocks); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 4a9e3dcee718c1349341ad609e17a8e0f079abbb..b22650d2498c17677607a99fce0dd4cba312d230 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -17,12 +17,12 @@ #include "tmsg.h" #include "tq.h" -#define MAX_CATCH_NUM 10240 +#define MAX_CACHE_TABLE_INFO_NUM 10240 -typedef struct STblInfo { +typedef struct STableSinkInfo { uint64_t uid; char tbName[TSDB_TABLE_NAME_LEN]; -} STblInfo; +} STableSinkInfo; int32_t tqBuildDeleteReq(const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, const char* pIdStr) { @@ -97,19 +97,21 @@ end: return ret; } -int32_t tqGetTableInfo(SSHashObj* tblInfo ,uint64_t groupId, STblInfo** pTbl) { - void* pVal = tSimpleHashGet(tblInfo, &groupId, sizeof(uint64_t)); +static int32_t tqGetTableInfo(SSHashObj* pTableInfoMap,uint64_t groupId, STableSinkInfo** pInfo) { + void* pVal = tSimpleHashGet(pTableInfoMap, &groupId, sizeof(uint64_t)); if (pVal) { - *pTbl = *(STblInfo**)pVal; + *pInfo = *(STableSinkInfo**)pVal; return TSDB_CODE_SUCCESS; } + return TSDB_CODE_FAILED; } -int32_t tqPutTableInfo(SSHashObj* tblInfo ,uint64_t groupId, STblInfo* pTbl) { - if (tSimpleHashGetSize(tblInfo) > MAX_CATCH_NUM) { - return TSDB_CODE_SUCCESS; +int32_t tqPutTableInfo(SSHashObj* tblInfo ,uint64_t groupId, STableSinkInfo* pTbl) { + if (tSimpleHashGetSize(tblInfo) > MAX_CACHE_TABLE_INFO_NUM) { + return TSDB_CODE_FAILED; } + return tSimpleHashPut(tblInfo, &groupId, sizeof(uint64_t), &pTbl, POINTER_BYTES); } @@ -135,7 +137,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d int32_t blockSz = taosArrayGetSize(pBlocks); - tqDebug("vgId:%d, s-task:%s write results blocks:%d into table", TD_VID(pVnode), pTask->id.idStr, blockSz); + tqDebug("vgId:%d, s-task:%s write results %d blocks into table", TD_VID(pVnode), pTask->id.idStr, blockSz); void* pBuf = NULL; SArray* tagArray = NULL; @@ -274,7 +276,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d crTblArray = NULL; } else { SSubmitTbData tbData = {0}; - tqDebug("tq sink pipe, convert block1 %d, rows: %d", i, rows); + tqDebug("tq sink pipe, convert block:%d, rows:%d", i, rows); if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { goto _end; @@ -283,35 +285,34 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d tbData.suid = suid; tbData.uid = 0; // uid is assigned by vnode tbData.sver = pTSchema->version; - STblInfo* pTblMeta = NULL; - int32_t res = tqGetTableInfo(pTask->tbSink.pTblInfo, pDataBlock->info.id.groupId, &pTblMeta); + STableSinkInfo* pTableSinkInfo = NULL; + int32_t res = tqGetTableInfo(pTask->tbSink.pTblInfo, pDataBlock->info.id.groupId, &pTableSinkInfo); if (res != TSDB_CODE_SUCCESS) { - pTblMeta = taosMemoryCalloc(1, sizeof(STblInfo)); + pTableSinkInfo = taosMemoryCalloc(1, sizeof(STableSinkInfo)); } char* ctbName = pDataBlock->info.parTbName; if (!ctbName[0]) { + memset(ctbName, 0, TSDB_TABLE_NAME_LEN); if (res == TSDB_CODE_SUCCESS) { - memcpy(ctbName, pTblMeta->tbName, strlen(pTblMeta->tbName)); + memcpy(ctbName, pTableSinkInfo->tbName, strlen(pTableSinkInfo->tbName)); } else { - char* tmp = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); - memcpy(ctbName, tmp, strlen(tmp)); - memcpy(pTblMeta->tbName, tmp, strlen(tmp)); - taosMemoryFree(tmp); - tqDebug("vgId:%d, gropuid:%" PRIu64 " datablock tabel name is null", TD_VID(pVnode), + buildCtbNameByGroupIdImpl(stbFullName, pDataBlock->info.id.groupId, ctbName); + memcpy(pTableSinkInfo->tbName, ctbName, strlen(ctbName)); + tqDebug("vgId:%d, gropuId:%" PRIu64 " datablock table name is null", TD_VID(pVnode), pDataBlock->info.id.groupId); } } if (res == TSDB_CODE_SUCCESS) { - tbData.uid = pTblMeta->uid; + tbData.uid = pTableSinkInfo->uid; } else { SMetaReader mr = {0}; - metaReaderInit(&mr, pVnode->pMeta, 0); + metaReaderDoInit(&mr, pVnode->pMeta, 0); if (metaGetTableEntryByName(&mr, ctbName) < 0) { metaReaderClear(&mr); - taosMemoryFree(pTblMeta); + taosMemoryFree(pTableSinkInfo); tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), ctbName); SVCreateTbReq* pCreateTbReq = NULL; @@ -334,6 +335,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d tagArray = taosArrayInit(1, sizeof(STagVal)); if (!tagArray) { tdDestroySVCreateTbReq(pCreateTbReq); + taosMemoryFreeClear(pCreateTbReq); goto _end; } STagVal tagVal = { @@ -349,6 +351,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d tagArray = taosArrayDestroy(tagArray); if (pTag == NULL) { tdDestroySVCreateTbReq(pCreateTbReq); + taosMemoryFreeClear(pCreateTbReq); terrno = TSDB_CODE_OUT_OF_MEMORY; goto _end; } @@ -371,7 +374,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d tqError("vgId:%d, failed to write into %s, since table type incorrect, type %d", TD_VID(pVnode), ctbName, mr.me.type); metaReaderClear(&mr); - taosMemoryFree(pTblMeta); + taosMemoryFree(pTableSinkInfo); continue; } @@ -380,13 +383,16 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d ", actual suid %" PRId64 "", TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid); metaReaderClear(&mr); - taosMemoryFree(pTblMeta); + taosMemoryFree(pTableSinkInfo); continue; } tbData.uid = mr.me.uid; - pTblMeta->uid = mr.me.uid; - tqPutTableInfo(pTask->tbSink.pTblInfo, pDataBlock->info.id.groupId, pTblMeta); + pTableSinkInfo->uid = mr.me.uid; + int32_t code = tqPutTableInfo(pTask->tbSink.pTblInfo, pDataBlock->info.id.groupId, pTableSinkInfo); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pTableSinkInfo); + } metaReaderClear(&mr); } } @@ -406,7 +412,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d if (k == 0) { SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex); void* colData = colDataGetData(pColData, j); - tqDebug("tq sink pipe2, row %d, col %d ts %" PRId64, j, k, *(int64_t*)colData); + tqTrace("tq sink pipe2, row %d, col %d ts %" PRId64, j, k, *(int64_t*)colData); } if (IS_SET_NULL(pCol)) { SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); @@ -477,17 +483,15 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d tEncoderClear(&encoder); tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); - SRpcMsg msg = { - .msgType = TDMT_VND_SUBMIT, - .pCont = pBuf, - .contLen = len, - }; - + SRpcMsg msg = { .msgType = TDMT_VND_SUBMIT, .pCont = pBuf, .contLen = len }; if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { tqDebug("failed to put into write-queue since %s", terrstr()); } } } + + tqDebug("vgId:%d, s-task:%s write results completed", TD_VID(pVnode), pTask->id.idStr); + _end: taosArrayDestroy(tagArray); taosArrayDestroy(pVals); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index b2286ed02c43a1ed3c6563a7eae98ca518b8bc2d..5cbca6e0f2dbe7a60231cafd1fc223f9c39eff05 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -16,33 +16,13 @@ #include "tq.h" #define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) -#define NO_POLL_CNT 5 -static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId); +static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, + const SMqMetaRsp* pRsp, int32_t vgId); -char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { - char buf[128] = {0}; - sprintf(buf, "0x%" PRIx64 "-0x%x", streamId, taskId); - return taosStrdup(buf); -} - -int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) { - int32_t code = tAppendDataToInputQueue(pTask, pQueueItem); - if (code < 0) { - tqError("s-task:%s failed to put into queue, too many, next start ver:%" PRId64, pTask->id.idStr, ver); - return -1; - } - - if (streamSchedExec(pTask) < 0) { - tqError("stream task:%d failed to be launched, code:%s", pTask->id.taskId, tstrerror(terrno)); - return -1; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq) { - pRsp->reqOffset = pReq->reqOffset; +int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { + pRsp->reqOffset = pOffset; + pRsp->rspOffset = pOffset; pRsp->blockData = taosArrayInit(0, sizeof(void*)); pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); @@ -56,8 +36,9 @@ int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq) { return 0; } -static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { - pRsp->reqOffset = pReq->reqOffset; +static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset) { + pRsp->reqOffset = pOffset; + pRsp->rspOffset = pOffset; pRsp->withTbName = 1; pRsp->withSchema = 1; @@ -88,27 +69,25 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { return 0; } -static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, - SRpcMsg* pMsg, bool* pBlockReturned) { +static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, bool* pBlockReturned) { uint64_t consumerId = pRequest->consumerId; - STqOffsetVal reqOffset = pRequest->reqOffset; STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); int32_t vgId = TD_VID(pTq->pVnode); *pBlockReturned = false; - // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. if (pOffset != NULL) { *pOffsetVal = pOffset->val; - char formatBuf[80]; - tFormatOffset(formatBuf, 80, pOffsetVal); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%"PRIx64, + char formatBuf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(formatBuf, TSDB_OFFSET_LEN, pOffsetVal); + tqDebug("tmq poll: consumer:0x%" PRIx64 + ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%" PRIx64, consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId); return 0; } else { // no poll occurs in this vnode for this topic, let's seek to the right offset value. - if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { + if (pRequest->reqOffset.type == TMQ_OFFSET__RESET_EARLIEST) { if (pRequest->useSnapshot) { tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey:%s, vgId:%d, (earliest) set offset to be snapshot", consumerId, pHandle->subKey, vgId); @@ -119,40 +98,25 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand tqOffsetResetToData(pOffsetVal, 0, 0); } } else { - pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); - if (pHandle->pRef == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - // offset set to previous version when init - tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); - } - } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, pRequest); - - tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, - pHandle->subKey, vgId, dataRsp.rspOffset.version); - int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); - tDeleteMqDataRsp(&dataRsp); - - *pBlockReturned = true; - return code; - } else { - STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, pRequest); - tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); - tDeleteSTaosxRsp(&taosxRsp); - - *pBlockReturned = true; - return code; + walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); + tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer); } - } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { - tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", + } else if (pRequest->reqOffset.type == TMQ_OFFSET__RESET_LATEST) { + walRefLastVer(pTq->pVnode->pWal, pHandle->pRef); + SMqDataRsp dataRsp = {0}; + tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer + 1); + + tqInitDataRsp(&dataRsp, *pOffsetVal); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, + pHandle->subKey, vgId, dataRsp.rspOffset.version); + int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + tDeleteMqDataRsp(&dataRsp); + + *pBlockReturned = true; + return code; + } else if (pRequest->reqOffset.type == TMQ_OFFSET__RESET_NONE) { + tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 + " in vg %d, subkey %s, reset none failed", pHandle->subKey, consumerId, vgId, pRequest->subKey); terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; return -1; @@ -162,59 +126,64 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand return 0; } +//static void setRequestVersion(STqOffsetVal* offset, int64_t ver){ +// if(offset->type == TMQ_OFFSET__LOG){ +// offset->version = ver; +// } +//} + static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* pOffset) { uint64_t consumerId = pRequest->consumerId; int32_t vgId = TD_VID(pTq->pVnode); - int code = 0; + terrno = 0; SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, pRequest); + tqInitDataRsp(&dataRsp, *pOffset); +// dataRsp.reqOffset.type = pOffset->type; // store origin type for getting offset in tmq_get_vgroup_offset qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); - code = tqScanData(pTq, pHandle, &dataRsp, pOffset); - if (code != 0) { + int code = tqScanData(pTq, pHandle, &dataRsp, pOffset); + if (code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { goto end; } // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && - dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { - if (pHandle->noDataPollCnt >= NO_POLL_CNT) { // send poll result to client if no data 5 times to avoid lost data - pHandle->noDataPollCnt = 0; - // lock - taosWLockLatch(&pTq->lock); + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.blockNum == 0) { + // lock + taosWLockLatch(&pTq->lock); + int64_t ver = walGetCommittedVer(pTq->pVnode->pWal); + if (dataRsp.rspOffset.version > ver) { // check if there are data again to avoid lost data code = tqRegisterPushHandle(pTq, pHandle, pMsg); taosWUnLockLatch(&pTq->lock); - tDeleteMqDataRsp(&dataRsp); - return code; - } else { - pHandle->noDataPollCnt++; + goto end; } + taosWUnLockLatch(&pTq->lock); } - // NOTE: this pHandle->consumerId may have been changed already. - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); - - end: - { - char buf[80] = {0}; - tFormatOffset(buf, 80, &dataRsp.rspOffset); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 " code:%d", - consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); - tDeleteMqDataRsp(&dataRsp); - } +// setRequestVersion(&dataRsp.reqOffset, pOffset->version); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); +end : { + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.rspOffset); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 + " code:%d", + consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); + tDeleteMqDataRsp(&dataRsp); return code; + } } -static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal *offset) { - int code = 0; - int32_t vgId = TD_VID(pTq->pVnode); - SWalCkHead* pCkHead = NULL; - SMqMetaRsp metaRsp = {0}; - STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, pRequest); +static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, + SRpcMsg* pMsg, STqOffsetVal* offset) { + int code = 0; + int32_t vgId = TD_VID(pTq->pVnode); + SWalCkHead* pCkHead = NULL; + SMqMetaRsp metaRsp = {0}; + STaosxRsp taosxRsp = {0}; + tqInitTaosxRsp(&taosxRsp, *offset); +// taosxRsp.reqOffset.type = offset->type; // store origin type for getting offset in tmq_get_vgroup_offset if (offset->type != TMQ_OFFSET__LOG) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { @@ -224,25 +193,29 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (metaRsp.metaRspLen > 0) { code = tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId); - tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 ",ts:%" PRId64, - pRequest->consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, metaRsp.rspOffset.ts); + tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 + ",ts:%" PRId64, + pRequest->consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, + metaRsp.rspOffset.ts); taosMemoryFree(metaRsp.metaRsp); goto end; } tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64 - ",ts:%" PRId64,pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid,taosxRsp.rspOffset.ts); + ",ts:%" PRId64, + pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, + taosxRsp.rspOffset.uid, taosxRsp.rspOffset.ts); if (taosxRsp.blockNum > 0) { - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; - }else { + } else { *offset = taosxRsp.rspOffset; } } if (offset->type == TMQ_OFFSET__LOG) { - verifyOffset(pHandle->pWalReader, offset); - int64_t fetchVer = offset->version + 1; + walReaderVerifyOffset(pHandle->pWalReader, offset); + int64_t fetchVer = offset->version; pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); if (pCkHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -263,24 +236,26 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead, pRequest->reqId) < 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); +// setRequestVersion(&taosxRsp.reqOffset, offset->version); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } SWalCont* pHead = &pCkHead->head; - tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", pRequest->consumerId, - pRequest->epoch, vgId, fetchVer, pHead->msgType); + tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", + pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); // process meta if (pHead->msgType != TDMT_VND_SUBMIT) { - if(totalRows > 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer - 1); - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + if (totalRows > 0) { + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); +// setRequestVersion(&taosxRsp.reqOffset, offset->version); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType)); - tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer); + tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer + 1); metaRsp.resMsgType = pHead->msgType; metaRsp.metaRspLen = pHead->bodyLen; metaRsp.metaRsp = pHead->body; @@ -297,13 +272,15 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, code = tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows); if (code < 0) { - tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId, pRequest->subKey); + tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId, + pRequest->subKey); goto end; } if (totalRows >= 4096 || taosxRsp.createTableNum > 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer + 1); +// setRequestVersion(&taosxRsp.reqOffset, offset->version); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } else { fetchVer++; @@ -319,15 +296,13 @@ end: } int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) { - int32_t code = -1; - STqOffsetVal offset = {0}; STqOffsetVal reqOffset = pRequest->reqOffset; // 1. reset the offset if needed - if (IS_OFFSET_RESET_TYPE(reqOffset.type)) { + if (IS_OFFSET_RESET_TYPE(pRequest->reqOffset.type)) { // handle the reset offset cases, according to the consumer's choice. bool blockReturned = false; - code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned); + int32_t code = extractResetOffsetVal(&reqOffset, pTq, pHandle, pRequest, pMsg, &blockReturned); if (code != 0) { return code; } @@ -336,17 +311,17 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ if (blockReturned) { return 0; } - } else { // use the consumer specified offset - // the offset value can not be monotonious increase?? - offset = reqOffset; + } else if(reqOffset.type == 0){ // use the consumer specified offset + uError("req offset type is 0"); + return TSDB_CODE_TMQ_INVALID_MSG; } // this is a normal subscribe requirement if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - return extractDataAndRspForNormalSubscribe(pTq, pHandle, pRequest, pMsg, &offset); - } else { // todo handle the case where re-balance occurs. + return extractDataAndRspForNormalSubscribe(pTq, pHandle, pRequest, pMsg, &reqOffset); + } else { // todo handle the case where re-balance occurs. // for taosx - return extractDataAndRspForDbStbSubscribe(pTq, pHandle, pRequest, pMsg, &offset); + return extractDataAndRspForDbStbSubscribe(pTq, pHandle, pRequest, pMsg, &reqOffset); } } @@ -359,7 +334,8 @@ static void initMqRspHead(SMqRspHead* pMsgHead, int32_t type, int32_t epoch, int pMsgHead->walever = ever; } -int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId) { +int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, + int32_t vgId) { int32_t len = 0; int32_t code = 0; tEncodeSize(tEncodeMqMetaRsp, pRsp, len, code); @@ -383,7 +359,7 @@ int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPoll tEncodeMqMetaRsp(&encoder, pRsp); tEncoderClear(&encoder); - SRpcMsg resp = { .info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0 }; + SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0}; tmsgSendRsp(&resp); tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type %d, offset type:%d", vgId, @@ -393,13 +369,13 @@ int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPoll } int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, - int32_t type, int64_t sver, int64_t ever) { + int32_t type, int64_t sver, int64_t ever) { int32_t len = 0; int32_t code = 0; - if (type == TMQ_MSG_TYPE__POLL_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { + if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { tEncodeSize(tEncodeMqDataRsp, pRsp, len, code); - } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) { + } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code); } @@ -421,15 +397,15 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* SEncoder encoder = {0}; tEncoderInit(&encoder, abuf, len); - if (type == TMQ_MSG_TYPE__POLL_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { + if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { tEncodeMqDataRsp(&encoder, pRsp); - } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) { + } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { tEncodeSTaosxRsp(&encoder, (STaosxRsp*)pRsp); } tEncoderClear(&encoder); - SRpcMsg rsp = { .info = *pRpcHandleInfo, .pCont = buf, .contLen = tlen, .code = 0 }; + SRpcMsg rsp = {.info = *pRpcHandleInfo, .pCont = buf, .contLen = tlen, .code = 0}; tmsgSendRsp(&rsp); return 0; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index c0a8de57437f8557dc87cbef800ff1c40e0effa8..7d8cf5b67852c689920ef610a9c9e4442bfb5956 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -13,6 +13,11 @@ * along with this program. If not, see . */ #include "tsdb.h" +#include "tsdbDataFileRW.h" +#include "tsdbReadUtil.h" +#include "vnd.h" + +#define ROCKS_BATCH_SIZE (4096) static int32_t tsdbOpenBICache(STsdb *pTsdb) { int32_t code = 0; @@ -46,25 +51,72 @@ static void tsdbCloseBICache(STsdb *pTsdb) { } } -#define ROCKS_KEY_LEN 64 +#define ROCKS_KEY_LEN (sizeof(tb_uid_t) + sizeof(int16_t) + sizeof(int8_t)) + +typedef struct { + tb_uid_t uid; + int16_t cid; + int8_t ltype; +} SLastKey; static void tsdbGetRocksPath(STsdb *pTsdb, char *path) { SVnode *pVnode = pTsdb->pVnode; - if (pVnode->pTfs) { - if (path) { - snprintf(path, TSDB_FILENAME_LEN, "%s%s%s%scache.rdb", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), TD_DIRSEP, - pTsdb->path, TD_DIRSEP); - } - } else { - if (path) { - snprintf(path, TSDB_FILENAME_LEN, "%s%scache.rdb", pTsdb->path, TD_DIRSEP); - } + vnodeGetPrimaryDir(pTsdb->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN); + + int32_t offset = strlen(path); + snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%scache.rdb", TD_DIRSEP); +} + +static const char *myCmpName(void *state) { + (void)state; + return "myCmp"; +} + +static void myCmpDestroy(void *state) { (void)state; } + +static int myCmp(void *state, const char *a, size_t alen, const char *b, size_t blen) { + (void)state; + (void)alen; + (void)blen; + SLastKey *lhs = (SLastKey *)a; + SLastKey *rhs = (SLastKey *)b; + + if (lhs->uid < rhs->uid) { + return -1; + } else if (lhs->uid > rhs->uid) { + return 1; + } + + if (lhs->cid < rhs->cid) { + return -1; + } else if (lhs->cid > rhs->cid) { + return 1; + } + + if (lhs->ltype < rhs->ltype) { + return -1; + } else if (lhs->ltype > rhs->ltype) { + return 1; } + + return 0; } static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { int32_t code = 0; + rocksdb_comparator_t *cmp = rocksdb_comparator_create(NULL, myCmpDestroy, myCmp, myCmpName); + if (NULL == cmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + + rocksdb_cache_t *cache = rocksdb_cache_create_lru(5 * 1024 * 1024); + pTsdb->rCache.blockcache = cache; + + rocksdb_block_based_table_options_t *tableoptions = rocksdb_block_based_options_create(); + pTsdb->rCache.tableoptions = tableoptions; + rocksdb_options_t *options = rocksdb_options_create(); if (NULL == options) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -72,6 +124,10 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { } rocksdb_options_set_create_if_missing(options, 1); + rocksdb_options_set_comparator(options, cmp); + rocksdb_block_based_options_set_block_cache(tableoptions, cache); + rocksdb_options_set_block_based_table_factory(options, tableoptions); + rocksdb_options_set_info_log_level(options, 2); // WARN_LEVEL // rocksdb_options_set_inplace_update_support(options, 1); // rocksdb_options_set_allow_concurrent_memtable_write(options, 0); @@ -80,12 +136,12 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err2; } - // rocksdb_writeoptions_disable_WAL(writeoptions, 1); + rocksdb_writeoptions_disable_WAL(writeoptions, 1); rocksdb_readoptions_t *readoptions = rocksdb_readoptions_create(); if (NULL == readoptions) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err2; + goto _err3; } char *err = NULL; @@ -94,19 +150,25 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { rocksdb_t *db = rocksdb_open(options, cachePath, &err); if (NULL == db) { - code = -1; - goto _err3; + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); + rocksdb_free(err); + + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err4; } rocksdb_flushoptions_t *flushoptions = rocksdb_flushoptions_create(); if (NULL == flushoptions) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err4; + goto _err5; } rocksdb_writebatch_t *writebatch = rocksdb_writebatch_create(); + rocksdb_writebatch_t *rwritebatch = rocksdb_writebatch_create(); pTsdb->rCache.writebatch = writebatch; + pTsdb->rCache.rwritebatch = rwritebatch; + pTsdb->rCache.my_comparator = cmp; pTsdb->rCache.options = options; pTsdb->rCache.writeoptions = writeoptions; pTsdb->rCache.readoptions = readoptions; @@ -115,15 +177,22 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { taosThreadMutexInit(&pTsdb->rCache.rMutex, NULL); + pTsdb->rCache.pTSchema = NULL; + return code; +_err5: + rocksdb_close(pTsdb->rCache.db); _err4: rocksdb_readoptions_destroy(readoptions); _err3: rocksdb_writeoptions_destroy(writeoptions); _err2: rocksdb_options_destroy(options); + rocksdb_block_based_options_destroy(tableoptions); + rocksdb_cache_destroy(cache); _err: + rocksdb_comparator_destroy(cmp); return code; } @@ -131,27 +200,55 @@ static void tsdbCloseRocksCache(STsdb *pTsdb) { rocksdb_close(pTsdb->rCache.db); rocksdb_flushoptions_destroy(pTsdb->rCache.flushoptions); rocksdb_writebatch_destroy(pTsdb->rCache.writebatch); + rocksdb_writebatch_destroy(pTsdb->rCache.rwritebatch); rocksdb_readoptions_destroy(pTsdb->rCache.readoptions); rocksdb_writeoptions_destroy(pTsdb->rCache.writeoptions); rocksdb_options_destroy(pTsdb->rCache.options); + rocksdb_block_based_options_destroy(pTsdb->rCache.tableoptions); + rocksdb_cache_destroy(pTsdb->rCache.blockcache); + rocksdb_comparator_destroy(pTsdb->rCache.my_comparator); taosThreadMutexDestroy(&pTsdb->rCache.rMutex); + taosMemoryFree(pTsdb->rCache.pTSchema); } -int32_t tsdbCacheCommit(STsdb *pTsdb) { - int32_t code = 0; - char *err = NULL; +static void rocksMayWrite(STsdb *pTsdb, bool force, bool read, bool lock) { + rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; + if (read) { + if (lock) { + taosThreadMutexLock(&pTsdb->lruMutex); + } + wb = pTsdb->rCache.rwritebatch; + } else { + if (lock) { + taosThreadMutexLock(&pTsdb->rCache.rMutex); + } + } - rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); - if (NULL != err) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); - rocksdb_free(err); - code = -1; + int count = rocksdb_writebatch_count(wb); + if ((force && count > 0) || count >= ROCKS_BATCH_SIZE) { + char *err = NULL; + + rocksdb_write(pTsdb->rCache.db, pTsdb->rCache.writeoptions, wb, &err); + if (NULL != err) { + tsdbError("vgId:%d, %s failed at line %d, count: %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, count, + err); + rocksdb_free(err); + // pTsdb->flushState.flush_count = 0; + } + + rocksdb_writebatch_clear(wb); } - return code; + if (lock) { + if (read) { + taosThreadMutexUnlock(&pTsdb->lruMutex); + } else { + taosThreadMutexUnlock(&pTsdb->rCache.rMutex); + } + } } -SLastCol *tsdbCacheDeserialize(char const *value) { +static SLastCol *tsdbCacheDeserialize(char const *value) { if (!value) { return NULL; } @@ -169,7 +266,7 @@ SLastCol *tsdbCacheDeserialize(char const *value) { return pLastCol; } -void tsdbCacheSerialize(SLastCol *pLastCol, char **value, size_t *size) { +static void tsdbCacheSerialize(SLastCol *pLastCol, char **value, size_t *size) { SColVal *pColVal = &pLastCol->colVal; size_t length = sizeof(*pLastCol); if (IS_VAR_DATA_TYPE(pColVal->type)) { @@ -191,15 +288,86 @@ void tsdbCacheSerialize(SLastCol *pLastCol, char **value, size_t *size) { *size = length; } -static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, char const *lstring) { +static void tsdbCachePutBatch(SLastCol *pLastCol, const void *key, size_t klen, SCacheFlushState *state) { + STsdb *pTsdb = state->pTsdb; + SRocksCache *rCache = &pTsdb->rCache; + rocksdb_writebatch_t *wb = rCache->writebatch; + char *rocks_value = NULL; + size_t vlen = 0; + + tsdbCacheSerialize(pLastCol, &rocks_value, &vlen); + + taosThreadMutexLock(&rCache->rMutex); + + rocksdb_writebatch_put(wb, (char *)key, klen, rocks_value, vlen); + + taosMemoryFree(rocks_value); + + if (++state->flush_count >= ROCKS_BATCH_SIZE) { + char *err = NULL; + + rocksdb_write(rCache->db, rCache->writeoptions, wb, &err); + if (NULL != err) { + tsdbError("vgId:%d, %s failed at line %d, count: %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, + state->flush_count, err); + rocksdb_free(err); + } + + rocksdb_writebatch_clear(wb); + + state->flush_count = 0; + } + + taosThreadMutexUnlock(&rCache->rMutex); +} + +int tsdbCacheFlushDirty(const void *key, size_t klen, void *value, void *ud) { + SLastCol *pLastCol = (SLastCol *)value; + + if (pLastCol->dirty) { + tsdbCachePutBatch(pLastCol, key, klen, (SCacheFlushState *)ud); + + pLastCol->dirty = 0; + } + + return 0; +} + +int32_t tsdbCacheCommit(STsdb *pTsdb) { + int32_t code = 0; + char *err = NULL; + + SLRUCache *pCache = pTsdb->lruCache; + rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; + + taosThreadMutexLock(&pTsdb->lruMutex); + + taosLRUCacheApply(pCache, tsdbCacheFlushDirty, &pTsdb->flushState); + + rocksMayWrite(pTsdb, true, false, false); + rocksMayWrite(pTsdb, true, true, false); + rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); + + taosThreadMutexUnlock(&pTsdb->lruMutex); + + if (NULL != err) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); + rocksdb_free(err); + code = -1; + } + + return code; +} + +static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t ltype) { SLastCol *pLastCol = NULL; - char *err = NULL; - size_t vlen = 0; - char key[ROCKS_KEY_LEN]; - size_t klen = snprintf(key, ROCKS_KEY_LEN, "%" PRIi64 ":%" PRIi16 ":%s", uid, cid, lstring); - char *value = NULL; - value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, key, klen, &vlen, &err); + char *err = NULL; + size_t vlen = 0; + SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid}; + size_t klen = ROCKS_KEY_LEN; + char *value = NULL; + value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, (char *)key, klen, &vlen, &err); if (NULL != err) { tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); rocksdb_free(err); @@ -210,12 +378,42 @@ static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, char c return pLastCol; } +static void reallocVarData(SColVal *pColVal) { + if (IS_VAR_DATA_TYPE(pColVal->type)) { + uint8_t *pVal = pColVal->value.pData; + pColVal->value.pData = taosMemoryMalloc(pColVal->value.nData); + if (pColVal->value.nData) { + memcpy(pColVal->value.pData, pVal, pColVal->value.nData); + } + } +} + +static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud) { + SLastCol *pLastCol = (SLastCol *)value; + + if (pLastCol->dirty) { + tsdbCachePutBatch(pLastCol, key, klen, (SCacheFlushState *)ud); + } + + if (IS_VAR_DATA_TYPE(pLastCol->colVal.type) /* && pLastCol->colVal.value.nData > 0*/) { + taosMemoryFree(pLastCol->colVal.value.pData); + } + + taosMemoryFree(value); +} + +typedef struct { + int idx; + SLastKey key; +} SIdxKey; + int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow) { int32_t code = 0; // 1, fetch schema STSchema *pTSchema = NULL; int32_t sver = TSDBROW_SVERSION(pRow); + code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema); if (code != TSDB_CODE_SUCCESS) { terrno = code; @@ -229,111 +427,212 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow tsdbRowIterOpen(&iter, pRow, pTSchema); for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal; pColVal = tsdbRowIterNext(&iter)) { - /* - if (IS_VAR_DATA_TYPE(pColVal->type)) { - uint8_t *pVal = pColVal->value.pData; - - pColVal->value.pData = NULL; - code = tRealloc(&pColVal->value.pData, pColVal->value.nData); - if (code) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - if (pColVal->value.nData) { - memcpy(pColVal->value.pData, pVal, pColVal->value.nData); - } - } - */ taosArrayPush(aColVal, pColVal); } tsdbRowClose(&iter); // 3, build keys & multi get from rocks - int num_keys = TARRAY_SIZE(aColVal); - char **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); - size_t *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); + int num_keys = TARRAY_SIZE(aColVal); + TSKEY keyTs = TSDBROW_TS(pRow); + SArray *remainCols = NULL; + SLRUCache *pCache = pTsdb->lruCache; + + taosThreadMutexLock(&pTsdb->lruMutex); for (int i = 0; i < num_keys; ++i) { SColVal *pColVal = (SColVal *)taosArrayGet(aColVal, i); int16_t cid = pColVal->cid; - char *keys = taosMemoryCalloc(2, ROCKS_KEY_LEN); - int last_key_len = snprintf(keys, ROCKS_KEY_LEN, "%" PRIi64 ":%" PRIi16 ":last", uid, cid); - if (last_key_len >= ROCKS_KEY_LEN) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + SLastKey *key = &(SLastKey){.ltype = 0, .uid = uid, .cid = cid}; + size_t klen = ROCKS_KEY_LEN; + LRUHandle *h = taosLRUCacheLookup(pCache, key, klen); + if (h) { + SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h); + + if (pLastCol->ts <= keyTs) { + uint8_t *pVal = NULL; + int nData = pLastCol->colVal.value.nData; + if (IS_VAR_DATA_TYPE(pColVal->type)) { + pVal = pLastCol->colVal.value.pData; + } + pLastCol->ts = keyTs; + pLastCol->colVal = *pColVal; + if (IS_VAR_DATA_TYPE(pColVal->type)) { + if (nData < pColVal->value.nData) { + taosMemoryFree(pVal); + pLastCol->colVal.value.pData = taosMemoryCalloc(1, pColVal->value.nData); + } else { + pLastCol->colVal.value.pData = pVal; + } + if (pColVal->value.nData) { + memcpy(pLastCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); + } + } + + if (!pLastCol->dirty) { + pLastCol->dirty = 1; + } + } + + taosLRUCacheRelease(pCache, h, false); + } else { + if (!remainCols) { + remainCols = taosArrayInit(num_keys * 2, sizeof(SIdxKey)); + } + taosArrayPush(remainCols, &(SIdxKey){i, *key}); } - int lr_key_len = snprintf(keys + ROCKS_KEY_LEN, ROCKS_KEY_LEN, "%" PRIi64 ":%" PRIi16 ":last_row", uid, cid); - if (lr_key_len >= ROCKS_KEY_LEN) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + + if (COL_VAL_IS_VALUE(pColVal)) { + key->ltype = 1; + LRUHandle *h = taosLRUCacheLookup(pCache, key, klen); + if (h) { + SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h); + + if (pLastCol->ts <= keyTs) { + uint8_t *pVal = NULL; + int nData = pLastCol->colVal.value.nData; + if (IS_VAR_DATA_TYPE(pColVal->type)) { + pVal = pLastCol->colVal.value.pData; + } + pLastCol->ts = keyTs; + pLastCol->colVal = *pColVal; + if (IS_VAR_DATA_TYPE(pColVal->type)) { + if (nData < pColVal->value.nData) { + taosMemoryFree(pVal); + pLastCol->colVal.value.pData = taosMemoryCalloc(1, pColVal->value.nData); + } else { + pLastCol->colVal.value.pData = pVal; + } + if (pColVal->value.nData) { + memcpy(pLastCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); + } + } + + if (!pLastCol->dirty) { + pLastCol->dirty = 1; + } + } + + taosLRUCacheRelease(pCache, h, false); + } else { + if (!remainCols) { + remainCols = taosArrayInit(num_keys * 2, sizeof(SIdxKey)); + } + taosArrayPush(remainCols, &(SIdxKey){i, *key}); + } } - keys_list[i] = keys; - keys_list[num_keys + i] = keys + ROCKS_KEY_LEN; - keys_list_sizes[i] = last_key_len; - keys_list_sizes[num_keys + i] = lr_key_len; - } - char **values_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); - size_t *values_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); - char **errs = taosMemoryCalloc(num_keys * 2, sizeof(char *)); - taosThreadMutexLock(&pTsdb->rCache.rMutex); - rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys * 2, (const char *const *)keys_list, - keys_list_sizes, values_list, values_list_sizes, errs); - for (int i = 0; i < num_keys; ++i) { - taosMemoryFree(keys_list[i]); } - for (int i = 0; i < num_keys * 2; ++i) { - rocksdb_free(errs[i]); + + if (remainCols) { + num_keys = TARRAY_SIZE(remainCols); } - taosMemoryFree(keys_list); - taosMemoryFree(keys_list_sizes); - taosMemoryFree(errs); + if (remainCols && num_keys > 0) { + char **keys_list = taosMemoryCalloc(num_keys, sizeof(char *)); + size_t *keys_list_sizes = taosMemoryCalloc(num_keys, sizeof(size_t)); + for (int i = 0; i < num_keys; ++i) { + SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i]; + + keys_list[i] = (char *)&idxKey->key; + keys_list_sizes[i] = ROCKS_KEY_LEN; + } + char **values_list = taosMemoryCalloc(num_keys, sizeof(char *)); + size_t *values_list_sizes = taosMemoryCalloc(num_keys, sizeof(size_t)); + char **errs = taosMemoryCalloc(num_keys, sizeof(char *)); + rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys, (const char *const *)keys_list, + keys_list_sizes, values_list, values_list_sizes, errs); + for (int i = 0; i < num_keys; ++i) { + rocksdb_free(errs[i]); + } + taosMemoryFree(errs); + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); + taosMemoryFree(values_list_sizes); + + rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; + for (int i = 0; i < num_keys; ++i) { + SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i]; + SColVal *pColVal = (SColVal *)TARRAY_DATA(aColVal) + idxKey->idx; + // SColVal *pColVal = (SColVal *)taosArrayGet(aColVal, idxKey->idx); - TSKEY keyTs = TSDBROW_TS(pRow); - rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; - for (int i = 0; i < num_keys; ++i) { - SColVal *pColVal = (SColVal *)taosArrayGet(aColVal, i); - if (COL_VAL_IS_VALUE(pColVal)) { SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i]); - if (NULL == pLastCol || pLastCol->ts <= keyTs) { - char *value = NULL; - size_t vlen = 0; - tsdbCacheSerialize(&(SLastCol){.ts = keyTs, .colVal = *pColVal}, &value, &vlen); - char key[ROCKS_KEY_LEN]; - size_t klen = snprintf(key, ROCKS_KEY_LEN, "%" PRIi64 ":%" PRIi16 ":last", uid, pColVal->cid); - rocksdb_writebatch_put(wb, key, klen, value, vlen); - taosMemoryFree(value); - } - } + if (idxKey->key.ltype == 0) { + if (NULL == pLastCol || pLastCol->ts <= keyTs) { + char *value = NULL; + size_t vlen = 0; + tsdbCacheSerialize(&(SLastCol){.ts = keyTs, .colVal = *pColVal}, &value, &vlen); + // SLastKey key = (SLastKey){.ltype = 0, .uid = uid, .cid = pColVal->cid}; + taosThreadMutexLock(&pTsdb->rCache.rMutex); - if (!COL_VAL_IS_NONE(pColVal)) { - SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i + num_keys]); + rocksdb_writebatch_put(wb, (char *)&idxKey->key, ROCKS_KEY_LEN, value, vlen); - if (NULL == pLastCol || pLastCol->ts <= keyTs) { - char *value = NULL; - size_t vlen = 0; - tsdbCacheSerialize(&(SLastCol){.ts = keyTs, .colVal = *pColVal}, &value, &vlen); - char key[ROCKS_KEY_LEN]; - size_t klen = snprintf(key, ROCKS_KEY_LEN, "%" PRIi64 ":%" PRIi16 ":last_row", uid, pColVal->cid); - rocksdb_writebatch_put(wb, key, klen, value, vlen); - taosMemoryFree(value); + taosThreadMutexUnlock(&pTsdb->rCache.rMutex); + + pLastCol = (SLastCol *)value; + SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); + *pTmpLastCol = *pLastCol; + pLastCol = pTmpLastCol; + + reallocVarData(&pLastCol->colVal); + size_t charge = sizeof(*pLastCol); + if (IS_VAR_DATA_TYPE(pLastCol->colVal.type)) { + charge += pLastCol->colVal.value.nData; + } + + LRUStatus status = taosLRUCacheInsert(pTsdb->lruCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge, + tsdbCacheDeleter, NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + if (status != TAOS_LRU_STATUS_OK) { + code = -1; + } + + taosMemoryFree(value); + } + } else { + if (COL_VAL_IS_VALUE(pColVal)) { + if (NULL == pLastCol || pLastCol->ts <= keyTs) { + char *value = NULL; + size_t vlen = 0; + tsdbCacheSerialize(&(SLastCol){.ts = keyTs, .colVal = *pColVal}, &value, &vlen); + // SLastKey key = (SLastKey){.ltype = 1, .uid = uid, .cid = pColVal->cid}; + taosThreadMutexLock(&pTsdb->rCache.rMutex); + + rocksdb_writebatch_put(wb, (char *)&idxKey->key, ROCKS_KEY_LEN, value, vlen); + + taosThreadMutexUnlock(&pTsdb->rCache.rMutex); + + pLastCol = (SLastCol *)value; + SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); + *pTmpLastCol = *pLastCol; + pLastCol = pTmpLastCol; + + reallocVarData(&pLastCol->colVal); + size_t charge = sizeof(*pLastCol); + if (IS_VAR_DATA_TYPE(pLastCol->colVal.type)) { + charge += pLastCol->colVal.value.nData; + } + + LRUStatus status = taosLRUCacheInsert(pTsdb->lruCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge, + tsdbCacheDeleter, NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + if (status != TAOS_LRU_STATUS_OK) { + code = -1; + } + + taosMemoryFree(value); + } + } } + + rocksdb_free(values_list[i]); } - rocksdb_free(values_list[i]); - rocksdb_free(values_list[i + num_keys]); - } - taosMemoryFree(values_list); - taosMemoryFree(values_list_sizes); + rocksMayWrite(pTsdb, true, false, true); - char *err = NULL; - rocksdb_write(pTsdb->rCache.db, pTsdb->rCache.writeoptions, wb, &err); - if (NULL != err) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); - rocksdb_free(err); + taosMemoryFree(values_list); + + taosArrayDestroy(remainCols); } - taosThreadMutexUnlock(&pTsdb->rCache.rMutex); - rocksdb_writebatch_clear(wb); + + taosThreadMutexUnlock(&pTsdb->lruMutex); _exit: taosArrayDestroy(aColVal); @@ -341,53 +640,41 @@ _exit: return code; } -static void reallocVarData(SColVal *pColVal) { - if (IS_VAR_DATA_TYPE(pColVal->type)) { - uint8_t *pVal = pColVal->value.pData; - pColVal->value.pData = taosMemoryMalloc(pColVal->value.nData); - if (pColVal->value.nData) { - memcpy(pColVal->value.pData, pVal, pColVal->value.nData); - } - } -} - static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols, int nCols, int16_t *slotIds); static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols, int nCols, int16_t *slotIds); - -int32_t tsdbCacheGet(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int32_t ltype) { - static char const *alstring[2] = {"last_row", "last"}; - char const *lstring = alstring[ltype]; +#if 1 +int32_t tsdbCacheGetSlow(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) { rocksdb_writebatch_t *wb = NULL; int32_t code = 0; SArray *pCidList = pr->pCidList; int num_keys = TARRAY_SIZE(pCidList); - char **keys_list = taosMemoryCalloc(num_keys, sizeof(char *)); - size_t *keys_list_sizes = taosMemoryCalloc(num_keys, sizeof(size_t)); + + char **keys_list = taosMemoryMalloc(num_keys * sizeof(char *)); + size_t *keys_list_sizes = taosMemoryMalloc(num_keys * sizeof(size_t)); + char *key_list = taosMemoryMalloc(num_keys * ROCKS_KEY_LEN); for (int i = 0; i < num_keys; ++i) { int16_t cid = *(int16_t *)taosArrayGet(pCidList, i); - char *keys = taosMemoryCalloc(2, ROCKS_KEY_LEN); - int last_key_len = snprintf(keys, ROCKS_KEY_LEN, "%" PRIi64 ":%" PRIi16 ":%s", uid, cid, lstring); - if (last_key_len >= ROCKS_KEY_LEN) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); - } - - keys_list[i] = keys; - keys_list_sizes[i] = last_key_len; + memcpy(key_list + i * ROCKS_KEY_LEN, &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid}, ROCKS_KEY_LEN); + keys_list[i] = key_list + i * ROCKS_KEY_LEN; + keys_list_sizes[i] = ROCKS_KEY_LEN; } + char **values_list = taosMemoryCalloc(num_keys, sizeof(char *)); size_t *values_list_sizes = taosMemoryCalloc(num_keys, sizeof(size_t)); - char **errs = taosMemoryCalloc(num_keys, sizeof(char *)); + char **errs = taosMemoryMalloc(num_keys * sizeof(char *)); rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys, (const char *const *)keys_list, keys_list_sizes, values_list, values_list_sizes, errs); for (int i = 0; i < num_keys; ++i) { - taosMemoryFree(keys_list[i]); - rocksdb_free(errs[i]); + if (errs[i]) { + rocksdb_free(errs[i]); + } } + taosMemoryFree(key_list); taosMemoryFree(keys_list); taosMemoryFree(keys_list_sizes); taosMemoryFree(errs); @@ -403,7 +690,7 @@ int32_t tsdbCacheGet(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsR } else { taosThreadMutexLock(&pTsdb->rCache.rMutex); - pLastCol = tsdbCacheLookup(pTsdb, uid, cid, lstring); + pLastCol = tsdbCacheLookup(pTsdb, uid, cid, ltype); if (!pLastCol) { // recalc: load from tsdb int16_t aCols[1] = {cid}; @@ -428,13 +715,14 @@ int32_t tsdbCacheGet(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsR } // store result back to rocks cache - wb = pTsdb->rCache.writebatch; + wb = pTsdb->rCache.rwritebatch; char *value = NULL; size_t vlen = 0; tsdbCacheSerialize(pLastCol, &value, &vlen); - char key[ROCKS_KEY_LEN]; - size_t klen = snprintf(key, ROCKS_KEY_LEN, "%" PRIi64 ":%" PRIi16 ":%s", uid, pLastCol->colVal.cid, lstring); - rocksdb_writebatch_put(wb, key, klen, value, vlen); + + SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = pLastCol->colVal.cid}; + size_t klen = ROCKS_KEY_LEN; + rocksdb_writebatch_put(wb, (char *)key, klen, value, vlen); taosMemoryFree(value); } else { @@ -442,21 +730,13 @@ int32_t tsdbCacheGet(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsR } if (wb) { - char *err = NULL; - rocksdb_write(pTsdb->rCache.db, pTsdb->rCache.writeoptions, wb, &err); - if (NULL != err) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); - rocksdb_free(err); - } - - rocksdb_writebatch_clear(wb); + rocksMayWrite(pTsdb, false, true, false); } taosThreadMutexUnlock(&pTsdb->rCache.rMutex); } taosArrayPush(pLastArray, pLastCol); - taosArrayDestroy(pTmpColArray); if (freeCol) { taosMemoryFree(pLastCol); @@ -467,85 +747,384 @@ int32_t tsdbCacheGet(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsR return code; } +#endif + +static SLastCol *tsdbCacheLoadCol(STsdb *pTsdb, SCacheRowsReader *pr, int16_t slotid, tb_uid_t uid, int16_t cid, + int8_t ltype) { + SLastCol *pLastCol = tsdbCacheLookup(pTsdb, uid, cid, ltype); + if (!pLastCol) { + rocksdb_writebatch_t *wb = NULL; + + taosThreadMutexLock(&pTsdb->rCache.rMutex); + pLastCol = tsdbCacheLookup(pTsdb, uid, cid, ltype); + if (!pLastCol) { + // recalc: load from tsdb + int16_t aCols[1] = {cid}; + int16_t slotIds[1] = {slotid}; + SArray *pTmpColArray = NULL; + + if (ltype) { + mergeLastCid(uid, pTsdb, &pTmpColArray, pr, aCols, 1, slotIds); + } else { + mergeLastRowCid(uid, pTsdb, &pTmpColArray, pr, aCols, 1, slotIds); + } -int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) { - int32_t code = 0; - // 1, fetch schema - STSchema *pTSchema = NULL; - int32_t sver = -1; - code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return -1; - } + if (pTmpColArray && TARRAY_SIZE(pTmpColArray) >= 1) { + pLastCol = taosArrayGet(pTmpColArray, 0); + } - // 3, build keys & multi get from rocks - int num_keys = pTSchema->numOfCols; - char **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); - size_t *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); - for (int i = 0; i < num_keys; ++i) { - int16_t cid = pTSchema->columns[i].colId; + // still null, then make up a none col value + SLastCol noneCol = {.ts = TSKEY_MIN, .colVal = COL_VAL_NONE(cid, pr->pSchema->columns[slotid].type)}; + if (!pLastCol) { + pLastCol = &noneCol; + } + + // store result back to rocks cache + wb = pTsdb->rCache.rwritebatch; + char *value = NULL; + size_t vlen = 0; + tsdbCacheSerialize(pLastCol, &value, &vlen); + + SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = pLastCol->colVal.cid}; + size_t klen = ROCKS_KEY_LEN; + rocksdb_writebatch_put(wb, (char *)key, klen, value, vlen); + taosMemoryFree(value); + + SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); + *pTmpLastCol = *pLastCol; + pLastCol = pTmpLastCol; - char *keys = taosMemoryCalloc(2, ROCKS_KEY_LEN); - int last_key_len = snprintf(keys, ROCKS_KEY_LEN, "%" PRIi64 ":%" PRIi16 ":last", uid, cid); - if (last_key_len >= ROCKS_KEY_LEN) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + taosArrayDestroy(pTmpColArray); } - int lr_key_len = snprintf(keys + ROCKS_KEY_LEN, ROCKS_KEY_LEN, "%" PRIi64 ":%" PRIi16 ":last_row", uid, cid); - if (lr_key_len >= ROCKS_KEY_LEN) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + + if (wb) { + rocksMayWrite(pTsdb, false, true, false); + } + + taosThreadMutexUnlock(&pTsdb->rCache.rMutex); + } + + return pLastCol; +} + +static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SArray *remainCols, + SCacheRowsReader *pr, int8_t ltype) { + int32_t code = 0; + rocksdb_writebatch_t *wb = NULL; + SArray *pTmpColArray = NULL; + int num_keys = TARRAY_SIZE(remainCols); + int16_t *aCols = taosMemoryMalloc(num_keys * sizeof(int16_t)); + int16_t *slotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); + + for (int i = 0; i < num_keys; ++i) { + SIdxKey *idxKey = taosArrayGet(remainCols, i); + aCols[i] = idxKey->key.cid; + slotIds[i] = pr->pSlotIds[idxKey->idx]; + } + + if (ltype) { + mergeLastCid(uid, pTsdb, &pTmpColArray, pr, aCols, num_keys, slotIds); + } else { + mergeLastRowCid(uid, pTsdb, &pTmpColArray, pr, aCols, num_keys, slotIds); + } + + SLRUCache *pCache = pTsdb->lruCache; + for (int i = 0; i < num_keys; ++i) { + SIdxKey *idxKey = taosArrayGet(remainCols, i); + SLastCol *pLastCol = NULL; + + if (pTmpColArray && TARRAY_SIZE(pTmpColArray) >= i + 1) { + pLastCol = taosArrayGet(pTmpColArray, i); + } + + // still null, then make up a none col value + SLastCol noneCol = {.ts = TSKEY_MIN, + .colVal = COL_VAL_NONE(idxKey->key.cid, pr->pSchema->columns[slotIds[i]].type)}; + if (!pLastCol) { + pLastCol = &noneCol; + reallocVarData(&pLastCol->colVal); + } + + taosArraySet(pLastArray, idxKey->idx, pLastCol); + // taosArrayRemove(remainCols, i); + + if (!pTmpColArray) { + continue; + } + + SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); + *pTmpLastCol = *pLastCol; + pLastCol = pTmpLastCol; + + reallocVarData(&pLastCol->colVal); + size_t charge = sizeof(*pLastCol); + if (IS_VAR_DATA_TYPE(pLastCol->colVal.type)) { + charge += pLastCol->colVal.value.nData; + } + + LRUStatus status = taosLRUCacheInsert(pCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, NULL, + TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + if (status != TAOS_LRU_STATUS_OK) { + code = -1; + } + + // store result back to rocks cache + wb = pTsdb->rCache.rwritebatch; + char *value = NULL; + size_t vlen = 0; + tsdbCacheSerialize(pLastCol, &value, &vlen); + + SLastKey *key = &idxKey->key; + size_t klen = ROCKS_KEY_LEN; + rocksdb_writebatch_put(wb, (char *)key, klen, value, vlen); + taosMemoryFree(value); + } + + if (wb) { + rocksMayWrite(pTsdb, false, true, false); + } + + taosArrayDestroy(pTmpColArray); + + taosMemoryFree(aCols); + taosMemoryFree(slotIds); + + return code; +} + +static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SArray *remainCols, + SCacheRowsReader *pr, int8_t ltype) { + int32_t code = 0; + int num_keys = TARRAY_SIZE(remainCols); + char **keys_list = taosMemoryMalloc(num_keys * sizeof(char *)); + size_t *keys_list_sizes = taosMemoryMalloc(num_keys * sizeof(size_t)); + char *key_list = taosMemoryMalloc(num_keys * ROCKS_KEY_LEN); + for (int i = 0; i < num_keys; ++i) { + int16_t cid = *(int16_t *)taosArrayGet(remainCols, i); + + memcpy(key_list + i * ROCKS_KEY_LEN, &((SIdxKey *)taosArrayGet(remainCols, i))->key, ROCKS_KEY_LEN); + keys_list[i] = key_list + i * ROCKS_KEY_LEN; + keys_list_sizes[i] = ROCKS_KEY_LEN; + } + + char **values_list = taosMemoryCalloc(num_keys, sizeof(char *)); + size_t *values_list_sizes = taosMemoryCalloc(num_keys, sizeof(size_t)); + char **errs = taosMemoryMalloc(num_keys * sizeof(char *)); + rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys, (const char *const *)keys_list, + keys_list_sizes, values_list, values_list_sizes, errs); + for (int i = 0; i < num_keys; ++i) { + if (errs[i]) { + rocksdb_free(errs[i]); + } + } + taosMemoryFree(key_list); + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); + taosMemoryFree(errs); + + SLRUCache *pCache = pTsdb->lruCache; + for (int i = 0, j = 0; i < num_keys && j < TARRAY_SIZE(remainCols); ++i) { + SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i]); + SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[j]; + if (pLastCol) { + SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); + *pTmpLastCol = *pLastCol; + pLastCol = pTmpLastCol; + + reallocVarData(&pLastCol->colVal); + size_t charge = sizeof(*pLastCol); + if (IS_VAR_DATA_TYPE(pLastCol->colVal.type)) { + charge += pLastCol->colVal.value.nData; + } + + LRUStatus status = taosLRUCacheInsert(pCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, + NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); + if (status != TAOS_LRU_STATUS_OK) { + code = -1; + } + + SLastCol lastCol = *pLastCol; + reallocVarData(&lastCol.colVal); + taosArraySet(pLastArray, idxKey->idx, &lastCol); + taosArrayRemove(remainCols, j); + + taosMemoryFree(values_list[i]); + } else { + ++j; + } + } + + taosMemoryFree(values_list); + taosMemoryFree(values_list_sizes); + + if (TARRAY_SIZE(remainCols) > 0) { + code = tsdbCacheLoadFromRaw(pTsdb, uid, pLastArray, remainCols, pr, ltype); + } + + return code; +} + +int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) { + int32_t code = 0; + SArray *remainCols = NULL; + SLRUCache *pCache = pTsdb->lruCache; + SArray *pCidList = pr->pCidList; + int num_keys = TARRAY_SIZE(pCidList); + + for (int i = 0; i < num_keys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + + SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid}; + + LRUHandle *h = taosLRUCacheLookup(pCache, key, ROCKS_KEY_LEN); + if (h) { + SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h); + + SLastCol lastCol = *pLastCol; + reallocVarData(&lastCol.colVal); + taosArrayPush(pLastArray, &lastCol); + + taosLRUCacheRelease(pCache, h, false); + } else { + SLastCol noneCol = {.ts = TSKEY_MIN, .colVal = COL_VAL_NONE(cid, pr->pSchema->columns[pr->pSlotIds[i]].type)}; + + taosArrayPush(pLastArray, &noneCol); + + if (!remainCols) { + remainCols = taosArrayInit(num_keys, sizeof(SIdxKey)); + } + taosArrayPush(remainCols, &(SIdxKey){i, *key}); } + } + + if (remainCols && TARRAY_SIZE(remainCols) > 0) { + taosThreadMutexLock(&pTsdb->lruMutex); + for (int i = 0; i < TARRAY_SIZE(remainCols);) { + SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i]; + LRUHandle *h = taosLRUCacheLookup(pCache, &idxKey->key, ROCKS_KEY_LEN); + if (h) { + SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h); + + SLastCol lastCol = *pLastCol; + reallocVarData(&lastCol.colVal); + taosArraySet(pLastArray, idxKey->idx, &lastCol); + + taosLRUCacheRelease(pCache, h, false); + + taosArrayRemove(remainCols, i); + } else { + ++i; + } + } + + code = tsdbCacheLoadFromRocks(pTsdb, uid, pLastArray, remainCols, pr, ltype); + + taosThreadMutexUnlock(&pTsdb->lruMutex); + + if (remainCols) { + taosArrayDestroy(remainCols); + } + } + + return code; +} + +int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) { + int32_t code = 0; + // fetch schema + STSchema *pTSchema = NULL; + int sver = -1; + code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return -1; + } + + // build keys & multi get from rocks + int num_keys = pTSchema->numOfCols; + char **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); + size_t *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); + const size_t klen = ROCKS_KEY_LEN; + for (int i = 0; i < num_keys; ++i) { + int16_t cid = pTSchema->columns[i].colId; + + char *keys = taosMemoryCalloc(2, sizeof(SLastKey)); + ((SLastKey *)keys)[0] = (SLastKey){.ltype = 1, .uid = uid, .cid = cid}; + ((SLastKey *)keys)[1] = (SLastKey){.ltype = 0, .uid = uid, .cid = cid}; + keys_list[i] = keys; - keys_list[num_keys + i] = keys + ROCKS_KEY_LEN; - keys_list_sizes[i] = last_key_len; - keys_list_sizes[num_keys + i] = lr_key_len; + keys_list[num_keys + i] = keys + sizeof(SLastKey); + keys_list_sizes[i] = klen; + keys_list_sizes[num_keys + i] = klen; } char **values_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); size_t *values_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); char **errs = taosMemoryCalloc(num_keys * 2, sizeof(char *)); + taosThreadMutexLock(&pTsdb->lruMutex); taosThreadMutexLock(&pTsdb->rCache.rMutex); + rocksMayWrite(pTsdb, true, false, false); rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys * 2, (const char *const *)keys_list, keys_list_sizes, values_list, values_list_sizes, errs); - for (int i = 0; i < num_keys; ++i) { - taosMemoryFree(keys_list[i]); - } + taosThreadMutexUnlock(&pTsdb->rCache.rMutex); + for (int i = 0; i < num_keys * 2; ++i) { - rocksdb_free(errs[i]); + if (errs[i]) { + rocksdb_free(errs[i]); + } } - taosMemoryFree(keys_list); - taosMemoryFree(keys_list_sizes); taosMemoryFree(errs); rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; for (int i = 0; i < num_keys; ++i) { SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i]); + taosThreadMutexLock(&pTsdb->rCache.rMutex); if (NULL != pLastCol && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { - char key[ROCKS_KEY_LEN]; - size_t klen = snprintf(key, ROCKS_KEY_LEN, "%" PRIi64 ":%" PRIi16 ":last", uid, pLastCol->colVal.cid); - rocksdb_writebatch_delete(wb, key, klen); + rocksdb_writebatch_delete(wb, keys_list[i], klen); } - pLastCol = tsdbCacheDeserialize(values_list[i + num_keys]); if (NULL != pLastCol && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { - char key[ROCKS_KEY_LEN]; - size_t klen = snprintf(key, ROCKS_KEY_LEN, "%" PRIi64 ":%" PRIi16 ":last_row", uid, pLastCol->colVal.cid); - rocksdb_writebatch_delete(wb, key, klen); + rocksdb_writebatch_delete(wb, keys_list[num_keys + i], klen); } + taosThreadMutexUnlock(&pTsdb->rCache.rMutex); rocksdb_free(values_list[i]); rocksdb_free(values_list[i + num_keys]); + + // taosThreadMutexLock(&pTsdb->lruMutex); + + LRUHandle *h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[i], klen); + if (h) { + SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h); + if (pLastCol->dirty) { + pLastCol->dirty = 0; + } + taosLRUCacheRelease(pTsdb->lruCache, h, true); + } + taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen); + + h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[num_keys + i], klen); + if (h) { + SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h); + if (pLastCol->dirty) { + pLastCol->dirty = 0; + } + taosLRUCacheRelease(pTsdb->lruCache, h, true); + } + taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen); + + // taosThreadMutexUnlock(&pTsdb->lruMutex); + } + for (int i = 0; i < num_keys; ++i) { + taosMemoryFree(keys_list[i]); } + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); taosMemoryFree(values_list); taosMemoryFree(values_list_sizes); - char *err = NULL; - rocksdb_write(pTsdb->rCache.db, pTsdb->rCache.writeoptions, wb, &err); - if (NULL != err) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); - rocksdb_free(err); - } - taosThreadMutexUnlock(&pTsdb->rCache.rMutex); - rocksdb_writebatch_clear(wb); + rocksMayWrite(pTsdb, true, false, true); + + taosThreadMutexUnlock(&pTsdb->lruMutex); _exit: taosMemoryFree(pTSchema); @@ -558,7 +1137,7 @@ int32_t tsdbOpenCache(STsdb *pTsdb) { SLRUCache *pCache = NULL; size_t cfgCapacity = pTsdb->pVnode->config.cacheLastSize * 1024 * 1024; - pCache = taosLRUCacheInit(cfgCapacity, 1, .5); + pCache = taosLRUCacheInit(cfgCapacity, 0, .5); if (pCache == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -580,6 +1159,9 @@ int32_t tsdbOpenCache(STsdb *pTsdb) { taosThreadMutexInit(&pTsdb->lruMutex, NULL); + pTsdb->flushState.pTsdb = pTsdb; + pTsdb->flushState.flush_count = 0; + _err: pTsdb->lruCache = pCache; return code; @@ -609,7 +1191,8 @@ static void getTableCacheKey(tb_uid_t uid, int cacheType, char *key, int *len) { *len = sizeof(uint64_t); } -static void deleteTableCacheLast(const void *key, size_t keyLen, void *value) { +static void deleteTableCacheLast(const void *key, size_t keyLen, void *value, void *ud) { + (void)ud; SArray *pLastArray = (SArray *)value; int16_t nCol = taosArrayGetSize(pLastArray); for (int16_t iCol = 0; iCol < nCol; ++iCol) { @@ -683,62 +1266,7 @@ int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { return code; } -/* -int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { - int32_t code = 0; - char key[32] = {0}; - int keyLen = 0; - - // getTableCacheKey(uid, "lr", key, &keyLen); - getTableCacheKey(uid, 0, key, &keyLen); - LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (h) { - SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); - bool invalidate = false; - int16_t nCol = taosArrayGetSize(pLast); - - for (int16_t iCol = 0; iCol < nCol; ++iCol) { - SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); - if (eKey >= tTsVal->ts) { - invalidate = true; - break; - } - } - - if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); - } - } - - // getTableCacheKey(uid, "l", key, &keyLen); - getTableCacheKey(uid, 1, key, &keyLen); - h = taosLRUCacheLookup(pCache, key, keyLen); - if (h) { - SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); - bool invalidate = false; - int16_t nCol = taosArrayGetSize(pLast); - - for (int16_t iCol = 0; iCol < nCol; ++iCol) { - SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); - if (eKey >= tTsVal->ts) { - invalidate = true; - break; - } - } - - if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); - } - // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); - } - return code; -} -*/ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup) { int32_t code = 0; STSRow *cacheRow = NULL; @@ -979,8 +1507,8 @@ static tb_uid_t getTableSuidByUid(tb_uid_t uid, STsdb *pTsdb) { tb_uid_t suid = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0); - if (metaGetTableEntryByUidCache(&mr, uid) < 0) { + metaReaderDoInit(&mr, pTsdb->pVnode->pMeta, 0); + if (metaReaderGetTableEntryByUidCache(&mr, uid) < 0) { metaReaderClear(&mr); // table not esist return 0; } @@ -1002,7 +1530,7 @@ static int32_t getTableDelDataFromDelIdx(SDelFReader *pDelReader, SDelIdx *pDelI int32_t code = 0; if (pDelIdx) { - code = tsdbReadDelData(pDelReader, pDelIdx, aDelData); + code = tsdbReadDelDatav1(pDelReader, pDelIdx, aDelData, INT64_MAX); } return code; @@ -1063,516 +1591,590 @@ _err: } return code; } -/* -static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t uid, SDelIdx *pDelIdx) { - int32_t code = 0; - SArray *pDelIdxArray = NULL; - // SMapData delIdxMap; - pDelIdxArray = taosArrayInit(32, sizeof(SDelIdx)); - SDelIdx idx = {.suid = suid, .uid = uid}; +static void freeTableInfoFunc(void *param) { + void **p = (void **)param; + taosMemoryFreeClear(*p); +} - // tMapDataReset(&delIdxMap); - code = tsdbReadDelIdx(pDelFReader, pDelIdxArray); - if (code) goto _err; +static STableLoadInfo *getTableLoadInfo(SCacheRowsReader *pReader, uint64_t uid) { + if (!pReader->pTableMap) { + pReader->pTableMap = tSimpleHashInit(pReader->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); - // code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx); - SDelIdx *pIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ); + tSimpleHashSetFreeFp(pReader->pTableMap, freeTableInfoFunc); + } - *pDelIdx = *pIdx; + STableLoadInfo *pInfo = NULL; + STableLoadInfo **ppInfo = tSimpleHashGet(pReader->pTableMap, &uid, sizeof(uid)); + if (!ppInfo) { + pInfo = taosMemoryCalloc(1, sizeof(STableLoadInfo)); + tSimpleHashPut(pReader->pTableMap, &uid, sizeof(uint64_t), &pInfo, POINTER_BYTES); -_err: - if (pDelIdxArray) { - taosArrayDestroy(pDelIdxArray); + return pInfo; } - return code; + + return *ppInfo; } -*/ -typedef enum { - SFSLASTNEXTROW_FS, - SFSLASTNEXTROW_FILESET, - SFSLASTNEXTROW_BLOCKDATA, - SFSLASTNEXTROW_BLOCKROW -} SFSLASTNEXTROWSTATES; -typedef struct { - SFSLASTNEXTROWSTATES state; // [input] - STsdb *pTsdb; // [input] - STSchema *pTSchema; // [input] - tb_uid_t suid; - tb_uid_t uid; - int32_t nFileSet; - int32_t iFileSet; - SArray *aDFileSet; - SDataFReader **pDataFReader; - TSDBROW row; - - bool checkRemainingRow; - SMergeTree mergeTree; - SMergeTree *pMergeTree; - SSttBlockLoadInfo *pLoadInfo; - SLDataIter* pDataIter; - int64_t lastTs; -} SFSLastNextRowIter; - -static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, - int nCols) { - SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter; - int32_t code = 0; - bool checkRemainingRow = true; +static uint64_t *getUidList(SCacheRowsReader *pReader) { + if (!pReader->uidList) { + int32_t numOfTables = pReader->numOfTables; - switch (state->state) { - case SFSLASTNEXTROW_FS: - state->nFileSet = taosArrayGetSize(state->aDFileSet); - state->iFileSet = state->nFileSet; + pReader->uidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); - case SFSLASTNEXTROW_FILESET: { - SDFileSet *pFileSet = NULL; - _next_fileset: - if (state->pMergeTree != NULL) { - tMergeTreeClose(state->pMergeTree); - state->pMergeTree = NULL; - } + for (int32_t i = 0; i < numOfTables; ++i) { + uint64_t uid = pReader->pTableList[i].uid; + pReader->uidList[i] = uid; + } - if (--state->iFileSet >= 0) { - pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); - } else { - *ppRow = NULL; - return code; - } + taosSort(pReader->uidList, numOfTables, sizeof(uint64_t), uidComparFunc); + } - if (*state->pDataFReader == NULL || (*state->pDataFReader)->pSet->fid != pFileSet->fid) { - if (*state->pDataFReader != NULL) { - tsdbDataFReaderClose(state->pDataFReader); + return pReader->uidList; +} - resetLastBlockLoadInfo(state->pLoadInfo); - } +static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsReader *pReader, void *pFileReader, + bool isFile) { + int32_t code = 0; + uint64_t *uidList = getUidList(pReader); + int32_t numOfTables = pReader->numOfTables; + int64_t suid = pReader->info.suid; - code = tsdbDataFReaderOpen(state->pDataFReader, state->pTsdb, pFileSet); - if (code) goto _err; + for (int i = 0, j = 0; i < pTombBlkArray->size && j < numOfTables; ++i) { + STombBlk *pTombBlk = &pTombBlkArray->data[i]; + if (pTombBlk->maxTbid.suid < suid || (pTombBlk->maxTbid.suid == suid && pTombBlk->maxTbid.uid < uidList[0])) { + continue; + } + + if (pTombBlk->minTbid.suid > suid || + (pTombBlk->minTbid.suid == suid && pTombBlk->minTbid.uid > uidList[numOfTables - 1])) { + break; + } + + STombBlock block = {0}; + code = isFile ? tsdbDataFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block) + : tsdbSttFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + uint64_t uid = uidList[j]; + STableLoadInfo *pInfo = getTableLoadInfo(pReader, uid); + if (pInfo->pTombData == NULL) { + pInfo->pTombData = taosArrayInit(4, sizeof(SDelData)); + } + + STombRecord record = {0}; + bool finished = false; + for (int32_t k = 0; k < TARRAY2_SIZE(block.suid); ++k) { + code = tTombBlockGet(&block, k, &record); + if (code != TSDB_CODE_SUCCESS) { + finished = true; + break; } - for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) { - state->pLoadInfo[i].colIds = aCols; - state->pLoadInfo[i].numOfCols = nCols; - state->pLoadInfo[i].isLast = isLast; + if (record.suid < suid) { + continue; + } + if (record.suid > suid) { + finished = true; + break; } - tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid, - &(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX}, - &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true, state->pDataIter); - state->pMergeTree = &state->mergeTree; - state->state = SFSLASTNEXTROW_BLOCKROW; - } - case SFSLASTNEXTROW_BLOCKROW: { - if (nCols != state->pLoadInfo->numOfCols) { - for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) { - state->pLoadInfo[i].numOfCols = nCols; - state->pLoadInfo[i].checkRemainingRow = state->checkRemainingRow; + bool newTable = false; + if (uid < record.uid) { + while (j < numOfTables && uidList[j] < record.uid) { + ++j; + newTable = true; } - } - bool hasVal = tMergeTreeNext(&state->mergeTree); - if (!hasVal) { - if (tMergeTreeIgnoreEarlierTs(&state->mergeTree)) { - *pIgnoreEarlierTs = true; - *ppRow = NULL; - return code; + + if (j >= numOfTables) { + finished = true; + break; } - state->state = SFSLASTNEXTROW_FILESET; - goto _next_fileset; - } - state->row = *tMergeTreeGetRow(&state->mergeTree); - *ppRow = &state->row; - if (TSDBROW_TS(&state->row) <= state->lastTs) { - *pIgnoreEarlierTs = true; - *ppRow = NULL; - return code; + uid = uidList[j]; } - *pIgnoreEarlierTs = false; - if (!hasVal) { - state->state = SFSLASTNEXTROW_FILESET; + if (record.uid < uid) { + continue; } - if (!state->checkRemainingRow) { - state->checkRemainingRow = true; + if (newTable) { + pInfo = getTableLoadInfo(pReader, uid); + if (pInfo->pTombData == NULL) { + pInfo->pTombData = taosArrayInit(4, sizeof(SDelData)); + } } + + if (record.version <= pReader->info.verRange.maxVer) { + tsdbError("tomb xx load/cache: vgId:%d fid:%d commit %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records", + TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid); + + SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey}; + taosArrayPush(pInfo->pTombData, &delData); + } + } + + tTombBlockDestroy(&block); + + if (finished) { return code; } - default: - ASSERT(0); - break; } -_err: - /*if (state->pDataFReader) { - tsdbDataFReaderClose(&state->pDataFReader); - state->pDataFReader = NULL; - }*/ - if (state->pMergeTree != NULL) { - tMergeTreeClose(state->pMergeTree); - state->pMergeTree = NULL; - } + return TSDB_CODE_SUCCESS; +} - *ppRow = NULL; +static int32_t loadDataTomb(SCacheRowsReader *pReader, SDataFileReader *pFileReader) { + int32_t code = 0; - return code; + const TTombBlkArray *pBlkArray = NULL; + code = tsdbDataFileReadTombBlk(pFileReader, &pBlkArray); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + return loadTombFromBlk(pBlkArray, pReader, pFileReader, true); } -int32_t clearNextRowFromFSLast(void *iter) { - SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter; - int32_t code = 0; +static int32_t loadSttTomb(STsdbReader *pTsdbReader, SSttFileReader *pSttFileReader, SSttBlockLoadInfo *pLoadInfo) { + int32_t code = 0; - if (!state) { + SCacheRowsReader *pReader = (SCacheRowsReader *)pTsdbReader; + + const TTombBlkArray *pBlkArray = NULL; + code = tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray); + if (code != TSDB_CODE_SUCCESS) { return code; } - /* - if (state->pDataFReader) { - tsdbDataFReaderClose(&state->pDataFReader); - state->pDataFReader = NULL; + + return loadTombFromBlk(pBlkArray, pReader, pSttFileReader, false); +} + +typedef struct { + SMergeTree mergeTree; + SMergeTree *pMergeTree; +} SFSLastIter; + +static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, + tb_uid_t uid, SCacheRowsReader *pr, int64_t lastTs, int16_t *aCols, int nCols) { + int32_t code = 0; + + int64_t loadBlocks = 0; + double elapse = 0; + pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, &loadBlocks, &elapse); + pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + + SMergeTreeConf conf = { + .uid = uid, + .suid = suid, + .pTsdb = pTsdb, + .timewindow = (STimeWindow){.skey = lastTs, .ekey = TSKEY_MAX}, + .verRange = (SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, + .strictTimeRange = false, + .pSchema = pTSchema, + .pCurrentFileset = pFileSet, + .backward = 1, + .pSttFileBlockIterArray = pr->pLDataIterArray, + .pCols = aCols, + .numOfCols = nCols, + .loadTombFn = loadSttTomb, + .pReader = pr, + .idstr = pr->idstr, + }; + + code = tMergeTreeOpen2(&iter->mergeTree, &conf); + if (code != TSDB_CODE_SUCCESS) { + return -1; + } + + iter->pMergeTree = &iter->mergeTree; + + return code; +} + +static int32_t lastIterClose(SFSLastIter **iter) { + int32_t code = 0; + + if ((*iter)->pMergeTree) { + tMergeTreeClose((*iter)->pMergeTree); + (*iter)->pMergeTree = NULL; } - */ - if (state->pMergeTree != NULL) { - tMergeTreeClose(state->pMergeTree); - state->pMergeTree = NULL; + + *iter = NULL; + + return code; +} + +static int32_t lastIterNext(SFSLastIter *iter, TSDBROW **ppRow) { + int32_t code = 0; + + bool hasVal = tMergeTreeNext(iter->pMergeTree); + if (!hasVal) { + *ppRow = NULL; + return code; } + *ppRow = tMergeTreeGetRow(iter->pMergeTree); + return code; } typedef enum SFSNEXTROWSTATES { SFSNEXTROW_FS, SFSNEXTROW_FILESET, + SFSNEXTROW_INDEXLIST, + SFSNEXTROW_BRINBLOCK, + SFSNEXTROW_BRINRECORD, SFSNEXTROW_BLOCKDATA, - SFSNEXTROW_BLOCKROW + SFSNEXTROW_BLOCKROW, + SFSNEXTROW_NEXTSTTROW } SFSNEXTROWSTATES; +struct CacheNextRowIter; + typedef struct SFSNextRowIter { - SFSNEXTROWSTATES state; // [input] - STsdb *pTsdb; // [input] - SBlockIdx *pBlockIdxExp; // [input] - STSchema *pTSchema; // [input] - tb_uid_t suid; - tb_uid_t uid; - int32_t nFileSet; - int32_t iFileSet; - SArray *aDFileSet; - SDataFReader **pDataFReader; - SArray *aBlockIdx; - LRUHandle *aBlockIdxHandle; - SBlockIdx *pBlockIdx; - SMapData blockMap; - int32_t nBlock; - int32_t iBlock; - SDataBlk block; - SBlockData blockData; - SBlockData *pBlockData; - int32_t nRow; - int32_t iRow; - TSDBROW row; - SSttBlockLoadInfo *pLoadInfo; - int64_t lastTs; + SFSNEXTROWSTATES state; // [input] + SBlockIdx *pBlockIdxExp; // [input] + STSchema *pTSchema; // [input] + tb_uid_t suid; + tb_uid_t uid; + int32_t iFileSet; + STFileSet *pFileSet; + TFileSetArray *aDFileSet; + SArray *pIndexList; + int32_t iBrinIndex; + SBrinBlock brinBlock; + int32_t iBrinRecord; + SBrinRecord brinRecord; + SBlockData blockData; + SBlockData *pBlockData; + int32_t nRow; + int32_t iRow; + TSDBROW row; + int64_t lastTs; + SFSLastIter lastIter; + SFSLastIter *pLastIter; + int8_t lastEmpty; + TSDBROW *pLastRow; + SRow *pTSRow; + SRowMerger rowMerger; + SCacheRowsReader *pr; + struct CacheNextRowIter *pRowIter; } SFSNextRowIter; +static void clearLastFileSet(SFSNextRowIter *state); + static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, int nCols) { SFSNextRowIter *state = (SFSNextRowIter *)iter; int32_t code = 0; - bool checkRemainingRow = true; + STsdb *pTsdb = state->pr->pTsdb; - switch (state->state) { - case SFSNEXTROW_FS: - // state->aDFileSet = state->pTsdb->pFS->cState->aDFileSet; - state->nFileSet = taosArrayGetSize(state->aDFileSet); - state->iFileSet = state->nFileSet; - - state->pBlockData = NULL; - - case SFSNEXTROW_FILESET: { - SDFileSet *pFileSet = NULL; - _next_fileset: - if (--state->iFileSet >= 0) { - pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); - } else { - // tBlockDataDestroy(&state->blockData, 1); - if (state->pBlockData) { - tBlockDataDestroy(state->pBlockData); - state->pBlockData = NULL; - } + if (SFSNEXTROW_FS == state->state) { + state->iFileSet = TARRAY2_SIZE(state->aDFileSet); - *ppRow = NULL; - return code; - } + state->state = SFSNEXTROW_FILESET; + } - if (*state->pDataFReader == NULL || (*state->pDataFReader)->pSet->fid != pFileSet->fid) { - if (*state->pDataFReader != NULL) { - tsdbDataFReaderClose(state->pDataFReader); + if (SFSNEXTROW_FILESET == state->state) { + _next_fileset: + if (--state->iFileSet < 0) { + clearLastFileSet(state); - // resetLastBlockLoadInfo(state->pLoadInfo); + *ppRow = NULL; + return code; + } else { + state->pFileSet = TARRAY2_GET(state->aDFileSet, state->iFileSet); + } + + STFileObj **pFileObj = state->pFileSet->farr; + if (pFileObj[0] != NULL || pFileObj[3] != NULL) { + if (state->pFileSet != state->pr->pCurFileSet) { + SDataFileReaderConfig conf = {.tsdb = pTsdb, .szPage = pTsdb->pVnode->config.tsdbPageSize}; + const char *filesName[4] = {0}; + if (pFileObj[0] != NULL) { + conf.files[0].file = *pFileObj[0]->f; + conf.files[0].exist = true; + filesName[0] = pFileObj[0]->fname; + + conf.files[1].file = *pFileObj[1]->f; + conf.files[1].exist = true; + filesName[1] = pFileObj[1]->fname; + + conf.files[2].file = *pFileObj[2]->f; + conf.files[2].exist = true; + filesName[2] = pFileObj[2]->fname; } - code = tsdbDataFReaderOpen(state->pDataFReader, state->pTsdb, pFileSet); - if (code) goto _err; + if (pFileObj[3] != NULL) { + conf.files[3].exist = true; + conf.files[3].file = *pFileObj[3]->f; + filesName[3] = pFileObj[3]->fname; + } + + code = tsdbDataFileReaderOpen(filesName, &conf, &state->pr->pFileReader); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + state->pr->pCurFileSet = state->pFileSet; + + loadDataTomb(state->pr, state->pr->pFileReader); + + int32_t code = tsdbDataFileReadBrinBlk(state->pr->pFileReader, &state->pr->pBlkArray); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } } - // tMapDataReset(&state->blockIdxMap); - /* - if (!state->aBlockIdx) { - state->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (!state->pIndexList) { + state->pIndexList = taosArrayInit(1, sizeof(SBrinBlk)); } else { - taosArrayClear(state->aBlockIdx); + taosArrayClear(state->pIndexList); } - code = tsdbReadBlockIdx(*state->pDataFReader, state->aBlockIdx); - if (code) goto _err; - */ - int32_t code = tsdbCacheGetBlockIdx(state->pTsdb->biCache, *state->pDataFReader, &state->aBlockIdxHandle); - if (code != TSDB_CODE_SUCCESS || state->aBlockIdxHandle == NULL) { - goto _err; + + const TBrinBlkArray *pBlkArray = state->pr->pBlkArray; + + for (int i = TARRAY2_SIZE(pBlkArray) - 1; i >= 0; --i) { + SBrinBlk *pBrinBlk = &pBlkArray->data[i]; + if (state->suid >= pBrinBlk->minTbid.suid && state->suid <= pBrinBlk->maxTbid.suid) { + if (state->uid >= pBrinBlk->minTbid.uid && state->uid <= pBrinBlk->maxTbid.uid) { + taosArrayPush(state->pIndexList, pBrinBlk); + } + } else if (state->suid > pBrinBlk->maxTbid.suid || + (state->suid == pBrinBlk->maxTbid.suid && state->uid > pBrinBlk->maxTbid.uid)) { + break; + } + } + + int indexSize = TARRAY_SIZE(state->pIndexList); + if (indexSize <= 0) { + goto _check_stt_data; } - state->aBlockIdx = (SArray *)taosLRUCacheValue(state->pTsdb->biCache, state->aBlockIdxHandle); - - /* if (state->pBlockIdx) { */ - /* } */ - /* code = tMapDataSearch(&state->blockIdxMap, state->pBlockIdxExp, tGetBlockIdx, tCmprBlockIdx, - * &state->blockIdx); - */ - state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ); - if (!state->pBlockIdx) { - tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); - - state->aBlockIdxHandle = NULL; - state->aBlockIdx = NULL; - /* - tsdbDataFReaderClose(state->pDataFReader); - *state->pDataFReader = NULL; - resetLastBlockLoadInfo(state->pLoadInfo);*/ + + state->state = SFSNEXTROW_INDEXLIST; + state->iBrinIndex = indexSize; + } + + _check_stt_data: + if (state->pFileSet != state->pr->pCurFileSet) { + state->pr->pCurFileSet = state->pFileSet; + } + + code = lastIterOpen(&state->lastIter, state->pFileSet, pTsdb, state->pTSchema, state->suid, state->uid, state->pr, + state->lastTs, aCols, nCols); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + code = lastIterNext(&state->lastIter, &state->pLastRow); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + if (!state->pLastRow) { + state->lastEmpty = 1; + + if (SFSNEXTROW_INDEXLIST != state->state) { + clearLastFileSet(state); goto _next_fileset; } + } else { + state->lastEmpty = 0; - tMapDataReset(&state->blockMap); - /* - if (state->blockMap.pData != NULL) { - tMapDataClear(&state->blockMap); + if (SFSNEXTROW_INDEXLIST != state->state) { + state->state = SFSNEXTROW_NEXTSTTROW; + + *ppRow = state->pLastRow; + state->pLastRow = NULL; + return code; } - */ - code = tsdbReadDataBlk(*state->pDataFReader, state->pBlockIdx, &state->blockMap); - if (code) goto _err; + } - state->nBlock = state->blockMap.nItem; - state->iBlock = state->nBlock - 1; + state->pLastIter = &state->lastIter; + } - if (!state->pBlockData) { - state->pBlockData = &state->blockData; + if (SFSNEXTROW_NEXTSTTROW == state->state) { + code = lastIterNext(&state->lastIter, &state->pLastRow); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } - code = tBlockDataCreate(&state->blockData); - if (code) goto _err; + if (!state->pLastRow) { + if (state->pLastIter) { + lastIterClose(&state->pLastIter); } + + clearLastFileSet(state); + state->state = SFSNEXTROW_FILESET; + goto _next_fileset; + } else { + *ppRow = state->pLastRow; + state->pLastRow = NULL; + return code; } - case SFSNEXTROW_BLOCKDATA: - _next_datablock: - if (state->iBlock >= 0) { - SDataBlk block = {0}; - bool skipBlock = true; - int inputColIndex = 0; + } - tDataBlkReset(&block); - tBlockDataReset(state->pBlockData); + if (SFSNEXTROW_INDEXLIST == state->state) { + SBrinBlk *pBrinBlk = NULL; + _next_brinindex: + if (--state->iBrinIndex < 0) { + if (state->pLastRow) { + state->state = SFSNEXTROW_NEXTSTTROW; + *ppRow = state->pLastRow; + state->pLastRow = NULL; + return code; + } - tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk); - if (block.maxKey.ts <= state->lastTs) { - *pIgnoreEarlierTs = true; - if (state->pBlockData) { - tBlockDataDestroy(state->pBlockData); - state->pBlockData = NULL; - } + clearLastFileSet(state); + goto _next_fileset; + } else { + pBrinBlk = taosArrayGet(state->pIndexList, state->iBrinIndex); + } - *ppRow = NULL; - return code; - } - *pIgnoreEarlierTs = false; - tBlockDataReset(state->pBlockData); - TABLEID tid = {.suid = state->suid, .uid = state->uid}; - int nTmpCols = nCols; - if (aCols[0] == PRIMARYKEY_TIMESTAMP_COL_ID && nCols == 1) { - nTmpCols = 0; - skipBlock = false; - } - code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, aCols, nTmpCols); - if (code) goto _err; + code = tsdbDataFileReadBrinBlock(state->pr->pFileReader, pBrinBlk, &state->brinBlock); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } - code = tsdbReadDataBlock(*state->pDataFReader, &block, state->pBlockData); - if (code) goto _err; + state->iBrinRecord = BRIN_BLOCK_SIZE(&state->brinBlock) - 1; + state->state = SFSNEXTROW_BRINBLOCK; + } - for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) { - SColData *pColData = &state->pBlockData->aColData[colIndex]; + if (SFSNEXTROW_BRINBLOCK == state->state) { + _next_brinrecord: + if (state->iBrinRecord < 0) { // empty brin block, goto _next_brinindex + tBrinBlockClear(&state->brinBlock); + goto _next_brinindex; + } + code = tBrinBlockGet(&state->brinBlock, state->iBrinRecord, &state->brinRecord); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } - if (isLast && (pColData->flag & HAS_VALUE)) { - skipBlock = false; - break; - } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { - skipBlock = false; - break; - } - } + SBrinRecord *pRecord = &state->brinRecord; + if (pRecord->uid != state->uid) { + // TODO: goto next brin block early + --state->iBrinRecord; + goto _next_brinrecord; + } - if (skipBlock) { - if (--state->iBlock < 0) { - tsdbDataFReaderClose(state->pDataFReader); - *state->pDataFReader = NULL; - // resetLastBlockLoadInfo(state->pLoadInfo); + state->state = SFSNEXTROW_BRINRECORD; + } - if (state->aBlockIdx) { - // taosArrayDestroy(state->aBlockIdx); - tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + if (SFSNEXTROW_BRINRECORD == state->state) { + SBrinRecord *pRecord = &state->brinRecord; - state->aBlockIdxHandle = NULL; - state->aBlockIdx = NULL; - } + if (!state->pBlockData) { + state->pBlockData = &state->blockData; + code = tBlockDataCreate(&state->blockData); + if (code) goto _err; + } else { + tBlockDataReset(state->pBlockData); + } - state->state = SFSNEXTROW_FILESET; - goto _next_fileset; - } else { - goto _next_datablock; - } - } + if (aCols[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { + --nCols; + ++aCols; + } + code = tsdbDataFileReadBlockDataByColumn(state->pr->pFileReader, pRecord, state->pBlockData, state->pTSchema, aCols, + nCols); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } - state->nRow = state->blockData.nRow; - state->iRow = state->nRow - 1; - - state->state = SFSNEXTROW_BLOCKROW; - checkRemainingRow = false; - } - case SFSNEXTROW_BLOCKROW: { - if (checkRemainingRow) { - bool skipBlock = true; - int inputColIndex = 0; - for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) { - SColData *pColData = &state->pBlockData->aColData[colIndex]; - int16_t cid = pColData->cid; - - if (inputColIndex < nCols && cid == aCols[inputColIndex]) { - if (isLast && (pColData->flag & HAS_VALUE)) { - skipBlock = false; - break; - } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { - skipBlock = false; - break; - } + state->nRow = state->blockData.nRow; + state->iRow = state->nRow - 1; - ++inputColIndex; - } - } + state->state = SFSNEXTROW_BLOCKROW; + } - if (skipBlock) { - if (--state->iBlock < 0) { - tsdbDataFReaderClose(state->pDataFReader); - *state->pDataFReader = NULL; - // resetLastBlockLoadInfo(state->pLoadInfo); + if (SFSNEXTROW_BLOCKROW == state->state) { + if (state->iRow < 0) { + --state->iBrinRecord; + goto _next_brinrecord; + } - if (state->aBlockIdx) { - // taosArrayDestroy(state->aBlockIdx); - tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + state->row = tsdbRowFromBlockData(state->pBlockData, state->iRow); + if (!state->pLastIter) { + *ppRow = &state->row; + --state->iRow; + return code; + } - state->aBlockIdxHandle = NULL; - state->aBlockIdx = NULL; - } + if (!state->pLastRow) { + // get next row from fslast and process with fs row, --state->Row if select fs row + code = lastIterNext(&state->lastIter, &state->pLastRow); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + } - state->state = SFSNEXTROW_FILESET; - goto _next_fileset; - } else { - goto _next_datablock; - } - } + if (!state->pLastRow) { + if (state->pLastIter) { + lastIterClose(&state->pLastIter); } - if (state->iRow >= 0) { - state->row = tsdbRowFromBlockData(state->pBlockData, state->iRow); - *ppRow = &state->row; + *ppRow = &state->row; + --state->iRow; + return code; + } - if (--state->iRow < 0) { - state->state = SFSNEXTROW_BLOCKDATA; - if (--state->iBlock < 0) { - tsdbDataFReaderClose(state->pDataFReader); - *state->pDataFReader = NULL; - // resetLastBlockLoadInfo(state->pLoadInfo); + // process state->pLastRow & state->row + TSKEY rowTs = TSDBROW_TS(&state->row); + TSKEY lastRowTs = TSDBROW_TS(state->pLastRow); + if (lastRowTs > rowTs) { + *ppRow = state->pLastRow; + state->pLastRow = NULL; + return code; + } else if (lastRowTs < rowTs) { + *ppRow = &state->row; + --state->iRow; + return code; + } else { + // TODO: merge rows and *ppRow = mergedRow + SRowMerger *pMerger = &state->rowMerger; + tsdbRowMergerInit(pMerger, state->pTSchema); - if (state->aBlockIdx) { - // taosArrayDestroy(state->aBlockIdx); - tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + code = tsdbRowMergerAdd(pMerger, &state->row, state->pTSchema); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + code = tsdbRowMergerAdd(pMerger, state->pLastRow, state->pTSchema); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } - state->aBlockIdxHandle = NULL; - state->aBlockIdx = NULL; - } + if (state->pTSRow) { + taosMemoryFree(state->pTSRow); + state->pTSRow = NULL; + } - state->state = SFSNEXTROW_FILESET; - } - } + code = tsdbRowMergerGetRow(pMerger, &state->pTSRow); + if (code != TSDB_CODE_SUCCESS) { + goto _err; } + state->row = tsdbRowFromTSRow(TSDBROW_VERSION(&state->row), state->pTSRow); + *ppRow = &state->row; + --state->iRow; + + tsdbRowMergerClear(pMerger); + return code; } - default: - ASSERT(0); - break; } _err: - /* - if (*state->pDataFReader) { - tsdbDataFReaderClose(state->pDataFReader); - *state->pDataFReader = NULL; - resetLastBlockLoadInfo(state->pLoadInfo); - }*/ - if (state->aBlockIdx) { - // taosArrayDestroy(state->aBlockIdx); - tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); - - state->aBlockIdxHandle = NULL; - state->aBlockIdx = NULL; - } - if (state->pBlockData) { - tBlockDataDestroy(state->pBlockData); - state->pBlockData = NULL; - } + clearLastFileSet(state); *ppRow = NULL; return code; } -int32_t clearNextRowFromFS(void *iter) { - int32_t code = 0; - - SFSNextRowIter *state = (SFSNextRowIter *)iter; - if (!state) { - return code; - } - /* - if (state->pDataFReader) { - tsdbDataFReaderClose(&state->pDataFReader); - state->pDataFReader = NULL; - }*/ - if (state->aBlockIdx) { - // taosArrayDestroy(state->aBlockIdx); - tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); - - state->aBlockIdxHandle = NULL; - state->aBlockIdx = NULL; - } - if (state->pBlockData) { - // tBlockDataDestroy(&state->blockData, 1); - tBlockDataDestroy(state->pBlockData); - state->pBlockData = NULL; - } - - if (state->blockMap.pData != NULL) { - tMapDataClear(&state->blockMap); - } - - return code; -} - typedef enum SMEMNEXTROWSTATES { SMEMNEXTROW_ENTER, SMEMNEXTROW_NEXT, @@ -1583,8 +2185,6 @@ typedef struct SMemNextRowIter { STbData *pMem; // [input] STbDataIter iter; // mem buffer skip list iterator int64_t lastTs; - // bool iterOpened; - // TSDBROW *curRow; } SMemNextRowIter; static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, @@ -1635,45 +2235,6 @@ _err: return code; } -/* static int32_t tsRowFromTsdbRow(STSchema *pTSchema, TSDBROW *pRow, STSRow **ppRow) { */ -/* int32_t code = 0; */ - -/* SColVal *pColVal = &(SColVal){0}; */ - -/* if (pRow->type == 0) { */ -/* *ppRow = tdRowDup(pRow->pTSRow); */ -/* } else { */ -/* SArray *pArray = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); */ -/* if (pArray == NULL) { */ -/* code = TSDB_CODE_OUT_OF_MEMORY; */ -/* goto _exit; */ -/* } */ - -/* TSDBKEY key = TSDBROW_KEY(pRow); */ -/* STColumn *pTColumn = &pTSchema->columns[0]; */ -/* *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts}); */ - -/* if (taosArrayPush(pArray, pColVal) == NULL) { */ -/* code = TSDB_CODE_OUT_OF_MEMORY; */ -/* goto _exit; */ -/* } */ - -/* for (int16_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { */ -/* tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); */ -/* if (taosArrayPush(pArray, pColVal) == NULL) { */ -/* code = TSDB_CODE_OUT_OF_MEMORY; */ -/* goto _exit; */ -/* } */ -/* } */ - -/* code = tdSTSRowNew(pArray, pTSchema, ppRow); */ -/* if (code) goto _exit; */ -/* } */ - -/* _exit: */ -/* return code; */ -/* } */ - static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { bool deleted = false; while (*iSkyline > 0) { @@ -1684,9 +2245,14 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { return false; } else if (key->ts >= pItemFront->ts && key->ts <= pItemBack->ts) { if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { + // if (key->version <= pItemFront->version || key->version <= pItemBack->version) { return true; } else { - return false; + if (*iSkyline > 1) { + --*iSkyline; + } else { + return false; + } } } else { if (*iSkyline > 1) { @@ -1714,103 +2280,121 @@ typedef struct { _next_row_clear_fn_t nextRowClearFn; } TsdbNextRowState; -typedef struct { - SArray *pSkyline; - int64_t iSkyline; - - SBlockIdx idx; - SMemNextRowIter memState; - SMemNextRowIter imemState; - SFSLastNextRowIter fsLastState; - SFSNextRowIter fsState; - TSDBROW memRow, imemRow, fsLastRow, fsRow; - - TsdbNextRowState input[4]; - STsdb *pTsdb; +typedef struct CacheNextRowIter { + SArray *pMemDelData; + SArray *pSkyline; + int64_t iSkyline; + SBlockIdx idx; + SMemNextRowIter memState; + SMemNextRowIter imemState; + SFSNextRowIter fsState; + TSDBROW memRow, imemRow, fsLastRow, fsRow; + TsdbNextRowState input[3]; + SCacheRowsReader *pr; + STsdb *pTsdb; } CacheNextRowIter; -static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, - SSttBlockLoadInfo *pLoadInfo, SLDataIter* pLDataIter, STsdbReadSnap *pReadSnap, SDataFReader **pDataFReader, - SDataFReader **pDataFReaderLast, int64_t lastTs) { - int code = 0; +int32_t clearNextRowFromFS(void *iter) { + int32_t code = 0; - STbData *pMem = NULL; - if (pReadSnap->pMem) { - pMem = tsdbGetTbDataFromMemTable(pReadSnap->pMem, suid, uid); + SFSNextRowIter *state = (SFSNextRowIter *)iter; + if (!state) { + return code; } - STbData *pIMem = NULL; - if (pReadSnap->pIMem) { - pIMem = tsdbGetTbDataFromMemTable(pReadSnap->pIMem, suid, uid); + if (state->pLastIter) { + lastIterClose(&state->pLastIter); } - pIter->pTsdb = pTsdb; + if (state->pBlockData) { + tBlockDataDestroy(state->pBlockData); + state->pBlockData = NULL; + } - pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); + if (state->pTSRow) { + taosMemoryFree(state->pTSRow); + state->pTSRow = NULL; + } - SDelFile *pDelFile = pReadSnap->fs.pDelFile; - if (pDelFile) { - SDelFReader *pDelFReader; + if (state->pRowIter->pSkyline) { + taosArrayDestroy(state->pRowIter->pSkyline); + state->pRowIter->pSkyline = NULL; + } - code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb); - if (code) goto _err; + return code; +} - SArray *pDelIdxArray = taosArrayInit(32, sizeof(SDelIdx)); +static void clearLastFileSet(SFSNextRowIter *state) { + if (state->pLastIter) { + lastIterClose(&state->pLastIter); + } - code = tsdbReadDelIdx(pDelFReader, pDelIdxArray); - if (code) { - taosArrayDestroy(pDelIdxArray); - tsdbDelFReaderClose(&pDelFReader); - goto _err; - } + if (state->pBlockData) { + tBlockDataDestroy(state->pBlockData); + state->pBlockData = NULL; + } - SDelIdx *delIdx = taosArraySearch(pDelIdxArray, &(SDelIdx){.suid = suid, .uid = uid}, tCmprDelIdx, TD_EQ); + if (state->pr->pFileReader) { + tsdbDataFileReaderClose(&state->pr->pFileReader); + state->pr->pFileReader = NULL; - code = getTableDelSkyline(pMem, pIMem, pDelFReader, delIdx, pIter->pSkyline); - if (code) { - taosArrayDestroy(pDelIdxArray); - tsdbDelFReaderClose(&pDelFReader); - goto _err; - } + state->pr->pCurFileSet = NULL; + } - taosArrayDestroy(pDelIdxArray); - tsdbDelFReaderClose(&pDelFReader); - } else { - code = getTableDelSkyline(pMem, pIMem, NULL, NULL, pIter->pSkyline); - if (code) goto _err; + if (state->pTSRow) { + taosMemoryFree(state->pTSRow); + state->pTSRow = NULL; } - pIter->iSkyline = taosArrayGetSize(pIter->pSkyline) - 1; + if (state->pRowIter->pSkyline) { + taosArrayDestroy(state->pRowIter->pSkyline); + state->pRowIter->pSkyline = NULL; - pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; + void *pe = NULL; + int32_t iter = 0; + while ((pe = tSimpleHashIterate(state->pr->pTableMap, pe, &iter)) != NULL) { + STableLoadInfo *pInfo = *(STableLoadInfo **)pe; + pInfo->pTombData = taosArrayDestroy(pInfo->pTombData); + } + } +} - pIter->fsLastState.state = (SFSLASTNEXTROWSTATES)SFSNEXTROW_FS; - pIter->fsLastState.pTsdb = pTsdb; - pIter->fsLastState.aDFileSet = pReadSnap->fs.aDFileSet; - pIter->fsLastState.pTSchema = pTSchema; - pIter->fsLastState.suid = suid; - pIter->fsLastState.uid = uid; - pIter->fsLastState.pLoadInfo = pLoadInfo; - pIter->fsLastState.pDataFReader = pDataFReaderLast; - pIter->fsLastState.lastTs = lastTs; - pIter->fsLastState.pDataIter = pLDataIter; +static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, + SArray *pLDataIterArray, STsdbReadSnap *pReadSnap, int64_t lastTs, + SCacheRowsReader *pr) { + int code = 0; - pIter->fsState.state = SFSNEXTROW_FS; - pIter->fsState.pTsdb = pTsdb; - pIter->fsState.aDFileSet = pReadSnap->fs.aDFileSet; - pIter->fsState.pBlockIdxExp = &pIter->idx; - pIter->fsState.pTSchema = pTSchema; - pIter->fsState.suid = suid; + STbData *pMem = NULL; + if (pReadSnap->pMem) { + pMem = tsdbGetTbDataFromMemTable(pReadSnap->pMem, suid, uid); + } + + STbData *pIMem = NULL; + if (pReadSnap->pIMem) { + pIMem = tsdbGetTbDataFromMemTable(pReadSnap->pIMem, suid, uid); + } + + pIter->pTsdb = pTsdb; + + pIter->pMemDelData = NULL; + + loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer); + + pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; + + pIter->fsState.pRowIter = pIter; + pIter->fsState.state = SFSNEXTROW_FS; + pIter->fsState.aDFileSet = pReadSnap->pfSetArray; + pIter->fsState.pBlockIdxExp = &pIter->idx; + pIter->fsState.pTSchema = pTSchema; + pIter->fsState.suid = suid; pIter->fsState.uid = uid; - pIter->fsState.pLoadInfo = pLoadInfo; - pIter->fsState.pDataFReader = pDataFReader; pIter->fsState.lastTs = lastTs; + pIter->fsState.pr = pr; pIter->input[0] = (TsdbNextRowState){&pIter->memRow, true, false, false, &pIter->memState, getNextRowFromMem, NULL}; pIter->input[1] = (TsdbNextRowState){&pIter->imemRow, true, false, false, &pIter->imemState, getNextRowFromMem, NULL}; - pIter->input[2] = (TsdbNextRowState){ - &pIter->fsLastRow, false, true, false, &pIter->fsLastState, getNextRowFromFSLast, clearNextRowFromFSLast}; - pIter->input[3] = + pIter->input[2] = (TsdbNextRowState){&pIter->fsRow, false, true, false, &pIter->fsState, getNextRowFromFS, clearNextRowFromFS}; if (pMem) { @@ -1829,7 +2413,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->input[1].next = true; } - return code; + pIter->pr = pr; _err: return code; } @@ -1837,7 +2421,7 @@ _err: static int32_t nextRowIterClose(CacheNextRowIter *pIter) { int code = 0; - for (int i = 0; i < 4; ++i) { + for (int i = 0; i < 3; ++i) { if (pIter->input[i].nextRowClearFn) { pIter->input[i].nextRowClearFn(pIter->input[i].iter); } @@ -1847,6 +2431,10 @@ static int32_t nextRowIterClose(CacheNextRowIter *pIter) { taosArrayDestroy(pIter->pSkyline); } + if (pIter->pMemDelData) { + taosArrayDestroy(pIter->pMemDelData); + } + _err: return code; } @@ -1856,7 +2444,7 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI int16_t *aCols, int nCols) { int code = 0; for (;;) { - for (int i = 0; i < 4; ++i) { + for (int i = 0; i < 3; ++i) { if (pIter->input[i].next && !pIter->input[i].stop) { code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, &pIter->input[i].ignoreEarlierTs, isLast, aCols, nCols); @@ -1869,10 +2457,10 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI } } - if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop && pIter->input[3].stop) { + if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop) { *ppRow = NULL; - *pIgnoreEarlierTs = (pIter->input[0].ignoreEarlierTs || pIter->input[1].ignoreEarlierTs || - pIter->input[2].ignoreEarlierTs || pIter->input[3].ignoreEarlierTs); + *pIgnoreEarlierTs = + (pIter->input[0].ignoreEarlierTs || pIter->input[1].ignoreEarlierTs || pIter->input[2].ignoreEarlierTs); return code; } @@ -1882,7 +2470,7 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI int nMax = 0; TSKEY maxKey = TSKEY_MIN; - for (int i = 0; i < 4; ++i) { + for (int i = 0; i < 3; ++i) { if (!pIter->input[i].stop && pIter->input[i].pRow != NULL) { TSDBKEY key = TSDBROW_KEY(pIter->input[i].pRow); @@ -1899,403 +2487,106 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI pIter->input[i].next = false; } } - } - - // delete detection - TSDBROW *merge[4] = {0}; - int iMerge[4] = {-1, -1, -1, -1}; - int nMerge = 0; - for (int i = 0; i < nMax; ++i) { - TSDBKEY maxKey1 = TSDBROW_KEY(max[i]); - - bool deleted = tsdbKeyDeleted(&maxKey1, pIter->pSkyline, &pIter->iSkyline); - if (!deleted) { - iMerge[nMerge] = iMax[i]; - merge[nMerge++] = max[i]; - } - - pIter->input[iMax[i]].next = deleted; - } - - if (nMerge > 0) { - pIter->input[iMerge[0]].next = true; - - *ppRow = merge[0]; - return code; - } - } - -_err: - return code; -} - -static int32_t initLastColArray(STSchema *pTSchema, SArray **ppColArray) { - SArray *pColArray = taosArrayInit(pTSchema->numOfCols, sizeof(SLastCol)); - if (NULL == pColArray) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pTSchema->numOfCols; ++i) { - SLastCol col = {.ts = 0, .colVal = COL_VAL_NULL(pTSchema->columns[i].colId, pTSchema->columns[i].type)}; - taosArrayPush(pColArray, &col); - } - *ppColArray = pColArray; - return TSDB_CODE_SUCCESS; -} - -static int32_t initLastColArrayPartial(STSchema *pTSchema, SArray **ppColArray, int16_t *slotIds, int nCols) { - SArray *pColArray = taosArrayInit(nCols, sizeof(SLastCol)); - if (NULL == pColArray) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < nCols; ++i) { - int16_t slotId = slotIds[i]; - SLastCol col = {.ts = 0, .colVal = COL_VAL_NULL(pTSchema->columns[slotId].colId, pTSchema->columns[slotId].type)}; - taosArrayPush(pColArray, &col); - } - *ppColArray = pColArray; - return TSDB_CODE_SUCCESS; -} - -static int32_t cloneTSchema(STSchema *pSrc, STSchema **ppDst) { - int32_t len = sizeof(STSchema) + sizeof(STColumn) * pSrc->numOfCols; - *ppDst = taosMemoryMalloc(len); - if (NULL == *ppDst) { - return TSDB_CODE_OUT_OF_MEMORY; - } - memcpy(*ppDst, pSrc, len); - return TSDB_CODE_SUCCESS; -} - -static int32_t updateTSchema(int32_t sversion, SCacheRowsReader *pReader, uint64_t uid) { - if (NULL == pReader->pCurrSchema && sversion == pReader->pSchema->version) { - return cloneTSchema(pReader->pSchema, &pReader->pCurrSchema); - } - - if (NULL != pReader->pCurrSchema && sversion == pReader->pCurrSchema->version) { - return TSDB_CODE_SUCCESS; - } - - taosMemoryFreeClear(pReader->pCurrSchema); - return metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pCurrSchema); -} - -static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppColArray, SCacheRowsReader *pr) { - STSchema *pTSchema = pr->pSchema; // metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - int16_t nLastCol = pTSchema->numOfCols; - int16_t iCol = 0; - int16_t noneCol = 0; - bool setNoneCol = false; - bool hasRow = false; - bool ignoreEarlierTs = false; - SArray *pColArray = NULL; - SColVal *pColVal = &(SColVal){0}; - - int32_t code = initLastColArray(pTSchema, &pColArray); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - - TSKEY lastRowTs = TSKEY_MAX; - - CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader, - &pr->pDataFReaderLast, pr->lastTs); - - do { - TSDBROW *pRow = NULL; - nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, NULL, 0); - - if (!pRow) { - break; - } - - hasRow = true; - - int32_t sversion = TSDBROW_SVERSION(pRow); - if (sversion != -1) { - code = updateTSchema(sversion, pr, uid); - if (TSDB_CODE_SUCCESS != code) { - goto _err; - } - pTSchema = pr->pCurrSchema; - } - int16_t nCol = pTSchema->numOfCols; - - TSKEY rowTs = TSDBROW_TS(pRow); - - if (lastRowTs == TSKEY_MAX) { - lastRowTs = rowTs; - STColumn *pTColumn = &pTSchema->columns[0]; - - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs}); - taosArraySet(pColArray, 0, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}); - - for (iCol = 1; iCol < nCol; ++iCol) { - if (iCol >= nLastCol) { - break; - } - SLastCol *pCol = taosArrayGet(pColArray, iCol); - if (pCol->colVal.cid != pTSchema->columns[iCol].colId) { - continue; - } - tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); - - *pCol = (SLastCol){.ts = lastRowTs, .colVal = *pColVal}; - if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) { - pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData); - if (pCol->colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); - } - - if (COL_VAL_IS_NONE(pColVal) && !setNoneCol) { - noneCol = iCol; - setNoneCol = true; - } - } - if (!setNoneCol) { - // done, goto return pColArray - break; - } else { - continue; - } - } - - if ((rowTs < lastRowTs)) { - // done, goto return pColArray - break; - } - - // merge into pColArray - setNoneCol = false; - for (iCol = noneCol; iCol < nCol; ++iCol) { - // high version's column value - SColVal *tColVal = (SColVal *)taosArrayGet(pColArray, iCol); - - tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); - if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { - SLastCol lastCol = {.ts = rowTs, .colVal = *pColVal}; - if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) { - SLastCol *pLastCol = (SLastCol *)taosArrayGet(pColArray, iCol); - taosMemoryFree(pLastCol->colVal.value.pData); - - lastCol.colVal.value.pData = taosMemoryMalloc(lastCol.colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - memcpy(lastCol.colVal.value.pData, pColVal->value.pData, pColVal->value.nData); - } - - taosArraySet(pColArray, iCol, &lastCol); - } else if (COL_VAL_IS_NONE(tColVal) && COL_VAL_IS_NONE(pColVal) && !setNoneCol) { - noneCol = iCol; - setNoneCol = true; - } - } - } while (setNoneCol); - - // build the result ts row here - *dup = false; - // if (taosArrayGetSize(pColArray) != nCol) { - //*ppColArray = NULL; - // taosArrayDestroy(pColArray); - //} else { - if (!hasRow) { - if (ignoreEarlierTs) { - taosArrayDestroy(pColArray); - pColArray = NULL; - } else { - taosArrayClear(pColArray); - } - } - *ppColArray = pColArray; - //} - - nextRowIterClose(&iter); - // taosMemoryFreeClear(pTSchema); - return code; - -_err: - nextRowIterClose(&iter); - taosArrayDestroy(pColArray); - // taosMemoryFreeClear(pTSchema); - return code; -} - -static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr) { - STSchema *pTSchema = pr->pSchema; // metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - int16_t nLastCol = pTSchema->numOfCols; - int16_t noneCol = 0; - bool setNoneCol = false; - bool hasRow = false; - bool ignoreEarlierTs = false; - SArray *pColArray = NULL; - SColVal *pColVal = &(SColVal){0}; - int16_t nCols = nLastCol; - - int32_t code = initLastColArray(pTSchema, &pColArray); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - SArray *aColArray = taosArrayInit(nCols, sizeof(int16_t)); - if (NULL == aColArray) { - taosArrayDestroy(pColArray); - - return TSDB_CODE_OUT_OF_MEMORY; - } - for (int i = 1; i < pTSchema->numOfCols; ++i) { - taosArrayPush(aColArray, &pTSchema->columns[i].colId); - } - - TSKEY lastRowTs = TSKEY_MAX; - - CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader, - &pr->pDataFReaderLast, pr->lastTs); - - do { - TSDBROW *pRow = NULL; - nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); - - if (!pRow) { - break; - } - - hasRow = true; - - int32_t sversion = TSDBROW_SVERSION(pRow); - if (sversion != -1) { - code = updateTSchema(sversion, pr, uid); - if (TSDB_CODE_SUCCESS != code) { - goto _err; - } - pTSchema = pr->pCurrSchema; - } - int16_t nCol = pTSchema->numOfCols; - - TSKEY rowTs = TSDBROW_TS(pRow); - - if (lastRowTs == TSKEY_MAX) { - lastRowTs = rowTs; - STColumn *pTColumn = &pTSchema->columns[0]; - - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs}); - taosArraySet(pColArray, 0, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}); - - for (int16_t iCol = 1; iCol < nCol; ++iCol) { - if (iCol >= nLastCol) { - break; - } - SLastCol *pCol = taosArrayGet(pColArray, iCol); - if (pCol->colVal.cid != pTSchema->columns[iCol].colId) { - continue; - } - tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); - - *pCol = (SLastCol){.ts = lastRowTs, .colVal = *pColVal}; - if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) { - pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData); - if (pCol->colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); - } - - if (!COL_VAL_IS_VALUE(pColVal)) { - if (!setNoneCol) { - noneCol = iCol; - setNoneCol = true; - } - } else { - int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ); - taosArrayRemove(aColArray, aColIndex); - } - } - if (!setNoneCol) { - // done, goto return pColArray - break; - } else { - continue; - } - } - - // merge into pColArray - setNoneCol = false; - for (int16_t iCol = noneCol; iCol < nCol; ++iCol) { - if (iCol >= nLastCol) { - break; - } - // high version's column value - SLastCol *lastColVal = (SLastCol *)taosArrayGet(pColArray, iCol); - if (lastColVal->colVal.cid != pTSchema->columns[iCol].colId) { - continue; - } - SColVal *tColVal = &lastColVal->colVal; + } - tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); - if (!COL_VAL_IS_VALUE(tColVal) && COL_VAL_IS_VALUE(pColVal)) { - SLastCol lastCol = {.ts = rowTs, .colVal = *pColVal}; - if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) { - SLastCol *pLastCol = (SLastCol *)taosArrayGet(pColArray, iCol); - taosMemoryFree(pLastCol->colVal.value.pData); + // delete detection + TSDBROW *merge[4] = {0}; + int iMerge[4] = {-1, -1, -1, -1}; + int nMerge = 0; + for (int i = 0; i < nMax; ++i) { + TSDBKEY maxKey1 = TSDBROW_KEY(max[i]); - lastCol.colVal.value.pData = taosMemoryMalloc(lastCol.colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - memcpy(lastCol.colVal.value.pData, pColVal->value.pData, pColVal->value.nData); + if (!pIter->pSkyline) { + pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); + + uint64_t uid = pIter->idx.uid; + STableLoadInfo *pInfo = getTableLoadInfo(pIter->pr, uid); + if (pInfo->pTombData == NULL) { + pInfo->pTombData = taosArrayInit(4, sizeof(SDelData)); } - taosArraySet(pColArray, iCol, &lastCol); - int32_t aColIndex = taosArraySearchIdx(aColArray, &lastCol.colVal.cid, compareInt16Val, TD_EQ); - taosArrayRemove(aColArray, aColIndex); - } else if (!COL_VAL_IS_VALUE(tColVal) && !COL_VAL_IS_VALUE(pColVal) && !setNoneCol) { - noneCol = iCol; - setNoneCol = true; + taosArrayAddAll(pInfo->pTombData, pIter->pMemDelData); + + size_t delSize = TARRAY_SIZE(pInfo->pTombData); + if (delSize > 0) { + code = tsdbBuildDeleteSkyline(pInfo->pTombData, 0, (int32_t)(delSize - 1), pIter->pSkyline); + } + pIter->iSkyline = taosArrayGetSize(pIter->pSkyline) - 1; + } + + bool deleted = tsdbKeyDeleted(&maxKey1, pIter->pSkyline, &pIter->iSkyline); + if (!deleted) { + iMerge[nMerge] = iMax[i]; + merge[nMerge++] = max[i]; } + + pIter->input[iMax[i]].next = deleted; } - } while (setNoneCol); - // if (taosArrayGetSize(pColArray) <= 0) { - //*ppLastArray = NULL; - // taosArrayDestroy(pColArray); - //} else { - if (!hasRow) { - if (ignoreEarlierTs) { - taosArrayDestroy(pColArray); - pColArray = NULL; - } else { - taosArrayClear(pColArray); + if (nMerge > 0) { + pIter->input[iMerge[0]].next = true; + + *ppRow = merge[0]; + return code; } } - *ppLastArray = pColArray; - //} - - nextRowIterClose(&iter); - taosArrayDestroy(aColArray); - // taosMemoryFreeClear(pTSchema); - return code; _err: - nextRowIterClose(&iter); - // taosMemoryFreeClear(pTSchema); - *ppLastArray = NULL; - taosArrayDestroy(pColArray); - taosArrayDestroy(aColArray); return code; } +static int32_t initLastColArray(STSchema *pTSchema, SArray **ppColArray) { + SArray *pColArray = taosArrayInit(pTSchema->numOfCols, sizeof(SLastCol)); + if (NULL == pColArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pTSchema->numOfCols; ++i) { + SLastCol col = {.ts = 0, .colVal = COL_VAL_NULL(pTSchema->columns[i].colId, pTSchema->columns[i].type)}; + taosArrayPush(pColArray, &col); + } + *ppColArray = pColArray; + return TSDB_CODE_SUCCESS; +} + +static int32_t initLastColArrayPartial(STSchema *pTSchema, SArray **ppColArray, int16_t *slotIds, int nCols) { + SArray *pColArray = taosArrayInit(nCols, sizeof(SLastCol)); + if (NULL == pColArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < nCols; ++i) { + int16_t slotId = slotIds[i]; + SLastCol col = {.ts = 0, .colVal = COL_VAL_NULL(pTSchema->columns[slotId].colId, pTSchema->columns[slotId].type)}; + taosArrayPush(pColArray, &col); + } + *ppColArray = pColArray; + return TSDB_CODE_SUCCESS; +} + +static int32_t cloneTSchema(STSchema *pSrc, STSchema **ppDst) { + int32_t len = sizeof(STSchema) + sizeof(STColumn) * pSrc->numOfCols; + *ppDst = taosMemoryMalloc(len); + if (NULL == *ppDst) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(*ppDst, pSrc, len); + return TSDB_CODE_SUCCESS; +} + +static int32_t updateTSchema(int32_t sversion, SCacheRowsReader *pReader, uint64_t uid) { + if (NULL == pReader->pCurrSchema && sversion == pReader->pSchema->version) { + return cloneTSchema(pReader->pSchema, &pReader->pCurrSchema); + } + + if (NULL != pReader->pCurrSchema && sversion == pReader->pCurrSchema->version) { + return TSDB_CODE_SUCCESS; + } + + taosMemoryFreeClear(pReader->pCurrSchema); + return metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, sversion, &pReader->pCurrSchema); +} + static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols, int nCols, int16_t *slotIds) { STSchema *pTSchema = pr->pSchema; // metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); @@ -2325,8 +2616,7 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC TSKEY lastRowTs = TSKEY_MAX; CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader, - &pr->pDataFReaderLast, pr->lastTs); + nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->info.suid, pr->pLDataIterArray, pr->pReadSnap, pr->lastTs, pr); do { TSDBROW *pRow = NULL; @@ -2371,14 +2661,16 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal); *pCol = (SLastCol){.ts = rowTs, .colVal = *pColVal}; - if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) { + if (IS_VAR_DATA_TYPE(pColVal->type) /*&& pColVal->value.nData > 0*/) { pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData); if (pCol->colVal.value.pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); + if (pColVal->value.nData > 0) { + memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); + } } if (!COL_VAL_IS_VALUE(pColVal)) { @@ -2493,12 +2785,11 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, TSKEY lastRowTs = TSKEY_MAX; CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader, - &pr->pDataFReaderLast, pr->lastTs); + nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->info.suid, pr->pLDataIterArray, pr->pReadSnap, pr->lastTs, pr); do { TSDBROW *pRow = NULL; - nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); + nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); if (!pRow) { break; @@ -2518,95 +2809,46 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, TSKEY rowTs = TSDBROW_TS(pRow); - if (lastRowTs == TSKEY_MAX) { - lastRowTs = rowTs; - - for (int16_t iCol = noneCol; iCol < nCols; ++iCol) { - if (iCol >= nLastCol) { - break; - } - SLastCol *pCol = taosArrayGet(pColArray, iCol); - if (pCol->colVal.cid != pTSchema->columns[slotIds[iCol]].colId) { - continue; - } - if (slotIds[iCol] == 0) { - STColumn *pTColumn = &pTSchema->columns[0]; - - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = rowTs}); - taosArraySet(pColArray, 0, &(SLastCol){.ts = rowTs, .colVal = *pColVal}); - continue; - } - tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal); - - *pCol = (SLastCol){.ts = rowTs, .colVal = *pColVal}; - if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) { - pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData); - if (pCol->colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); - } - - if (COL_VAL_IS_NONE(pColVal)) { - if (!setNoneCol) { - noneCol = iCol; - setNoneCol = true; - } - } else { - int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ); - if (aColIndex >= 0) { - taosArrayRemove(aColArray, aColIndex); - } - } - } - if (!setNoneCol) { - // done, goto return pColArray - break; - } else { - continue; - } - } + lastRowTs = rowTs; - // merge into pColArray - setNoneCol = false; for (int16_t iCol = noneCol; iCol < nCols; ++iCol) { if (iCol >= nLastCol) { break; } - // high version's column value - SLastCol *lastColVal = (SLastCol *)taosArrayGet(pColArray, iCol); - if (lastColVal->colVal.cid != pTSchema->columns[slotIds[iCol]].colId) { + SLastCol *pCol = taosArrayGet(pColArray, iCol); + if (pCol->colVal.cid != pTSchema->columns[slotIds[iCol]].colId) { continue; } - SColVal *tColVal = &lastColVal->colVal; + if (slotIds[iCol] == 0) { + STColumn *pTColumn = &pTSchema->columns[0]; + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = rowTs}); + taosArraySet(pColArray, 0, &(SLastCol){.ts = rowTs, .colVal = *pColVal}); + continue; + } tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal); - if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { - SLastCol lastCol = {.ts = rowTs, .colVal = *pColVal}; - if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) { - SLastCol *pLastCol = (SLastCol *)taosArrayGet(pColArray, iCol); - taosMemoryFree(pLastCol->colVal.value.pData); - lastCol.colVal.value.pData = taosMemoryMalloc(lastCol.colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - memcpy(lastCol.colVal.value.pData, pColVal->value.pData, pColVal->value.nData); + *pCol = (SLastCol){.ts = rowTs, .colVal = *pColVal}; + if (IS_VAR_DATA_TYPE(pColVal->type) /*&& pColVal->value.nData > 0*/) { + pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData); + if (pCol->colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } + if (pColVal->value.nData > 0) { + memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); + } + } - taosArraySet(pColArray, iCol, &lastCol); - int32_t aColIndex = taosArraySearchIdx(aColArray, &lastCol.colVal.cid, compareInt16Val, TD_EQ); + int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ); + if (aColIndex >= 0) { taosArrayRemove(aColArray, aColIndex); - } else if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal) && !setNoneCol) { - noneCol = iCol; - setNoneCol = true; } } - } while (setNoneCol); + + break; + } while (1); if (!hasRow) { if (ignoreEarlierTs) { @@ -2632,91 +2874,6 @@ _err: return code; } -int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **handle) { - int32_t code = 0; - char key[32] = {0}; - int keyLen = 0; - - // getTableCacheKeyS(uid, "lr", key, &keyLen); - getTableCacheKey(uid, 0, key, &keyLen); - LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (!h) { - STsdb *pTsdb = pr->pVnode->pTsdb; - taosThreadMutexLock(&pTsdb->lruMutex); - - h = taosLRUCacheLookup(pCache, key, keyLen); - if (!h) { - SArray *pArray = NULL; - bool dup = false; // which is always false for now - code = mergeLastRow(uid, pTsdb, &dup, &pArray, pr); - // if table's empty or error or ignore ignore earlier ts, set handle NULL and return - if (code < 0 || pArray == NULL) { - if (!dup && pArray) { - taosArrayDestroy(pArray); - } - - taosThreadMutexUnlock(&pTsdb->lruMutex); - - *handle = NULL; - - return 0; - } - - size_t charge = pArray->capacity * pArray->elemSize + sizeof(*pArray); - _taos_lru_deleter_t deleter = deleteTableCacheLast; - LRUStatus status = taosLRUCacheInsert(pCache, key, keyLen, pArray, charge, deleter, &h, TAOS_LRU_PRIORITY_LOW); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; - } - } - taosThreadMutexUnlock(&pTsdb->lruMutex); - } - - *handle = h; - - return code; -} - -int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **handle) { - int32_t code = 0; - char key[32] = {0}; - int keyLen = 0; - - // getTableCacheKeyS(uid, "l", key, &keyLen); - getTableCacheKey(uid, 1, key, &keyLen); - LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (!h) { - STsdb *pTsdb = pr->pVnode->pTsdb; - taosThreadMutexLock(&pTsdb->lruMutex); - - h = taosLRUCacheLookup(pCache, key, keyLen); - if (!h) { - SArray *pLastArray = NULL; - code = mergeLast(uid, pTsdb, &pLastArray, pr); - // if table's empty or error or ignore ignore earlier ts, set handle NULL and return - if (code < 0 || pLastArray == NULL) { - taosThreadMutexUnlock(&pTsdb->lruMutex); - - *handle = NULL; - return 0; - } - - size_t charge = pLastArray->capacity * pLastArray->elemSize + sizeof(*pLastArray); - _taos_lru_deleter_t deleter = deleteTableCacheLast; - LRUStatus status = - taosLRUCacheInsert(pCache, key, keyLen, pLastArray, charge, deleter, &h, TAOS_LRU_PRIORITY_LOW); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; - } - } - taosThreadMutexUnlock(&pTsdb->lruMutex); - } - - *handle = h; - - return code; -} - int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h) { int32_t code = 0; @@ -2777,7 +2934,8 @@ static int32_t tsdbCacheLoadBlockIdx(SDataFReader *pFileReader, SArray **aBlockI return code; } -static void deleteBICache(const void *key, size_t keyLen, void *value) { +static void deleteBICache(const void *key, size_t keyLen, void *value, void *ud) { + (void)ud; SArray *pArray = (SArray *)value; taosArrayDestroy(pArray); @@ -2808,7 +2966,8 @@ int32_t tsdbCacheGetBlockIdx(SLRUCache *pCache, SDataFReader *pFileReader, LRUHa size_t charge = pArray->capacity * pArray->elemSize + sizeof(*pArray); _taos_lru_deleter_t deleter = deleteBICache; - LRUStatus status = taosLRUCacheInsert(pCache, key, keyLen, pArray, charge, deleter, &h, TAOS_LRU_PRIORITY_LOW); + LRUStatus status = + taosLRUCacheInsert(pCache, key, keyLen, pArray, charge, deleter, &h, TAOS_LRU_PRIORITY_LOW, NULL); if (status != TAOS_LRU_STATUS_OK) { code = -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 53103e9fbbd46722dc461a1599145883d1733164..66c8cc06e24adb0bba36ec1e1af7eaa60b3f4de0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -17,6 +17,8 @@ #include "tarray.h" #include "tcommon.h" #include "tsdb.h" +#include "tsdbDataFileRW.h" +#include "tsdbReadUtil.h" #define HASTYPE(_type, _t) (((_type) & (_t)) == (_t)) @@ -63,9 +65,6 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); SColVal* pVal = &pColVal->colVal; - if (COL_VAL_IS_NONE(&pColVal->colVal)) { - continue; - } allNullRow = false; if (IS_VAR_DATA_TYPE(pColVal->colVal.type)) { if (!COL_VAL_IS_VALUE(&pColVal->colVal)) { @@ -120,6 +119,21 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id return TSDB_CODE_SUCCESS; } +int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOfTables) { + SCacheRowsReader* pReader = (SCacheRowsReader*)reader; + + pReader->pTableList = pTableIdList; + pReader->numOfTables = numOfTables; + pReader->lastTs = INT64_MIN; + + int64_t blocks; + double elapse; + pReader->pLDataIterArray = destroySttBlockReader(pReader->pLDataIterArray, &blocks, &elapse); + pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + + return TSDB_CODE_SUCCESS; +} + int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t numOfCols, SArray* pCidList, int32_t* pSlotIds, uint64_t suid, void** pReader, const char* idstr) { *pReader = NULL; @@ -131,11 +145,11 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, p->type = type; p->pVnode = pVnode; p->pTsdb = p->pVnode->pTsdb; - p->verRange = (SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}; + p->info.verRange = (SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}; + p->info.suid = suid; p->numOfCols = numOfCols; p->pCidList = pCidList; p->pSlotIds = pSlotIds; - p->suid = suid; if (numOfTables == 0) { *pReader = p; @@ -167,21 +181,6 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, } } - SVnodeCfg* pCfg = &((SVnode*)pVnode)->config; - - int32_t numOfStt = pCfg->sttTrigger; - p->pLoadInfo = tCreateLastBlockLoadInfo(p->pSchema, NULL, 0, numOfStt); - if (p->pLoadInfo == NULL) { - tsdbCacherowsReaderClose(p); - return TSDB_CODE_OUT_OF_MEMORY; - } - - p->pDataIter = taosMemoryCalloc(pCfg->sttTrigger, sizeof(SLDataIter)); - if (p->pDataIter == NULL) { - tsdbCacherowsReaderClose(p); - return TSDB_CODE_OUT_OF_MEMORY; - } - p->idstr = taosStrdup(idstr); taosThreadMutexInit(&p->readerMutex, NULL); @@ -193,6 +192,9 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, void* tsdbCacherowsReaderClose(void* pReader) { SCacheRowsReader* p = pReader; + if (p == NULL) { + return NULL; + } if (p->pSchema != NULL) { for (int32_t i = 0; i < p->pSchema->numOfCols; ++i) { @@ -203,14 +205,36 @@ void* tsdbCacherowsReaderClose(void* pReader) { taosMemoryFree(p->pSchema); } - taosMemoryFreeClear(p->pDataIter); taosMemoryFree(p->pCurrSchema); - destroyLastBlockLoadInfo(p->pLoadInfo); + if (p->pLDataIterArray) { + int64_t loadBlocks = 0; + double elapse = 0; + destroySttBlockReader(p->pLDataIterArray, &loadBlocks, &elapse); + } + + if (p->pFileReader) { + tsdbDataFileReaderClose(&p->pFileReader); + p->pFileReader = NULL; + } taosMemoryFree((void*)p->idstr); taosThreadMutexDestroy(&p->readerMutex); + if (p->pTableMap) { + void* pe = NULL; + int32_t iter = 0; + while ((pe = tSimpleHashIterate(p->pTableMap, pe, &iter)) != NULL) { + STableLoadInfo* pInfo = *(STableLoadInfo**)pe; + pInfo->pTombData = taosArrayDestroy(pInfo->pTombData); + } + + tSimpleHashCleanup(p->pTableMap); + } + if (p->uidList) { + taosMemoryFree(p->uidList); + } + taosMemoryFree(pReader); return NULL; } @@ -253,7 +277,6 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 int32_t code = TSDB_CODE_SUCCESS; SArray* pRow = taosArrayInit(TARRAY_SIZE(pr->pCidList), sizeof(SLastCol)); bool hasRes = false; - SArray* pLastCols = NULL; void** pRes = taosMemoryCalloc(pr->numOfCols, POINTER_BYTES); if (pRes == NULL) { @@ -262,54 +285,46 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } for (int32_t j = 0; j < pr->numOfCols; ++j) { - pRes[j] = - taosMemoryCalloc(1, sizeof(SFirstLastRes) + pr->pSchema->columns[/*-1 == slotIds[j] ? 0 : */ slotIds[j]].bytes + - VARSTR_HEADER_SIZE); + pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + pr->pSchema->columns[slotIds[j]].bytes + VARSTR_HEADER_SIZE); SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[j]); p->ts = INT64_MIN; } - pLastCols = taosArrayInit(pr->numOfCols, sizeof(SLastCol)); - if (pLastCols == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - - for (int32_t i = 0; i < pr->numOfCols; ++i) { - int32_t slotId = slotIds[i]; - struct STColumn* pCol = &pr->pSchema->columns[slotId]; - SLastCol p = {.ts = INT64_MIN, .colVal.type = pCol->type, .colVal.flag = CV_FLAG_NULL}; - - if (IS_VAR_DATA_TYPE(pCol->type)) { - p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char)); - } - taosArrayPush(pLastCols, &p); - } - taosThreadMutexLock(&pr->readerMutex); - code = tsdbTakeReadSnap((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap); + code = tsdbTakeReadSnap2((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap); if (code != TSDB_CODE_SUCCESS) { goto _end; } - pr->pDataFReader = NULL; - pr->pDataFReaderLast = NULL; - int32_t ltype = (pr->type & CACHESCAN_RETRIEVE_LAST) >> 3; + int8_t ltype = (pr->type & CACHESCAN_RETRIEVE_LAST) >> 3; + STableKeyInfo* pTableList = pr->pTableList; // retrieve the only one last row of all tables in the uid list. if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_SINGLE)) { + SArray* pLastCols = taosArrayInit(pr->numOfCols, sizeof(SLastCol)); + if (pLastCols == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + for (int32_t i = 0; i < pr->numOfCols; ++i) { + int32_t slotId = slotIds[i]; + struct STColumn* pCol = &pr->pSchema->columns[slotId]; + SLastCol p = {.ts = INT64_MIN, .colVal.type = pCol->type, .colVal.flag = CV_FLAG_NULL}; + + if (IS_VAR_DATA_TYPE(pCol->type)) { + p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char)); + } + taosArrayPush(pLastCols, &p); + } + int64_t st = taosGetTimestampUs(); int64_t totalLastTs = INT64_MAX; for (int32_t i = 0; i < pr->numOfTables; ++i) { - STableKeyInfo* pKeyInfo = &pr->pTableList[i]; + tb_uid_t uid = pTableList[i].uid; - tsdbCacheGet(pr->pTsdb, pKeyInfo->uid, pRow, pr, ltype); - if (TARRAY_SIZE(pRow) <= 0) { - taosArrayClearEx(pRow, freeItem); - continue; - } - SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, 0); - if (COL_VAL_IS_NONE(&pColVal->colVal)) { + tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); + if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { taosArrayClearEx(pRow, freeItem); continue; } @@ -331,6 +346,14 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 hasRes = true; p->ts = pColVal->ts; + if (k == 0) { + if (TARRAY_SIZE(pTableUidList) == 0) { + taosArrayPush(pTableUidList, &uid); + } else { + taosArraySet(pTableUidList, 0, &uid); + } + } + if (pColVal->ts < singleTableLastTs && HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST)) { singleTableLastTs = pColVal->ts; } @@ -361,29 +384,20 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } } - if (taosArrayGetSize(pTableUidList) == 0) { - taosArrayPush(pTableUidList, &pKeyInfo->uid); - } else { - taosArraySet(pTableUidList, 0, &pKeyInfo->uid); - } - taosArrayClearEx(pRow, freeItem); } if (hasRes) { saveOneRow(pLastCols, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); } + + taosArrayDestroyEx(pLastCols, freeItem); } else if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_ALL)) { for (int32_t i = pr->tableIndex; i < pr->numOfTables; ++i) { - STableKeyInfo* pKeyInfo = &pr->pTableList[i]; + tb_uid_t uid = pTableList[i].uid; - tsdbCacheGet(pr->pTsdb, pKeyInfo->uid, pRow, pr, ltype); - if (TARRAY_SIZE(pRow) <= 0) { - taosArrayClearEx(pRow, freeItem); - continue; - } - SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, 0); - if (COL_VAL_IS_NONE(&pColVal->colVal)) { + tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); + if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { taosArrayClearEx(pRow, freeItem); continue; } @@ -391,9 +405,9 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); taosArrayClearEx(pRow, freeItem); - taosArrayPush(pTableUidList, &pKeyInfo->uid); + taosArrayPush(pTableUidList, &uid); - pr->tableIndex += 1; + ++pr->tableIndex; if (pResBlock->info.rows >= pResBlock->info.capacity) { goto _end; } @@ -403,11 +417,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } _end: - tsdbDataFReaderClose(&pr->pDataFReaderLast); - tsdbDataFReaderClose(&pr->pDataFReader); + tsdbUntakeReadSnap2((STsdbReader*)pr, pr->pReadSnap, true); - resetLastBlockLoadInfo(pr->pLoadInfo); - tsdbUntakeReadSnap((STsdbReader*)pr, pr->pReadSnap, true); taosThreadMutexUnlock(&pr->readerMutex); if (pRes != NULL) { @@ -417,7 +428,7 @@ _end: } taosMemoryFree(pRes); - taosArrayDestroyEx(pRow, freeItem); - taosArrayDestroyEx(pLastCols, freeItem); + taosArrayDestroy(pRow); + return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index d15f848cfdcde11c946a943d96b0397933324b14..6376f375eaa584501997550be659131ca2ea52ca 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -266,7 +266,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel suid = pDelIdx->suid; uid = pDelIdx->uid; - code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData); + code = tsdbReadDelDatav1(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); } else { taosArrayClear(pCommitter->aDelData); @@ -562,7 +562,8 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } - tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); + code = tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); + TSDB_CHECK_CODE(code, lino, _exit); wSet.diskId = did; wSet.nSttF = 1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c new file mode 100644 index 0000000000000000000000000000000000000000..d4fa4de510997dfaa269a2a8e75884a5a410b899 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -0,0 +1,636 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbCommit2.h" + +// extern dependencies +typedef struct { + STsdb *tsdb; + TFileSetArray *fsetArr; + TFileOpArray fopArray[1]; + + // SSkmInfo skmTb[1]; + // SSkmInfo skmRow[1]; + + int32_t minutes; + int8_t precision; + int32_t minRow; + int32_t maxRow; + int8_t cmprAlg; + int32_t sttTrigger; + int32_t szPage; + int64_t compactVersion; + + struct { + int64_t cid; + int64_t now; + TSKEY nextKey; + TSKEY maxDelKey; + int32_t fid; + int32_t expLevel; + SDiskID did; + TSKEY minKey; + TSKEY maxKey; + STFileSet *fset; + TABLEID tbid[1]; + bool hasTSData; + } ctx[1]; + + // reader + TSttFileReaderArray sttReaderArray[1]; + + // iter + TTsdbIterArray dataIterArray[1]; + SIterMerger *dataIterMerger; + TTsdbIterArray tombIterArray[1]; + SIterMerger *tombIterMerger; + + // writer + SFSetWriter *writer; +} SCommitter2; + +static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + + SFSetWriterConfig config = { + .tsdb = committer->tsdb, + .toSttOnly = true, + .compactVersion = committer->compactVersion, + .minRow = committer->minRow, + .maxRow = committer->maxRow, + .szPage = committer->szPage, + .cmprAlg = committer->cmprAlg, + .fid = committer->ctx->fid, + .cid = committer->ctx->cid, + .did = committer->ctx->did, + .level = 0, + }; + + if (committer->sttTrigger == 1) { + config.toSttOnly = false; + + if (committer->ctx->fset) { + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ftype++) { + if (committer->ctx->fset->farr[ftype] != NULL) { + config.files[ftype].exist = true; + config.files[ftype].file = committer->ctx->fset->farr[ftype]->f[0]; + } + } + } + } + + code = tsdbFSetWriterOpen(&config, &committer->writer); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbCommitCloseWriter(SCommitter2 *committer) { + return tsdbFSetWriterClose(&committer->writer, 0, committer->fopArray); +} + +static int32_t tsdbCommitTSData(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + int64_t numOfRow = 0; + SMetaInfo info; + + committer->ctx->hasTSData = false; + + committer->ctx->tbid->suid = 0; + committer->ctx->tbid->uid = 0; + for (SRowInfo *row; (row = tsdbIterMergerGetData(committer->dataIterMerger)) != NULL;) { + if (row->uid != committer->ctx->tbid->uid) { + committer->ctx->tbid->suid = row->suid; + committer->ctx->tbid->uid = row->uid; + + if (metaGetInfo(committer->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) { + code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; + } + } + + int64_t ts = TSDBROW_TS(&row->row); + if (ts > committer->ctx->maxKey) { + committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts); + code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; + } + + committer->ctx->hasTSData = true; + numOfRow++; + + code = tsdbFSetWriteRow(committer->writer, row); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbIterMergerNext(committer->dataIterMerger); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->fid, numOfRow); + } + return code; +} + +static int32_t tsdbCommitTombData(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + int64_t numRecord = 0; + SMetaInfo info; + + if (committer->ctx->fset == NULL && !committer->ctx->hasTSData) { + if (committer->ctx->maxKey < committer->ctx->maxDelKey) { + committer->ctx->nextKey = committer->ctx->maxKey + 1; + } else { + committer->ctx->nextKey = TSKEY_MAX; + } + return 0; + } + + committer->ctx->tbid->suid = 0; + committer->ctx->tbid->uid = 0; + for (STombRecord *record; (record = tsdbIterMergerGetTombRecord(committer->tombIterMerger));) { + if (record->uid != committer->ctx->tbid->uid) { + committer->ctx->tbid->suid = record->suid; + committer->ctx->tbid->uid = record->uid; + + if (metaGetInfo(committer->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) { + code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; + } + } + + if (record->ekey < committer->ctx->minKey) { + goto _next; + } else if (record->skey > committer->ctx->maxKey) { + committer->ctx->maxKey = TMIN(record->skey, committer->ctx->maxKey); + goto _next; + } + + TSKEY maxKey = committer->ctx->maxKey; + if (record->ekey > committer->ctx->maxKey) { + maxKey = committer->ctx->maxKey + 1; + } + + if (record->ekey > committer->ctx->maxKey && committer->ctx->nextKey > maxKey) { + committer->ctx->nextKey = maxKey; + } + + record->skey = TMAX(record->skey, committer->ctx->minKey); + record->ekey = TMIN(record->ekey, maxKey); + + numRecord++; + code = tsdbFSetWriteTombRecord(committer->writer, record); + TSDB_CHECK_CODE(code, lino, _exit); + + _next: + code = tsdbIterMergerNext(committer->tombIterMerger); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode), committer->ctx->fid, + numRecord); + } + return code; +} + +static int32_t tsdbCommitOpenReader(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(TARRAY2_SIZE(committer->sttReaderArray) == 0); + + if (committer->ctx->fset == NULL // + || committer->sttTrigger > 1 // + || TARRAY2_SIZE(committer->ctx->fset->lvlArr) == 0 // + ) { + return 0; + } + + ASSERT(TARRAY2_SIZE(committer->ctx->fset->lvlArr) == 1); + + SSttLvl *lvl = TARRAY2_FIRST(committer->ctx->fset->lvlArr); + + ASSERT(lvl->level == 0); + + STFileObj *fobj = NULL; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + SSttFileReader *sttReader; + + SSttFileReaderConfig config = { + .tsdb = committer->tsdb, + .szPage = committer->szPage, + .file = fobj->f[0], + }; + + code = tsdbSttFileReaderOpen(fobj->fname, &config, &sttReader); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(committer->sttReaderArray, sttReader); + TSDB_CHECK_CODE(code, lino, _exit); + + STFileOp op = { + .optype = TSDB_FOP_REMOVE, + .fid = fobj->f->fid, + .of = fobj->f[0], + }; + + code = TARRAY2_APPEND(committer->fopArray, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbCommitCloseReader(SCommitter2 *committer) { + TARRAY2_CLEAR(committer->sttReaderArray, tsdbSttFileReaderClose); + return 0; +} + +static int32_t tsdbCommitOpenIter(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(TARRAY2_SIZE(committer->dataIterArray) == 0); + ASSERT(committer->dataIterMerger == NULL); + ASSERT(TARRAY2_SIZE(committer->tombIterArray) == 0); + ASSERT(committer->tombIterMerger == NULL); + + STsdbIter *iter; + STsdbIterConfig config = {0}; + + // mem data iter + config.type = TSDB_ITER_TYPE_MEMT; + config.memt = committer->tsdb->imem; + config.from->ts = committer->ctx->minKey; + config.from->version = VERSION_MIN; + + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(committer->dataIterArray, iter); + TSDB_CHECK_CODE(code, lino, _exit); + + // mem tomb iter + config.type = TSDB_ITER_TYPE_MEMT_TOMB; + config.memt = committer->tsdb->imem; + + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(committer->tombIterArray, iter); + TSDB_CHECK_CODE(code, lino, _exit); + + // STT + SSttFileReader *sttReader; + TARRAY2_FOREACH(committer->sttReaderArray, sttReader) { + // data iter + config.type = TSDB_ITER_TYPE_STT; + config.sttReader = sttReader; + + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(committer->dataIterArray, iter); + TSDB_CHECK_CODE(code, lino, _exit); + + // tomb iter + config.type = TSDB_ITER_TYPE_STT_TOMB; + config.sttReader = sttReader; + + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(committer->tombIterArray, iter); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // open merger + code = tsdbIterMergerOpen(committer->dataIterArray, &committer->dataIterMerger, false); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbIterMergerOpen(committer->tombIterArray, &committer->tombIterMerger, true); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbCommitCloseIter(SCommitter2 *committer) { + tsdbIterMergerClose(&committer->tombIterMerger); + tsdbIterMergerClose(&committer->dataIterMerger); + TARRAY2_CLEAR(committer->tombIterArray, tsdbIterClose); + TARRAY2_CLEAR(committer->dataIterArray, tsdbIterClose); + return 0; +} + +static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + STsdb *tsdb = committer->tsdb; + + committer->ctx->fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision); + committer->ctx->expLevel = tsdbFidLevel(committer->ctx->fid, &tsdb->keepCfg, committer->ctx->now); + tsdbFidKeyRange(committer->ctx->fid, committer->minutes, committer->precision, &committer->ctx->minKey, + &committer->ctx->maxKey); + code = tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did); + TSDB_CHECK_CODE(code, lino, _exit); + tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did); + STFileSet fset = {.fid = committer->ctx->fid}; + committer->ctx->fset = &fset; + STFileSet **fsetPtr = TARRAY2_SEARCH(committer->fsetArr, &committer->ctx->fset, tsdbTFileSetCmprFn, TD_EQ); + committer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr; + committer->ctx->tbid->suid = 0; + committer->ctx->tbid->uid = 0; + + ASSERT(TARRAY2_SIZE(committer->dataIterArray) == 0); + ASSERT(committer->dataIterMerger == NULL); + ASSERT(committer->writer == NULL); + + code = tsdbCommitOpenReader(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbCommitOpenIter(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbCommitOpenWriter(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + // reset nextKey + committer->ctx->nextKey = TSKEY_MAX; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode), + __func__, committer->ctx->fid, committer->ctx->minKey, committer->ctx->maxKey, committer->ctx->expLevel); + } + return 0; +} + +static int32_t tsdbCommitFileSetEnd(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbCommitCloseWriter(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbCommitCloseIter(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbCommitCloseReader(committer); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid); + } + return code; +} + +static int32_t tsdbCommitFileSet(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + + // fset commit start + code = tsdbCommitFileSetBegin(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + // commit fset + code = tsdbCommitTSData(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbCommitTombData(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + // fset commit end + code = tsdbCommitFileSetEnd(committer); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid); + } + return code; +} + +static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + + memset(committer, 0, sizeof(committer[0])); + + committer->tsdb = tsdb; + code = tsdbFSCreateCopySnapshot(tsdb->pFS, &committer->fsetArr); + TSDB_CHECK_CODE(code, lino, _exit); + committer->minutes = tsdb->keepCfg.days; + committer->precision = tsdb->keepCfg.precision; + committer->minRow = info->info.config.tsdbCfg.minRows; + committer->maxRow = info->info.config.tsdbCfg.maxRows; + committer->cmprAlg = info->info.config.tsdbCfg.compression; + committer->sttTrigger = info->info.config.sttTrigger; + committer->szPage = info->info.config.tsdbPageSize; + committer->compactVersion = INT64_MAX; + committer->ctx->cid = tsdbFSAllocEid(tsdb->pFS); + committer->ctx->now = taosGetTimestampSec(); + + committer->ctx->nextKey = tsdb->imem->minKey; + if (tsdb->imem->nDel > 0) { + SRBTreeIter iter[1] = {tRBTreeIterCreate(tsdb->imem->tbDataTree, 1)}; + + for (SRBTreeNode *node = tRBTreeIterNext(iter); node; node = tRBTreeIterNext(iter)) { + STbData *tbData = TCONTAINER_OF(node, STbData, rbtn); + + for (SDelData *delData = tbData->pHead; delData; delData = delData->pNext) { + if (delData->sKey < committer->ctx->nextKey) { + committer->ctx->nextKey = delData->sKey; + } + } + } + } + + committer->ctx->maxDelKey = TSKEY_MIN; + TSKEY minKey = TSKEY_MAX; + TSKEY maxKey = TSKEY_MIN; + if (TARRAY2_SIZE(committer->fsetArr) > 0) { + STFileSet *fset = TARRAY2_LAST(committer->fsetArr); + tsdbFidKeyRange(fset->fid, committer->minutes, committer->precision, &minKey, &committer->ctx->maxDelKey); + + fset = TARRAY2_FIRST(committer->fsetArr); + tsdbFidKeyRange(fset->fid, committer->minutes, committer->precision, &minKey, &maxKey); + } + + if (committer->ctx->nextKey < TMIN(tsdb->imem->minKey, minKey)) { + committer->ctx->nextKey = TMIN(tsdb->imem->minKey, minKey); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); + } + return code; +} + +static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) { + int32_t code = 0; + int32_t lino = 0; + + if (eno == 0) { + code = tsdbFSEditBegin(committer->tsdb->pFS, committer->fopArray, TSDB_FEDIT_COMMIT); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + // TODO + ASSERT(0); + } + + ASSERT(committer->writer == NULL); + ASSERT(committer->dataIterMerger == NULL); + ASSERT(committer->tombIterMerger == NULL); + TARRAY2_DESTROY(committer->dataIterArray, NULL); + TARRAY2_DESTROY(committer->tombIterArray, NULL); + TARRAY2_DESTROY(committer->fopArray, NULL); + tsdbFSDestroyCopySnapshot(&committer->fsetArr); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, lino, + tstrerror(code), committer->ctx->cid); + } else { + tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->cid); + } + return code; +} + +int32_t tsdbPreCommit(STsdb *tsdb) { + taosThreadRwlockWrlock(&tsdb->rwLock); + ASSERT(tsdb->imem == NULL); + tsdb->imem = tsdb->mem; + tsdb->mem = NULL; + taosThreadRwlockUnlock(&tsdb->rwLock); + return 0; +} + +int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) { + if (!tsdb) return 0; + + int32_t code = 0; + int32_t lino = 0; + + SMemTable *imem = tsdb->imem; + int64_t nRow = imem->nRow; + int64_t nDel = imem->nDel; + + if (nRow == 0 && nDel == 0) { + taosThreadRwlockWrlock(&tsdb->rwLock); + tsdb->imem = NULL; + taosThreadRwlockUnlock(&tsdb->rwLock); + tsdbUnrefMemTable(imem, NULL, true); + } else { + SCommitter2 committer[1]; + + code = tsdbOpenCommitter(tsdb, info, committer); + TSDB_CHECK_CODE(code, lino, _exit); + + while (committer->ctx->nextKey != TSKEY_MAX) { + code = tsdbCommitFileSet(committer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbCloseCommitter(committer, code); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else { + tsdbInfo("vgId:%d %s done, nRow:%" PRId64 " nDel:%" PRId64, TD_VID(tsdb->pVnode), __func__, nRow, nDel); + } + return code; +} + +int32_t tsdbCommitCommit(STsdb *tsdb) { + int32_t code = 0; + int32_t lino = 0; + + if (tsdb->imem == NULL) goto _exit; + + SMemTable *pMemTable = tsdb->imem; + taosThreadRwlockWrlock(&tsdb->rwLock); + code = tsdbFSEditCommit(tsdb->pFS); + if (code) { + taosThreadRwlockUnlock(&tsdb->rwLock); + TSDB_CHECK_CODE(code, lino, _exit); + } + tsdb->imem = NULL; + taosThreadRwlockUnlock(&tsdb->rwLock); + tsdbUnrefMemTable(pMemTable, NULL, true); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else { + tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); + } + return code; +} + +int32_t tsdbCommitAbort(STsdb *pTsdb) { + int32_t code = 0; + int32_t lino = 0; + + if (pTsdb->imem == NULL) goto _exit; + + code = tsdbFSEditAbort(pTsdb->pFS); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + } + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.h b/source/dnode/vnode/src/tsdb/tsdbCommit2.h new file mode 100644 index 0000000000000000000000000000000000000000..41f72f345b4575f90c0632b857e0d0eae6f89a7a --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDataFileRW.h" +#include "tsdbFS2.h" +#include "tsdbFSetRW.h" +#include "tsdbIter.h" +#include "tsdbSttFileRW.h" + +#ifndef _TSDB_COMMIT_H_ +#define _TSDB_COMMIT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_COMMIT_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c new file mode 100644 index 0000000000000000000000000000000000000000..dc5e3649cc3b737b4b0a0dc340751cd2ce7853d3 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -0,0 +1,1696 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDataFileRW.h" + +extern int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, + TTombBlkArray *tombBlkArray, uint8_t **bufArr); +extern int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize); + +// SDataFileReader ============================================= +struct SDataFileReader { + SDataFileReaderConfig config[1]; + + uint8_t *bufArr[5]; + + struct { + bool headFooterLoaded; + bool tombFooterLoaded; + bool brinBlkLoaded; + bool tombBlkLoaded; + } ctx[1]; + + STsdbFD *fd[TSDB_FTYPE_MAX]; + + SHeadFooter headFooter[1]; + STombFooter tombFooter[1]; + TBrinBlkArray brinBlkArray[1]; + TTombBlkArray tombBlkArray[1]; +}; + +static int32_t tsdbDataFileReadHeadFooter(SDataFileReader *reader) { + if (reader->ctx->headFooterLoaded) return 0; + + int32_t code = 0; + int32_t lino = 0; + + int32_t ftype = TSDB_FTYPE_HEAD; + if (reader->fd[ftype]) { + code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(SHeadFooter), + (uint8_t *)reader->headFooter, sizeof(SHeadFooter)); + TSDB_CHECK_CODE(code, lino, _exit); + } + + reader->ctx->headFooterLoaded = true; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileReadTombFooter(SDataFileReader *reader) { + if (reader->ctx->tombFooterLoaded) return 0; + + int32_t code = 0; + int32_t lino = 0; + + int32_t ftype = TSDB_FTYPE_TOMB; + if (reader->fd[ftype]) { + code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(STombFooter), + (uint8_t *)reader->tombFooter, sizeof(STombFooter)); + TSDB_CHECK_CODE(code, lino, _exit); + } + reader->ctx->tombFooterLoaded = true; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReaderOpen(const char *fname[], const SDataFileReaderConfig *config, SDataFileReader **reader) { + int32_t code = 0; + int32_t lino = 0; + + reader[0] = taosMemoryCalloc(1, sizeof(**reader)); + if (reader[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + reader[0]->config[0] = config[0]; + if (reader[0]->config->bufArr == NULL) { + reader[0]->config->bufArr = reader[0]->bufArr; + } + + if (fname) { + for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { + if (fname[i]) { + code = tsdbOpenFile(fname[i], config->szPage, TD_FILE_READ, &reader[0]->fd[i]); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } else { + for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { + if (config->files[i].exist) { + char fname1[TSDB_FILENAME_LEN]; + tsdbTFileName(config->tsdb, &config->files[i].file, fname1); + code = tsdbOpenFile(fname1, config->szPage, TD_FILE_READ, &reader[0]->fd[i]); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReaderClose(SDataFileReader **reader) { + if (reader[0] == NULL) return 0; + + TARRAY2_DESTROY(reader[0]->tombBlkArray, NULL); + TARRAY2_DESTROY(reader[0]->brinBlkArray, NULL); + +#if 0 + TARRAY2_DESTROY(reader[0]->dataBlkArray, NULL); + TARRAY2_DESTROY(reader[0]->blockIdxArray, NULL); +#endif + + for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { + if (reader[0]->fd[i]) { + tsdbCloseFile(&reader[0]->fd[i]); + } + } + + for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->bufArr); ++i) { + tFree(reader[0]->bufArr[i]); + } + + taosMemoryFree(reader[0]); + reader[0] = NULL; + return 0; +} + +int32_t tsdbDataFileReadBrinBlk(SDataFileReader *reader, const TBrinBlkArray **brinBlkArray) { + int32_t code = 0; + int32_t lino = 0; + + if (!reader->ctx->brinBlkLoaded) { + code = tsdbDataFileReadHeadFooter(reader); + TSDB_CHECK_CODE(code, lino, _exit); + + if (reader->headFooter->brinBlkPtr->size > 0) { + void *data = taosMemoryMalloc(reader->headFooter->brinBlkPtr->size); + if (data == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], reader->headFooter->brinBlkPtr->offset, data, + reader->headFooter->brinBlkPtr->size); + if (code) { + taosMemoryFree(data); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int32_t size = reader->headFooter->brinBlkPtr->size / sizeof(SBrinBlk); + TARRAY2_INIT_EX(reader->brinBlkArray, size, size, data); + } else { + TARRAY2_INIT(reader->brinBlkArray); + } + + reader->ctx->brinBlkLoaded = true; + } + brinBlkArray[0] = reader->brinBlkArray; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinBlk, SBrinBlock *brinBlock) { + int32_t code = 0; + int32_t lino = 0; + + code = tRealloc(&reader->config->bufArr[0], brinBlk->dp->size); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], brinBlk->dp->offset, reader->config->bufArr[0], brinBlk->dp->size); + TSDB_CHECK_CODE(code, lino, _exit); + + int32_t size = 0; + tBrinBlockClear(brinBlock); + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); i++) { + code = tsdbDecmprData(reader->config->bufArr[0] + size, brinBlk->size[i], TSDB_DATA_TYPE_BIGINT, brinBlk->cmprAlg, + &reader->config->bufArr[1], brinBlk->numRec * sizeof(int64_t), &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND_BATCH(&brinBlock->dataArr1[i], reader->config->bufArr[1], brinBlk->numRec); + TSDB_CHECK_CODE(code, lino, _exit); + + size += brinBlk->size[i]; + } + + for (int32_t i = 0, j = ARRAY_SIZE(brinBlock->dataArr1); i < ARRAY_SIZE(brinBlock->dataArr2); i++, j++) { + code = tsdbDecmprData(reader->config->bufArr[0] + size, brinBlk->size[j], TSDB_DATA_TYPE_INT, brinBlk->cmprAlg, + &reader->config->bufArr[1], brinBlk->numRec * sizeof(int32_t), &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND_BATCH(&brinBlock->dataArr2[i], reader->config->bufArr[1], brinBlk->numRec); + TSDB_CHECK_CODE(code, lino, _exit); + + size += brinBlk->size[j]; + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReadBlockData(SDataFileReader *reader, const SBrinRecord *record, SBlockData *bData) { + int32_t code = 0; + int32_t lino = 0; + + code = tRealloc(&reader->config->bufArr[0], record->blockSize); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, reader->config->bufArr[0], record->blockSize); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tDecmprBlockData(reader->config->bufArr[0], record->blockSize, bData, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRecord *record, SBlockData *bData, + STSchema *pTSchema, int16_t cids[], int32_t ncid) { + int32_t code = 0; + int32_t lino = 0; + + code = tBlockDataInit(bData, (TABLEID *)record, pTSchema, cids, ncid); + TSDB_CHECK_CODE(code, lino, _exit); + + // uid + version + tskey + code = tRealloc(&reader->config->bufArr[0], record->blockKeySize); + TSDB_CHECK_CODE(code, lino, _exit); + + code = + tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, reader->config->bufArr[0], record->blockKeySize); + TSDB_CHECK_CODE(code, lino, _exit); + + // hdr + SDiskDataHdr hdr[1]; + int32_t size = 0; + + size += tGetDiskDataHdr(reader->config->bufArr[0] + size, hdr); + + ASSERT(hdr->delimiter == TSDB_FILE_DLMT); + ASSERT(record->uid == hdr->uid); + + bData->nRow = hdr->nRow; + + // uid + ASSERT(hdr->uid); + + // version + code = tsdbDecmprData(reader->config->bufArr[0] + size, hdr->szVer, TSDB_DATA_TYPE_BIGINT, hdr->cmprAlg, + (uint8_t **)&bData->aVersion, sizeof(int64_t) * hdr->nRow, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + size += hdr->szVer; + + // ts + code = tsdbDecmprData(reader->config->bufArr[0] + size, hdr->szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr->cmprAlg, + (uint8_t **)&bData->aTSKEY, sizeof(TSKEY) * hdr->nRow, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + size += hdr->szKey; + + ASSERT(size == record->blockKeySize); + + // other columns + if (bData->nColData > 0) { + if (hdr->szBlkCol > 0) { + code = tRealloc(&reader->config->bufArr[0], hdr->szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize, + reader->config->bufArr[0], hdr->szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + } + + SBlockCol bc[1] = {{.cid = 0}}; + SBlockCol *blockCol = bc; + + size = 0; + for (int32_t i = 0; i < bData->nColData; i++) { + SColData *colData = tBlockDataGetColDataByIdx(bData, i); + + while (blockCol && blockCol->cid < colData->cid) { + if (size < hdr->szBlkCol) { + size += tGetBlockCol(reader->config->bufArr[0] + size, blockCol); + } else { + ASSERT(size == hdr->szBlkCol); + blockCol = NULL; + } + } + + if (blockCol == NULL || blockCol->cid > colData->cid) { + for (int32_t iRow = 0; iRow < hdr->nRow; iRow++) { + code = tColDataAppendValue(colData, &COL_VAL_NONE(colData->cid, colData->type)); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + ASSERT(blockCol->type == colData->type); + ASSERT(blockCol->flag && blockCol->flag != HAS_NONE); + + if (blockCol->flag == HAS_NULL) { + for (int32_t iRow = 0; iRow < hdr->nRow; iRow++) { + code = tColDataAppendValue(colData, &COL_VAL_NULL(blockCol->cid, blockCol->type)); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + int32_t size1 = blockCol->szBitmap + blockCol->szOffset + blockCol->szValue; + + code = tRealloc(&reader->config->bufArr[1], size1); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], + record->blockOffset + record->blockKeySize + hdr->szBlkCol + blockCol->offset, + reader->config->bufArr[1], size1); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDecmprColData(reader->config->bufArr[1], blockCol, hdr->cmprAlg, hdr->nRow, colData, + &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReadBlockSma(SDataFileReader *reader, const SBrinRecord *record, + TColumnDataAggArray *columnDataAggArray) { + int32_t code = 0; + int32_t lino = 0; + + TARRAY2_CLEAR(columnDataAggArray, NULL); + if (record->smaSize > 0) { + code = tRealloc(&reader->config->bufArr[0], record->smaSize); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_SMA], record->smaOffset, reader->config->bufArr[0], record->smaSize); + TSDB_CHECK_CODE(code, lino, _exit); + + // decode sma data + int32_t size = 0; + while (size < record->smaSize) { + SColumnDataAgg sma[1]; + + size += tGetColumnDataAgg(reader->config->bufArr[0] + size, sma); + + code = TARRAY2_APPEND_PTR(columnDataAggArray, sma); + TSDB_CHECK_CODE(code, lino, _exit); + } + ASSERT(size == record->smaSize); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReadTombBlk(SDataFileReader *reader, const TTombBlkArray **tombBlkArray) { + int32_t code = 0; + int32_t lino = 0; + + if (!reader->ctx->tombBlkLoaded) { + code = tsdbDataFileReadTombFooter(reader); + TSDB_CHECK_CODE(code, lino, _exit); + + if (reader->tombFooter->tombBlkPtr->size > 0) { + void *data = taosMemoryMalloc(reader->tombFooter->tombBlkPtr->size); + if (data == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], reader->tombFooter->tombBlkPtr->offset, data, + reader->tombFooter->tombBlkPtr->size); + if (code) { + taosMemoryFree(data); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int32_t size = reader->tombFooter->tombBlkPtr->size / sizeof(STombBlk); + TARRAY2_INIT_EX(reader->tombBlkArray, size, size, data); + } else { + TARRAY2_INIT(reader->tombBlkArray); + } + + reader->ctx->tombBlkLoaded = true; + } + tombBlkArray[0] = reader->tombBlkArray; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReadTombBlock(SDataFileReader *reader, const STombBlk *tombBlk, STombBlock *tData) { + int32_t code = 0; + int32_t lino = 0; + + code = tRealloc(&reader->config->bufArr[0], tombBlk->dp->size); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size); + TSDB_CHECK_CODE(code, lino, _exit); + + int32_t size = 0; + tTombBlockClear(tData); + for (int32_t i = 0; i < ARRAY_SIZE(tData->dataArr); ++i) { + code = tsdbDecmprData(reader->config->bufArr[0] + size, tombBlk->size[i], TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, + &reader->config->bufArr[1], sizeof(int64_t) * tombBlk->numRec, &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND_BATCH(&tData->dataArr[i], reader->config->bufArr[1], tombBlk->numRec); + TSDB_CHECK_CODE(code, lino, _exit); + + size += tombBlk->size[i]; + } + ASSERT(size == tombBlk->dp->size); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +// SDataFileWriter ============================================= +struct SDataFileWriter { + SDataFileWriterConfig config[1]; + + SSkmInfo skmTb[1]; + SSkmInfo skmRow[1]; + uint8_t *bufArr[5]; + + struct { + bool opened; + SDataFileReader *reader; + + // for ts data + TABLEID tbid[1]; + bool tbHasOldData; + + const TBrinBlkArray *brinBlkArray; + int32_t brinBlkArrayIdx; + SBrinBlock brinBlock[1]; + int32_t brinBlockIdx; + SBlockData blockData[1]; + int32_t blockDataIdx; + // for tomb data + bool hasOldTomb; + const TTombBlkArray *tombBlkArray; + int32_t tombBlkArrayIdx; + STombBlock tombBlock[1]; + int32_t tombBlockIdx; + } ctx[1]; + + STFile files[TSDB_FTYPE_MAX]; + STsdbFD *fd[TSDB_FTYPE_MAX]; + + SHeadFooter headFooter[1]; + STombFooter tombFooter[1]; + + TBrinBlkArray brinBlkArray[1]; + SBrinBlock brinBlock[1]; + SBlockData blockData[1]; + + TTombBlkArray tombBlkArray[1]; + STombBlock tombBlock[1]; +}; + +static int32_t tsdbDataFileWriterCloseAbort(SDataFileWriter *writer) { + ASSERT(0); + return 0; +} + +static int32_t tsdbDataFileWriterDoClose(SDataFileWriter *writer) { + if (writer->ctx->reader) { + tsdbDataFileReaderClose(&writer->ctx->reader); + } + + tTombBlockDestroy(writer->tombBlock); + TARRAY2_DESTROY(writer->tombBlkArray, NULL); + tBlockDataDestroy(writer->blockData); + tBrinBlockDestroy(writer->brinBlock); + TARRAY2_DESTROY(writer->brinBlkArray, NULL); + + tTombBlockDestroy(writer->ctx->tombBlock); + tBlockDataDestroy(writer->ctx->blockData); + tBrinBlockDestroy(writer->ctx->brinBlock); + + for (int32_t i = 0; i < ARRAY_SIZE(writer->bufArr); ++i) { + tFree(writer->bufArr[i]); + } + + tDestroyTSchema(writer->skmRow->pTSchema); + tDestroyTSchema(writer->skmTb->pTSchema); + return 0; +} + +static int32_t tsdbDataFileWriterDoOpenReader(SDataFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { + if (writer->config->files[i].exist) { + SDataFileReaderConfig config[1] = {{ + .tsdb = writer->config->tsdb, + .szPage = writer->config->szPage, + .bufArr = writer->config->bufArr, + }}; + + for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { + config->files[i].exist = writer->config->files[i].exist; + if (config->files[i].exist) { + config->files[i].file = writer->config->files[i].file; + } + } + + code = tsdbDataFileReaderOpen(NULL, config, &writer->ctx->reader); + TSDB_CHECK_CODE(code, lino, _exit); + break; + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + int32_t ftype; + + if (!writer->config->skmTb) writer->config->skmTb = writer->skmTb; + if (!writer->config->skmRow) writer->config->skmRow = writer->skmRow; + if (!writer->config->bufArr) writer->config->bufArr = writer->bufArr; + + // open reader + code = tsdbDataFileWriterDoOpenReader(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + // .head + ftype = TSDB_FTYPE_HEAD; + writer->files[ftype] = (STFile){ + .type = ftype, + .did = writer->config->did, + .fid = writer->config->fid, + .cid = writer->config->cid, + .size = 0, + }; + + // .data + ftype = TSDB_FTYPE_DATA; + if (writer->config->files[ftype].exist) { + writer->files[ftype] = writer->config->files[ftype].file; + } else { + writer->files[ftype] = (STFile){ + .type = ftype, + .did = writer->config->did, + .fid = writer->config->fid, + .cid = writer->config->cid, + .size = 0, + }; + } + + // .sma + ftype = TSDB_FTYPE_SMA; + if (writer->config->files[ftype].exist) { + writer->files[ftype] = writer->config->files[ftype].file; + } else { + writer->files[ftype] = (STFile){ + .type = ftype, + .did = writer->config->did, + .fid = writer->config->fid, + .cid = writer->config->cid, + .size = 0, + }; + } + + // .tomb + ftype = TSDB_FTYPE_TOMB; + writer->files[ftype] = (STFile){ + .type = ftype, + .did = writer->config->did, + .fid = writer->config->fid, + .cid = writer->config->cid, + .size = 0, + }; + + writer->ctx->opened = true; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize, + TBrinBlkArray *brinBlkArray, uint8_t **bufArr) { + if (BRIN_BLOCK_SIZE(brinBlock) == 0) return 0; + + int32_t code; + + // get SBrinBlk + SBrinBlk brinBlk[1] = { + { + .dp[0] = + { + .offset = *fileSize, + .size = 0, + }, + .minTbid = + { + .suid = TARRAY2_FIRST(brinBlock->suid), + .uid = TARRAY2_FIRST(brinBlock->uid), + }, + .maxTbid = + { + .suid = TARRAY2_LAST(brinBlock->suid), + .uid = TARRAY2_LAST(brinBlock->uid), + }, + .minVer = TARRAY2_FIRST(brinBlock->minVer), + .maxVer = TARRAY2_FIRST(brinBlock->minVer), + .numRec = BRIN_BLOCK_SIZE(brinBlock), + .cmprAlg = cmprAlg, + }, + }; + + for (int32_t i = 1; i < BRIN_BLOCK_SIZE(brinBlock); i++) { + if (brinBlk->minVer > TARRAY2_GET(brinBlock->minVer, i)) { + brinBlk->minVer = TARRAY2_GET(brinBlock->minVer, i); + } + if (brinBlk->maxVer < TARRAY2_GET(brinBlock->maxVer, i)) { + brinBlk->maxVer = TARRAY2_GET(brinBlock->maxVer, i); + } + } + + // write to file + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); i++) { + code = tsdbCmprData((uint8_t *)TARRAY2_DATA(brinBlock->dataArr1 + i), TARRAY2_DATA_LEN(brinBlock->dataArr1 + i), + TSDB_DATA_TYPE_BIGINT, brinBlk->cmprAlg, &bufArr[0], 0, &brinBlk->size[i], &bufArr[1]); + if (code) return code; + + code = tsdbWriteFile(fd, *fileSize, bufArr[0], brinBlk->size[i]); + if (code) return code; + + brinBlk->dp->size += brinBlk->size[i]; + *fileSize += brinBlk->size[i]; + } + + for (int32_t i = 0, j = ARRAY_SIZE(brinBlock->dataArr1); i < ARRAY_SIZE(brinBlock->dataArr2); i++, j++) { + code = tsdbCmprData((uint8_t *)TARRAY2_DATA(brinBlock->dataArr2 + i), TARRAY2_DATA_LEN(brinBlock->dataArr2 + i), + TSDB_DATA_TYPE_INT, brinBlk->cmprAlg, &bufArr[0], 0, &brinBlk->size[j], &bufArr[1]); + if (code) return code; + + code = tsdbWriteFile(fd, *fileSize, bufArr[0], brinBlk->size[j]); + if (code) return code; + + brinBlk->dp->size += brinBlk->size[j]; + *fileSize += brinBlk->size[j]; + } + +#if 0 + SBrinRecord record; + for (int32_t i = 0; i < BRIN_BLOCK_SIZE(brinBlock); i++) { + tBrinBlockGet(brinBlock, i, &record); + tsdbInfo("write brin block, block num:%04d, idx:%04d suid:%ld, uid:%ld, offset:%ld, numRow:%d, count:%d", + TARRAY2_SIZE(brinBlkArray), i, record.suid, record.uid, record.blockOffset, record.numRow, record.count); + } +#endif + + // append to brinBlkArray + code = TARRAY2_APPEND_PTR(brinBlkArray, brinBlk); + if (code) return code; + + tBrinBlockClear(brinBlock); + + return 0; +} + +static int32_t tsdbDataFileWriteBrinBlock(SDataFileWriter *writer) { + if (BRIN_BLOCK_SIZE(writer->brinBlock) == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteBrinBlock(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlock, writer->config->cmprAlg, + &writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->config->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileWriteBrinRecord(SDataFileWriter *writer, const SBrinRecord *record) { + int32_t code = 0; + int32_t lino = 0; + + code = tBrinBlockPut(writer->brinBlock, record); + TSDB_CHECK_CODE(code, lino, _exit); + + if (BRIN_BLOCK_SIZE(writer->brinBlock) >= writer->config->maxRow) { + code = tsdbDataFileWriteBrinBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData *bData) { + if (bData->nRow == 0) return 0; + + ASSERT(bData->uid); + + int32_t code = 0; + int32_t lino = 0; + + SBrinRecord record[1] = {{ + .suid = bData->suid, + .uid = bData->uid, + .firstKey = bData->aTSKEY[0], + .firstKeyVer = bData->aVersion[0], + .lastKey = bData->aTSKEY[bData->nRow - 1], + .lastKeyVer = bData->aVersion[bData->nRow - 1], + .minVer = bData->aVersion[0], + .maxVer = bData->aVersion[0], + .blockOffset = writer->files[TSDB_FTYPE_DATA].size, + .smaOffset = writer->files[TSDB_FTYPE_SMA].size, + .blockSize = 0, + .blockKeySize = 0, + .smaSize = 0, + .numRow = bData->nRow, + .count = 1, + }}; + + for (int32_t i = 1; i < bData->nRow; ++i) { + if (bData->aTSKEY[i] != bData->aTSKEY[i - 1]) { + record->count++; + } + if (bData->aVersion[i] < record->minVer) { + record->minVer = bData->aVersion[i]; + } + if (bData->aVersion[i] > record->maxVer) { + record->maxVer = bData->aVersion[i]; + } + } + + // to .data file + int32_t sizeArr[5] = {0}; + + code = tCmprBlockData(bData, writer->config->cmprAlg, NULL, NULL, writer->config->bufArr, sizeArr); + TSDB_CHECK_CODE(code, lino, _exit); + + record->blockKeySize = sizeArr[3] + sizeArr[2]; + record->blockSize = sizeArr[0] + sizeArr[1] + record->blockKeySize; + + for (int32_t i = 3; i >= 0; --i) { + if (sizeArr[i]) { + code = tsdbWriteFile(writer->fd[TSDB_FTYPE_DATA], writer->files[TSDB_FTYPE_DATA].size, writer->config->bufArr[i], + sizeArr[i]); + TSDB_CHECK_CODE(code, lino, _exit); + writer->files[TSDB_FTYPE_DATA].size += sizeArr[i]; + } + } + + // to .sma file + for (int32_t i = 0; i < bData->nColData; ++i) { + SColData *colData = bData->aColData + i; + if ((!colData->smaOn) || ((colData->flag & HAS_VALUE) == 0)) continue; + + SColumnDataAgg sma[1] = {{.colId = colData->cid}}; + tColDataCalcSMA[colData->type](colData, &sma->sum, &sma->max, &sma->min, &sma->numOfNull); + + int32_t size = tPutColumnDataAgg(NULL, sma); + + code = tRealloc(&writer->config->bufArr[0], record->smaSize + size); + TSDB_CHECK_CODE(code, lino, _exit); + + tPutColumnDataAgg(writer->config->bufArr[0] + record->smaSize, sma); + record->smaSize += size; + } + + if (record->smaSize > 0) { + code = tsdbWriteFile(writer->fd[TSDB_FTYPE_SMA], record->smaOffset, writer->config->bufArr[0], record->smaSize); + TSDB_CHECK_CODE(code, lino, _exit); + writer->files[TSDB_FTYPE_SMA].size += record->smaSize; + } + + // append SBrinRecord + code = tsdbDataFileWriteBrinRecord(writer, record); + TSDB_CHECK_CODE(code, lino, _exit); + + tBlockDataClear(bData); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileWriteDataBlk(SDataFileWriter *writer, const TDataBlkArray *dataBlkArray) { + if (TARRAY2_SIZE(dataBlkArray) == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + int32_t ftype = TSDB_FTYPE_HEAD; + SBlockIdx blockIdx[1] = {{ + .suid = writer->ctx->tbid->suid, + .uid = writer->ctx->tbid->uid, + .offset = writer->files[ftype].size, + .size = TARRAY2_DATA_LEN(dataBlkArray), + }}; + + code = + tsdbWriteFile(writer->fd[ftype], blockIdx->offset, (const uint8_t *)TARRAY2_DATA(dataBlkArray), blockIdx->size); + TSDB_CHECK_CODE(code, lino, _exit); + writer->files[ftype].size += blockIdx->size; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileDoWriteTSRow(SDataFileWriter *writer, TSDBROW *row) { + int32_t code = 0; + int32_t lino = 0; + + // update/append + if (row->type == TSDBROW_ROW_FMT) { + code = tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, TSDBROW_SVERSION(row), writer->config->skmRow); + TSDB_CHECK_CODE(code, lino, _exit); + } + + TSDBKEY key[1]; + if (row->type == TSDBROW_ROW_FMT) { + key->ts = row->pTSRow->ts; + key->version = row->version; + } else { + key->ts = row->pBlockData->aTSKEY[row->iRow]; + key->version = row->pBlockData->aVersion[row->iRow]; + } + if (key->version <= writer->config->compactVersion // + && writer->blockData->nRow > 0 // + && writer->blockData->aTSKEY[writer->blockData->nRow - 1] == key->ts // + ) { + code = tBlockDataUpdateRow(writer->blockData, row, writer->config->skmRow->pTSchema); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + if (writer->blockData->nRow >= writer->config->maxRow) { + code = tsdbDataFileDoWriteBlockData(writer, writer->blockData); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tBlockDataAppendRow(writer->blockData, row, writer->config->skmRow->pTSchema, writer->ctx->tbid->uid); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const TSDBKEY *key) { + if (writer->ctx->tbHasOldData == false) return 0; + + int32_t code = 0; + int32_t lino = 0; + + for (;;) { + for (;;) { + // SBlockData + for (; writer->ctx->blockDataIdx < writer->ctx->blockData->nRow; writer->ctx->blockDataIdx++) { + if (key->ts < writer->ctx->blockData->aTSKEY[writer->ctx->blockDataIdx] // + || (key->ts == writer->ctx->blockData->aTSKEY[writer->ctx->blockDataIdx] && + key->version < writer->ctx->blockData->aVersion[writer->ctx->blockDataIdx])) { + goto _exit; + } else { + TSDBROW row = tsdbRowFromBlockData(writer->ctx->blockData, writer->ctx->blockDataIdx); + code = tsdbDataFileDoWriteTSRow(writer, &row); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + // SBrinBlock + if (writer->ctx->brinBlockIdx >= BRIN_BLOCK_SIZE(writer->ctx->brinBlock)) { + break; + } + + for (; writer->ctx->brinBlockIdx < BRIN_BLOCK_SIZE(writer->ctx->brinBlock); writer->ctx->brinBlockIdx++) { + if (TARRAY2_GET(writer->ctx->brinBlock->uid, writer->ctx->brinBlockIdx) != writer->ctx->tbid->uid) { + writer->ctx->tbHasOldData = false; + goto _exit; + } + + if (key->ts < TARRAY2_GET(writer->ctx->brinBlock->firstKey, writer->ctx->brinBlockIdx) // + || (key->ts == TARRAY2_GET(writer->ctx->brinBlock->firstKey, writer->ctx->brinBlockIdx) && + key->version < TARRAY2_GET(writer->ctx->brinBlock->firstKeyVer, writer->ctx->brinBlockIdx))) { + goto _exit; + } else { + SBrinRecord record[1]; + tBrinBlockGet(writer->ctx->brinBlock, writer->ctx->brinBlockIdx, record); + if (key->ts > record->lastKey || (key->ts == record->lastKey && key->version > record->maxVer)) { + if (writer->blockData->nRow > 0) { + code = tsdbDataFileDoWriteBlockData(writer, writer->blockData); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDataFileWriteBrinRecord(writer, record); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDataFileReadBlockData(writer->ctx->reader, record, writer->ctx->blockData); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->blockDataIdx = 0; + writer->ctx->brinBlockIdx++; + break; + } + } + } + } + + // SBrinBlk + if (writer->ctx->brinBlkArrayIdx >= TARRAY2_SIZE(writer->ctx->brinBlkArray)) { + writer->ctx->brinBlkArray = NULL; + writer->ctx->tbHasOldData = false; + goto _exit; + } + + for (; writer->ctx->brinBlkArrayIdx < TARRAY2_SIZE(writer->ctx->brinBlkArray); writer->ctx->brinBlkArrayIdx++) { + const SBrinBlk *brinBlk = TARRAY2_GET_PTR(writer->ctx->brinBlkArray, writer->ctx->brinBlkArrayIdx); + + if (brinBlk->minTbid.uid != writer->ctx->tbid->uid) { + writer->ctx->tbHasOldData = false; + goto _exit; + } + + code = tsdbDataFileReadBrinBlock(writer->ctx->reader, brinBlk, writer->ctx->brinBlock); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->brinBlockIdx = 0; + writer->ctx->brinBlkArrayIdx++; + break; + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileDoWriteTSData(SDataFileWriter *writer, TSDBROW *row) { + int32_t code = 0; + int32_t lino = 0; + + if (writer->ctx->tbHasOldData) { + TSDBKEY key[1]; + if (row->type == TSDBROW_ROW_FMT) { + key->ts = row->pTSRow->ts; + key->version = row->version; + } else { + key->ts = row->pBlockData->aTSKEY[row->iRow]; + key->version = row->pBlockData->aVersion[row->iRow]; + } + + code = tsdbDataFileDoWriteTableOldData(writer, key); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDataFileDoWriteTSRow(writer, row); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileWriteTableDataEnd(SDataFileWriter *writer) { + if (writer->ctx->tbid->uid == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + if (writer->ctx->tbHasOldData) { + TSDBKEY key = { + .ts = TSKEY_MAX, + .version = VERSION_MAX, + }; + + code = tsdbDataFileDoWriteTableOldData(writer, &key); + TSDB_CHECK_CODE(code, lino, _exit); + + ASSERT(writer->ctx->tbHasOldData == false); + } + + code = tsdbDataFileDoWriteBlockData(writer, writer->blockData); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TABLEID *tbid) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(writer->ctx->blockDataIdx == writer->ctx->blockData->nRow); + ASSERT(writer->blockData->nRow == 0); + + SMetaInfo info; + bool drop = false; + TABLEID tbid1[1]; + writer->ctx->tbHasOldData = false; + while (writer->ctx->brinBlkArray) { // skip data of previous table + for (; writer->ctx->brinBlockIdx < BRIN_BLOCK_SIZE(writer->ctx->brinBlock); writer->ctx->brinBlockIdx++) { + TABLEID tbid2[1] = {{ + .suid = TARRAY2_GET(writer->ctx->brinBlock->suid, writer->ctx->brinBlockIdx), + .uid = TARRAY2_GET(writer->ctx->brinBlock->uid, writer->ctx->brinBlockIdx), + }}; + + if (tbid2->uid == tbid->uid) { + writer->ctx->tbHasOldData = true; + goto _begin; + } else if (tbid2->suid > tbid->suid || (tbid2->suid == tbid->suid && tbid2->uid > tbid->uid)) { + goto _begin; + } else { + if (tbid2->uid != writer->ctx->tbid->uid) { + if (drop && tbid1->uid == tbid2->uid) { + continue; + } else if (metaGetInfo(writer->config->tsdb->pVnode->pMeta, tbid2->uid, &info, NULL) != 0) { + drop = true; + *tbid1 = *tbid2; + continue; + } else { + drop = false; + writer->ctx->tbid[0] = *tbid2; + } + } + + SBrinRecord record[1]; + tBrinBlockGet(writer->ctx->brinBlock, writer->ctx->brinBlockIdx, record); + + code = tsdbDataFileWriteBrinRecord(writer, record); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + if (writer->ctx->brinBlkArrayIdx >= TARRAY2_SIZE(writer->ctx->brinBlkArray)) { + writer->ctx->brinBlkArray = NULL; + break; + } + + for (; writer->ctx->brinBlkArrayIdx < TARRAY2_SIZE(writer->ctx->brinBlkArray); writer->ctx->brinBlkArrayIdx++) { + const SBrinBlk *brinBlk = TARRAY2_GET_PTR(writer->ctx->brinBlkArray, writer->ctx->brinBlkArrayIdx); + + code = tsdbDataFileReadBrinBlock(writer->ctx->reader, brinBlk, writer->ctx->brinBlock); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->brinBlockIdx = 0; + writer->ctx->brinBlkArrayIdx++; + break; + } + } + +_begin: + writer->ctx->tbid[0] = *tbid; + + if (tbid->uid == INT64_MAX) goto _exit; + + code = tsdbUpdateSkmTb(writer->config->tsdb, tbid, writer->config->skmTb); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tBlockDataInit(writer->blockData, writer->ctx->tbid, writer->config->skmTb->pTSchema, NULL, 0); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer) { + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer)); + if (code) return code; + *fileSize += sizeof(*footer); + return 0; +} + +static int32_t tsdbDataFileWriteHeadFooter(SDataFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteHeadFooter(writer->fd[TSDB_FTYPE_HEAD], &writer->files[TSDB_FTYPE_HEAD].size, writer->headFooter); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileDoWriteTombBlock(SDataFileWriter *writer) { + if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg, + &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->config->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileDoWriteTombBlk(SDataFileWriter *writer) { + ASSERT(TARRAY2_SIZE(writer->tombBlkArray) > 0); + + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteTombBlk(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlkArray, writer->tombFooter->tombBlkPtr, + &writer->files[TSDB_FTYPE_TOMB].size); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize) { + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer)); + if (code) return code; + *fileSize += sizeof(*footer); + return 0; +} + +static int32_t tsdbDataFileWriteTombFooter(SDataFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteTombFooter(writer->fd[TSDB_FTYPE_TOMB], writer->tombFooter, &writer->files[TSDB_FTYPE_TOMB].size); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STombRecord *record) { + int32_t code = 0; + int32_t lino = 0; + + while (writer->ctx->hasOldTomb) { + for (; writer->ctx->tombBlockIdx < TOMB_BLOCK_SIZE(writer->ctx->tombBlock); writer->ctx->tombBlockIdx++) { + STombRecord record1[1]; + tTombBlockGet(writer->ctx->tombBlock, writer->ctx->tombBlockIdx, record1); + + int32_t c = tTombRecordCompare(record, record1); + if (c < 0) { + goto _write; + } else if (c > 0) { + code = tTombBlockPut(writer->tombBlock, record1); + TSDB_CHECK_CODE(code, lino, _exit); + + tsdbTrace("vgId:%d write tomb record to tomb file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64 + ", version:%" PRId64, + TD_VID(writer->config->tsdb->pVnode), writer->fd[TSDB_FTYPE_TOMB]->path, writer->config->cid, + record1->suid, record1->uid, record1->version); + + if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) { + code = tsdbDataFileDoWriteTombBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + ASSERT(0); + } + } + + if (writer->ctx->tombBlkArrayIdx >= TARRAY2_SIZE(writer->ctx->tombBlkArray)) { + writer->ctx->hasOldTomb = false; + break; + } + + for (; writer->ctx->tombBlkArrayIdx < TARRAY2_SIZE(writer->ctx->tombBlkArray); ++writer->ctx->tombBlkArrayIdx) { + const STombBlk *tombBlk = TARRAY2_GET_PTR(writer->ctx->tombBlkArray, writer->ctx->tombBlkArrayIdx); + + code = tsdbDataFileReadTombBlock(writer->ctx->reader, tombBlk, writer->ctx->tombBlock); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->tombBlockIdx = 0; + writer->ctx->tombBlkArrayIdx++; + break; + } + } + +_write: + if (record->suid == INT64_MAX) goto _exit; + + code = tTombBlockPut(writer->tombBlock, record); + TSDB_CHECK_CODE(code, lino, _exit); + + tsdbTrace("vgId:%d write tomb record to tomb file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64 + ", version:%" PRId64, + TD_VID(writer->config->tsdb->pVnode), writer->fd[TSDB_FTYPE_TOMB]->path, writer->config->cid, record->suid, + record->uid, record->version); + + if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) { + code = tsdbDataFileDoWriteTombBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize) { + ASSERT(TARRAY2_SIZE(brinBlkArray) > 0); + ptr->offset = *fileSize; + ptr->size = TARRAY2_DATA_LEN(brinBlkArray); + + int32_t code = tsdbWriteFile(fd, ptr->offset, (uint8_t *)TARRAY2_DATA(brinBlkArray), ptr->size); + if (code) return code; + + *fileSize += ptr->size; + return 0; +} + +static int32_t tsdbDataFileWriteBrinBlk(SDataFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteBrinBlk(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlkArray, writer->headFooter->brinBlkPtr, + &writer->files[TSDB_FTYPE_HEAD].size); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArray *opArr) { + int32_t code = 0; + int32_t lino = 0; + + int32_t ftype; + STFileOp op; + + if (writer->fd[TSDB_FTYPE_HEAD]) { + TABLEID tbid[1] = {{ + .suid = INT64_MAX, + .uid = INT64_MAX, + }}; + + code = tsdbDataFileWriteTableDataEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriteTableDataBegin(writer, tbid); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriteBrinBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriteBrinBlk(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriteHeadFooter(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + // .head + ftype = TSDB_FTYPE_HEAD; + if (writer->config->files[ftype].exist) { + op = (STFileOp){ + .optype = TSDB_FOP_REMOVE, + .fid = writer->config->fid, + .of = writer->config->files[ftype].file, + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = writer->config->fid, + .nf = writer->files[ftype], + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + // .data + ftype = TSDB_FTYPE_DATA; + if (!writer->config->files[ftype].exist) { + op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = writer->config->fid, + .nf = writer->files[ftype], + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } else if (writer->config->files[ftype].file.size != writer->files[ftype].size) { + op = (STFileOp){ + .optype = TSDB_FOP_MODIFY, + .fid = writer->config->fid, + .of = writer->config->files[ftype].file, + .nf = writer->files[ftype], + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // .sma + ftype = TSDB_FTYPE_SMA; + if (!writer->config->files[ftype].exist) { + op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = writer->config->fid, + .nf = writer->files[ftype], + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } else if (writer->config->files[ftype].file.size != writer->files[ftype].size) { + op = (STFileOp){ + .optype = TSDB_FOP_MODIFY, + .fid = writer->config->fid, + .of = writer->config->files[ftype].file, + .nf = writer->files[ftype], + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + if (writer->fd[TSDB_FTYPE_TOMB]) { + STombRecord record[1] = {{ + .suid = INT64_MAX, + .uid = INT64_MAX, + .version = INT64_MAX, + }}; + + code = tsdbDataFileDoWriteTombRecord(writer, record); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileDoWriteTombBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileDoWriteTombBlk(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriteTombFooter(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + ftype = TSDB_FTYPE_TOMB; + if (writer->config->files[ftype].exist) { + op = (STFileOp){ + .optype = TSDB_FOP_REMOVE, + .fid = writer->config->fid, + .of = writer->config->files[ftype].file, + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = writer->config->fid, + .nf = writer->files[ftype], + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + + for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { + if (writer->fd[i]) { + code = tsdbFsyncFile(writer->fd[i]); + TSDB_CHECK_CODE(code, lino, _exit); + tsdbCloseFile(&writer->fd[i]); + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileWriterOpenDataFD(SDataFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + int32_t ftypes[] = {TSDB_FTYPE_HEAD, TSDB_FTYPE_DATA, TSDB_FTYPE_SMA}; + + for (int32_t i = 0; i < ARRAY_SIZE(ftypes); ++i) { + int32_t ftype = ftypes[i]; + + char fname[TSDB_FILENAME_LEN]; + int32_t flag = TD_FILE_READ | TD_FILE_WRITE; + + if (writer->files[ftype].size == 0) { + flag |= (TD_FILE_CREATE | TD_FILE_TRUNC); + } + + tsdbTFileName(writer->config->tsdb, &writer->files[ftype], fname); + code = tsdbOpenFile(fname, writer->config->szPage, flag, &writer->fd[ftype]); + TSDB_CHECK_CODE(code, lino, _exit); + + if (writer->files[ftype].size == 0) { + uint8_t hdr[TSDB_FHDR_SIZE] = {0}; + + code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->files[ftype].size += TSDB_FHDR_SIZE; + } + } + + if (writer->ctx->reader) { + code = tsdbDataFileReadBrinBlk(writer->ctx->reader, &writer->ctx->brinBlkArray); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileWriterOpen(const SDataFileWriterConfig *config, SDataFileWriter **writer) { + writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); + if (!writer[0]) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->config[0] = config[0]; + return 0; +} + +int32_t tsdbDataFileWriterClose(SDataFileWriter **writer, bool abort, TFileOpArray *opArr) { + if (writer[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + if (writer[0]->ctx->opened) { + if (abort) { + code = tsdbDataFileWriterCloseAbort(writer[0]); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDataFileWriterCloseCommit(writer[0], opArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + tsdbDataFileWriterDoClose(writer[0]); + } + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileWriteRow(SDataFileWriter *writer, SRowInfo *row) { + int32_t code = 0; + int32_t lino = 0; + + if (!writer->ctx->opened) { + code = tsdbDataFileWriterDoOpen(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (writer->fd[TSDB_FTYPE_HEAD] == NULL) { + code = tsdbDataFileWriterOpenDataFD(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (row->uid != writer->ctx->tbid->uid) { + code = tsdbDataFileWriteTableDataEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriteTableDataBegin(writer, (TABLEID *)row); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDataFileDoWriteTSData(writer, &row->row); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileWriteBlockData(SDataFileWriter *writer, SBlockData *bData) { + if (bData->nRow == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + ASSERT(bData->uid); + + if (!writer->ctx->opened) { + code = tsdbDataFileWriterDoOpen(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (writer->fd[TSDB_FTYPE_DATA] == NULL) { + code = tsdbDataFileWriterOpenDataFD(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (bData->uid != writer->ctx->tbid->uid) { + code = tsdbDataFileWriteTableDataEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriteTableDataBegin(writer, (TABLEID *)bData); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (writer->ctx->tbHasOldData) { + TSDBKEY key = { + .ts = bData->aTSKEY[0], + .version = bData->aVersion[0], + }; + + code = tsdbDataFileDoWriteTableOldData(writer, &key); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (!writer->ctx->tbHasOldData // + && writer->blockData->nRow == 0 // + ) { + code = tsdbDataFileDoWriteBlockData(writer, bData); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + for (int32_t i = 0; i < bData->nRow; ++i) { + TSDBROW row[1] = {tsdbRowFromBlockData(bData, i)}; + code = tsdbDataFileDoWriteTSData(writer, row); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileFlush(SDataFileWriter *writer) { + ASSERT(writer->ctx->opened); + + if (writer->blockData->nRow == 0) return 0; + if (writer->ctx->tbHasOldData) return 0; + + return tsdbDataFileDoWriteBlockData(writer, writer->blockData); +} + +static int32_t tsdbDataFileWriterOpenTombFD(SDataFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + char fname[TSDB_FILENAME_LEN]; + int32_t ftype = TSDB_FTYPE_TOMB; + + ASSERT(writer->files[ftype].size == 0); + + int32_t flag = (TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + + tsdbTFileName(writer->config->tsdb, writer->files + ftype, fname); + code = tsdbOpenFile(fname, writer->config->szPage, flag, &writer->fd[ftype]); + TSDB_CHECK_CODE(code, lino, _exit); + + uint8_t hdr[TSDB_FHDR_SIZE] = {0}; + code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE); + TSDB_CHECK_CODE(code, lino, _exit); + writer->files[ftype].size += TSDB_FHDR_SIZE; + + if (writer->ctx->reader) { + code = tsdbDataFileReadTombBlk(writer->ctx->reader, &writer->ctx->tombBlkArray); + TSDB_CHECK_CODE(code, lino, _exit); + + if (TARRAY2_SIZE(writer->ctx->tombBlkArray) > 0) { + writer->ctx->hasOldTomb = true; + } + + writer->ctx->tombBlkArrayIdx = 0; + tTombBlockClear(writer->ctx->tombBlock); + writer->ctx->tombBlockIdx = 0; + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record) { + int32_t code = 0; + int32_t lino = 0; + + if (!writer->ctx->opened) { + code = tsdbDataFileWriterDoOpen(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (writer->fd[TSDB_FTYPE_TOMB] == NULL) { + code = tsdbDataFileWriterOpenTombFD(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDataFileDoWriteTombRecord(writer, record); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h new file mode 100644 index 0000000000000000000000000000000000000000..827b58fb4a247a0d68c606c7c5a4eaa05b4db5c4 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDef.h" +#include "tsdbFSet2.h" +#include "tsdbSttFileRW.h" +#include "tsdbUtil2.h" + +#ifndef _TSDB_DATA_FILE_RW_H +#define _TSDB_DATA_FILE_RW_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef TARRAY2(SBlockIdx) TBlockIdxArray; +typedef TARRAY2(SDataBlk) TDataBlkArray; +typedef TARRAY2(SColumnDataAgg) TColumnDataAggArray; + +typedef struct { + SFDataPtr brinBlkPtr[1]; + SFDataPtr rsrvd[2]; +} SHeadFooter; + +typedef struct { + SFDataPtr tombBlkPtr[1]; + SFDataPtr rsrvd[2]; +} STombFooter; + +// SDataFileReader ============================================= +typedef struct SDataFileReader SDataFileReader; +typedef struct SDataFileReaderConfig { + STsdb *tsdb; + int32_t szPage; + struct { + bool exist; + STFile file; + } files[TSDB_FTYPE_MAX]; + uint8_t **bufArr; +} SDataFileReaderConfig; + +int32_t tsdbDataFileReaderOpen(const char *fname[/* TSDB_FTYPE_MAX */], const SDataFileReaderConfig *config, + SDataFileReader **reader); +int32_t tsdbDataFileReaderClose(SDataFileReader **reader); +// .head +int32_t tsdbDataFileReadBrinBlk(SDataFileReader *reader, const TBrinBlkArray **brinBlkArray); +int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinBlk, SBrinBlock *brinBlock); +// .data +int32_t tsdbDataFileReadBlockData(SDataFileReader *reader, const SBrinRecord *record, SBlockData *bData); +int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRecord *record, SBlockData *bData, + STSchema *pTSchema, int16_t cids[], int32_t ncid); +// .sma +int32_t tsdbDataFileReadBlockSma(SDataFileReader *reader, const SBrinRecord *record, + TColumnDataAggArray *columnDataAggArray); +// .tomb +int32_t tsdbDataFileReadTombBlk(SDataFileReader *reader, const TTombBlkArray **tombBlkArray); +int32_t tsdbDataFileReadTombBlock(SDataFileReader *reader, const STombBlk *tombBlk, STombBlock *tData); + +// SDataFileWriter ============================================= +typedef struct SDataFileWriter SDataFileWriter; +typedef struct SDataFileWriterConfig { + STsdb *tsdb; + int8_t cmprAlg; + int32_t maxRow; + int32_t szPage; + int32_t fid; + int64_t cid; + SDiskID did; + int64_t compactVersion; + struct { + bool exist; + STFile file; + } files[TSDB_FTYPE_MAX]; + SSkmInfo *skmTb; + SSkmInfo *skmRow; + uint8_t **bufArr; +} SDataFileWriterConfig; + +int32_t tsdbDataFileWriterOpen(const SDataFileWriterConfig *config, SDataFileWriter **writer); +int32_t tsdbDataFileWriterClose(SDataFileWriter **writer, bool abort, TFileOpArray *opArr); + +int32_t tsdbDataFileWriteRow(SDataFileWriter *writer, SRowInfo *row); +int32_t tsdbDataFileWriteBlockData(SDataFileWriter *writer, SBlockData *bData); +int32_t tsdbDataFileFlush(SDataFileWriter *writer); + +int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_DATA_FILE_RW_H*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbDataIter.c b/source/dnode/vnode/src/tsdb/tsdbDataIter.c index e27aec5b1bad10afeb06cb857661ee117aea7e12..8215c1ac290085fbb8efa158785151d7c68bfcc4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataIter.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataIter.c @@ -412,7 +412,7 @@ static int32_t tsdbTombFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* } } - code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData); + code = tsdbReadDelDatav1(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); pIter->delInfo.suid = pDelIdx->suid; diff --git a/source/dnode/vnode/src/tsdb/tsdbDef.h b/source/dnode/vnode/src/tsdb/tsdbDef.h new file mode 100644 index 0000000000000000000000000000000000000000..e768f68b15654ab6cc9327795b80783557896c3e --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbDef.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tarray2.h" +#include "tsdb.h" + +#ifndef _TD_TSDB_DEF_H_ +#define _TD_TSDB_DEF_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define TSDB_ERROR_LOG(vid, lino, code) \ + tsdbError("vgId:%d %s failed at line %d since %s", vid, __func__, lino, tstrerror(code)) + +typedef struct SFDataPtr { + int64_t offset; + int64_t size; +} SFDataPtr; + +extern int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD); +extern void tsdbCloseFile(STsdbFD **ppFD); +extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size); +extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size); +extern int32_t tsdbFsyncFile(STsdbFD *pFD); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TSDB_DEF_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 5519d43012f46dc2e24ad56083ff421d5fcff1c5..ec116c717e05bc3a1be7379b18386c20eb8ef67d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -14,6 +14,7 @@ */ #include "tsdb.h" +#include "vnd.h" // ================================================================================================= static int32_t tsdbFSToBinary(uint8_t *p, STsdbFS *pFS) { @@ -180,10 +181,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { TSDB_CHECK_CODE(code, lino, _exit); } - if (size != tsdbLogicToFileSize(pTsdb->fs.pDelFile->size, pTsdb->pVnode->config.tsdbPageSize)) { - code = TSDB_CODE_FILE_CORRUPTED; - TSDB_CHECK_CODE(code, lino, _exit); - } + // if (size != tsdbLogicToFileSize(pTsdb->fs.pDelFile->size, pTsdb->pVnode->config.tsdbPageSize)) { + // code = TSDB_CODE_FILE_CORRUPTED; + // TSDB_CHECK_CODE(code, lino, _exit); + // } } // SArray @@ -198,10 +199,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code, lino, _exit); } - if (size != tsdbLogicToFileSize(pSet->pHeadF->size, pTsdb->pVnode->config.tsdbPageSize)) { - code = TSDB_CODE_FILE_CORRUPTED; - TSDB_CHECK_CODE(code, lino, _exit); - } + // if (size != tsdbLogicToFileSize(pSet->pHeadF->size, pTsdb->pVnode->config.tsdbPageSize)) { + // code = TSDB_CODE_FILE_CORRUPTED; + // TSDB_CHECK_CODE(code, lino, _exit); + // } // data ========= tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); @@ -209,10 +210,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code, lino, _exit); } - if (size < tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) { - code = TSDB_CODE_FILE_CORRUPTED; - TSDB_CHECK_CODE(code, lino, _exit); - } + // if (size < tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) { + // code = TSDB_CODE_FILE_CORRUPTED; + // TSDB_CHECK_CODE(code, lino, _exit); + // } // else if (size > tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) { // code = tsdbDFileRollback(pTsdb, pSet, TSDB_DATA_FILE); // TSDB_CHECK_CODE(code, lino, _exit); @@ -224,10 +225,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code, lino, _exit); } - if (size < tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) { - code = TSDB_CODE_FILE_CORRUPTED; - TSDB_CHECK_CODE(code, lino, _exit); - } + // if (size < tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) { + // code = TSDB_CODE_FILE_CORRUPTED; + // TSDB_CHECK_CODE(code, lino, _exit); + // } // else if (size > tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) { // code = tsdbDFileRollback(pTsdb, pSet, TSDB_SMA_FILE); // TSDB_CHECK_CODE(code, lino, _exit); @@ -240,10 +241,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code, lino, _exit); } - if (size != tsdbLogicToFileSize(pSet->aSttF[iStt]->size, pTsdb->pVnode->config.tsdbPageSize)) { - code = TSDB_CODE_FILE_CORRUPTED; - TSDB_CHECK_CODE(code, lino, _exit); - } + // if (size != tsdbLogicToFileSize(pSet->aSttF[iStt]->size, pTsdb->pVnode->config.tsdbPageSize)) { + // code = TSDB_CODE_FILE_CORRUPTED; + // TSDB_CHECK_CODE(code, lino, _exit); + // } } } @@ -269,28 +270,26 @@ int32_t tDFileSetCmprFn(const void *p1, const void *p2) { return 0; } -static void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t) { +void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t) { SVnode *pVnode = pTsdb->pVnode; - if (pVnode->pTfs) { - if (current) { - snprintf(current, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), TD_DIRSEP, - pTsdb->path, TD_DIRSEP); - } - if (current_t) { - snprintf(current_t, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT.t", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), TD_DIRSEP, - pTsdb->path, TD_DIRSEP); - } - } else { - if (current) { - snprintf(current, TSDB_FILENAME_LEN - 1, "%s%sCURRENT", pTsdb->path, TD_DIRSEP); - } - if (current_t) { - snprintf(current_t, TSDB_FILENAME_LEN - 1, "%s%sCURRENT.t", pTsdb->path, TD_DIRSEP); - } + int32_t offset = 0; + + // CURRENT + if (current) { + vnodeGetPrimaryDir(pTsdb->path, pVnode->diskPrimary, pVnode->pTfs, current, TSDB_FILENAME_LEN); + offset = strlen(current); + snprintf(current + offset, TSDB_FILENAME_LEN - offset - 1, "%sCURRENT", TD_DIRSEP); + } + + // CURRENT.t + if (current_t) { + vnodeGetPrimaryDir(pTsdb->path, pVnode->diskPrimary, pVnode->pTfs, current_t, TSDB_FILENAME_LEN); + offset = strlen(current_t); + snprintf(current_t + offset, TSDB_FILENAME_LEN - offset - 1, "%sCURRENT.t", TD_DIRSEP); } } -static int32_t tsdbLoadFSFromFile(const char *fname, STsdbFS *pFS) { +static int32_t load_fs(const char *fname, STsdbFS *pFS) { int32_t code = 0; int32_t lino = 0; uint8_t *pData = NULL; @@ -667,7 +666,7 @@ static int32_t tsdbFSApplyChange(STsdb *pTsdb, STsdbFS *pFS) { taosArrayRemove(pTsdb->fs.aDFileSet, iOld); } else { code = tsdbNewFileSet(pTsdb, &fSet, pSetNew); - TSDB_CHECK_CODE(code, lino, _exit) + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -683,7 +682,7 @@ static int32_t tsdbFSApplyChange(STsdb *pTsdb, STsdbFS *pFS) { taosArrayRemove(pTsdb->fs.aDFileSet, iOld); } else { code = tsdbNewFileSet(pTsdb, &fSet, pSetNew); - TSDB_CHECK_CODE(code, lino, _exit) + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -724,7 +723,7 @@ int32_t tsdbFSCommit(STsdb *pTsdb) { code = tsdbFSCreate(&fs); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbLoadFSFromFile(current, &fs); + code = load_fs(current, &fs); TSDB_CHECK_CODE(code, lino, _exit); // apply file change @@ -769,7 +768,7 @@ int32_t tsdbFSOpen(STsdb *pTsdb, int8_t rollback) { tsdbGetCurrentFName(pTsdb, current, current_t); if (taosCheckExistFile(current)) { - code = tsdbLoadFSFromFile(current, &pTsdb->fs); + code = load_fs(current, &pTsdb->fs); TSDB_CHECK_CODE(code, lino, _exit); if (taosCheckExistFile(current_t)) { @@ -1142,4 +1141,4 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { } taosArrayDestroy(pFS->aDFileSet); -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c new file mode 100644 index 0000000000000000000000000000000000000000..afa294d3b011677acce93a5c3ba1041b00c2dc26 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -0,0 +1,894 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFS2.h" +#include "tsdbUpgrade.h" +#include "vnd.h" + +extern int vnodeScheduleTask(int (*execute)(void *), void *arg); +extern int vnodeScheduleTaskEx(int tpid, int (*execute)(void *), void *arg); + +#define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT +#define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1) + +enum { + TSDB_FS_STATE_NONE = 0, + TSDB_FS_STATE_OPEN, + TSDB_FS_STATE_EDIT, + TSDB_FS_STATE_CLOSE, +}; + +static const char *gCurrentFname[] = { + [TSDB_FCURRENT] = "current.json", + [TSDB_FCURRENT_C] = "current.c.json", + [TSDB_FCURRENT_M] = "current.m.json", +}; + +static int32_t create_fs(STsdb *pTsdb, STFileSystem **fs) { + fs[0] = taosMemoryCalloc(1, sizeof(*fs[0])); + if (fs[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + fs[0]->tsdb = pTsdb; + tsem_init(&fs[0]->canEdit, 0, 1); + fs[0]->state = TSDB_FS_STATE_NONE; + fs[0]->neid = 0; + TARRAY2_INIT(fs[0]->fSetArr); + TARRAY2_INIT(fs[0]->fSetArrTmp); + + // background task queue + taosThreadMutexInit(fs[0]->mutex, NULL); + fs[0]->bgTaskQueue->next = fs[0]->bgTaskQueue; + fs[0]->bgTaskQueue->prev = fs[0]->bgTaskQueue; + + return 0; +} + +static int32_t destroy_fs(STFileSystem **fs) { + if (fs[0] == NULL) return 0; + taosThreadMutexDestroy(fs[0]->mutex); + + ASSERT(fs[0]->bgTaskNum == 0); + + TARRAY2_DESTROY(fs[0]->fSetArr, NULL); + TARRAY2_DESTROY(fs[0]->fSetArrTmp, NULL); + tsem_destroy(&fs[0]->canEdit); + taosMemoryFree(fs[0]); + fs[0] = NULL; + return 0; +} + +int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype) { + int32_t offset = 0; + + vnodeGetPrimaryDir(pTsdb->path, pTsdb->pVnode->diskPrimary, pTsdb->pVnode->pTfs, fname, TSDB_FILENAME_LEN); + offset = strlen(fname); + snprintf(fname + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, gCurrentFname[ftype]); + + return 0; +} + +static int32_t save_json(const cJSON *json, const char *fname) { + int32_t code = 0; + + char *data = cJSON_PrintUnformatted(json); + if (data == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + TdFilePtr fp = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (fp == NULL) { + code = TAOS_SYSTEM_ERROR(code); + goto _exit; + } + + if (taosWriteFile(fp, data, strlen(data)) < 0) { + code = TAOS_SYSTEM_ERROR(code); + goto _exit; + } + + if (taosFsyncFile(fp) < 0) { + code = TAOS_SYSTEM_ERROR(code); + goto _exit; + } + + taosCloseFile(&fp); + +_exit: + taosMemoryFree(data); + return code; +} + +static int32_t load_json(const char *fname, cJSON **json) { + int32_t code = 0; + char *data = NULL; + + TdFilePtr fp = taosOpenFile(fname, TD_FILE_READ); + if (fp == NULL) return TAOS_SYSTEM_ERROR(code); + + int64_t size; + if (taosFStatFile(fp, &size, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(code); + goto _exit; + } + + data = taosMemoryMalloc(size + 1); + if (data == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + if (taosReadFile(fp, data, size) < 0) { + code = TAOS_SYSTEM_ERROR(code); + goto _exit; + } + data[size] = '\0'; + + json[0] = cJSON_Parse(data); + if (json[0] == NULL) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + +_exit: + taosCloseFile(&fp); + if (data) taosMemoryFree(data); + if (code) json[0] = NULL; + return code; +} + +int32_t save_fs(const TFileSetArray *arr, const char *fname) { + int32_t code = 0; + int32_t lino = 0; + + cJSON *json = cJSON_CreateObject(); + if (!json) return TSDB_CODE_OUT_OF_MEMORY; + + // fmtv + if (cJSON_AddNumberToObject(json, "fmtv", 1) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // fset + cJSON *ajson = cJSON_AddArrayToObject(json, "fset"); + if (!ajson) TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + const STFileSet *fset; + TARRAY2_FOREACH(arr, fset) { + cJSON *item = cJSON_CreateObject(); + if (!item) TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + cJSON_AddItemToArray(ajson, item); + + code = tsdbTFileSetToJson(fset, item); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = save_json(json, fname); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + cJSON_Delete(json); + return code; +} + +static int32_t load_fs(STsdb *pTsdb, const char *fname, TFileSetArray *arr) { + int32_t code = 0; + int32_t lino = 0; + + TARRAY2_CLEAR(arr, tsdbTFileSetClear); + + // load json + cJSON *json = NULL; + code = load_json(fname, &json); + TSDB_CHECK_CODE(code, lino, _exit); + + // parse json + const cJSON *item1; + + /* fmtv */ + item1 = cJSON_GetObjectItem(json, "fmtv"); + if (cJSON_IsNumber(item1)) { + ASSERT(item1->valuedouble == 1); + } else { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } + + /* fset */ + item1 = cJSON_GetObjectItem(json, "fset"); + if (cJSON_IsArray(item1)) { + const cJSON *item2; + cJSON_ArrayForEach(item2, item1) { + STFileSet *fset; + code = tsdbJsonToTFileSet(pTsdb, item2, &fset); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(arr, fset); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } + +_exit: + if (code) { + tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); + } + if (json) cJSON_Delete(json); + return code; +} + +static bool is_same_file(const STFile *f1, const STFile f2) { + if (f1->type != f2.type) return false; + if (f1->did.level != f2.did.level) return false; + if (f1->did.id != f2.did.id) return false; + if (f1->cid != f2.cid) return false; + return true; +} + +static int32_t apply_commit(STFileSystem *fs) { + int32_t code = 0; + TFileSetArray *fsetArray1 = fs->fSetArr; + TFileSetArray *fsetArray2 = fs->fSetArrTmp; + int32_t i1 = 0, i2 = 0; + + while (i1 < TARRAY2_SIZE(fsetArray1) || i2 < TARRAY2_SIZE(fsetArray2)) { + STFileSet *fset1 = i1 < TARRAY2_SIZE(fsetArray1) ? TARRAY2_GET(fsetArray1, i1) : NULL; + STFileSet *fset2 = i2 < TARRAY2_SIZE(fsetArray2) ? TARRAY2_GET(fsetArray2, i2) : NULL; + + if (fset1 && fset2) { + if (fset1->fid < fset2->fid) { + // delete fset1 + TARRAY2_REMOVE(fsetArray1, i1, tsdbTFileSetRemove); + } else if (fset1->fid > fset2->fid) { + // create new file set with fid of fset2->fid + code = tsdbTFileSetInitDup(fs->tsdb, fset2, &fset1); + if (code) return code; + code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn); + if (code) return code; + i1++; + i2++; + } else { + // edit + code = tsdbTFileSetApplyEdit(fs->tsdb, fset2, fset1); + if (code) return code; + i1++; + i2++; + } + } else if (fset1) { + // delete fset1 + TARRAY2_REMOVE(fsetArray1, i1, tsdbTFileSetRemove); + } else { + // create new file set with fid of fset2->fid + code = tsdbTFileSetInitDup(fs->tsdb, fset2, &fset1); + if (code) return code; + code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn); + if (code) return code; + i1++; + i2++; + } + } + + return 0; +} + +static int32_t commit_edit(STFileSystem *fs) { + char current[TSDB_FILENAME_LEN]; + char current_t[TSDB_FILENAME_LEN]; + + current_fname(fs->tsdb, current, TSDB_FCURRENT); + if (fs->etype == TSDB_FEDIT_COMMIT) { + current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C); + } else if (fs->etype == TSDB_FEDIT_MERGE) { + current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M); + } else { + ASSERT(0); + } + + int32_t code; + int32_t lino; + if ((code = taosRenameFile(current_t, current))) { + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); + } + + code = apply_commit(fs); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(fs->tsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d %s success, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); + } + return code; +} + +// static int32_t +static int32_t apply_abort(STFileSystem *fs) { + // TODO + return 0; +} + +static int32_t abort_edit(STFileSystem *fs) { + char fname[TSDB_FILENAME_LEN]; + + if (fs->etype == TSDB_FEDIT_COMMIT) { + current_fname(fs->tsdb, fname, TSDB_FCURRENT_C); + } else if (fs->etype == TSDB_FEDIT_MERGE) { + current_fname(fs->tsdb, fname, TSDB_FCURRENT_M); + } else { + ASSERT(0); + } + + int32_t code; + int32_t lino; + if ((code = taosRemoveFile(fname))) { + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); + } + + code = apply_abort(fs); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed since %s", TD_VID(fs->tsdb->pVnode), __func__, tstrerror(code)); + } else { + tsdbInfo("vgId:%d %s success, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); + } + return code; +} + +static int32_t tsdbFSScanAndFix(STFileSystem *fs) { + fs->neid = 0; + + // get max commit id + const STFileSet *fset; + TARRAY2_FOREACH(fs->fSetArr, fset) { fs->neid = TMAX(fs->neid, tsdbTFileSetMaxCid(fset)); } + + // TODO + return 0; +} + +static int32_t tsdbFSDupState(STFileSystem *fs) { + int32_t code; + + const TFileSetArray *src = fs->fSetArr; + TFileSetArray *dst = fs->fSetArrTmp; + + TARRAY2_CLEAR(dst, tsdbTFileSetClear); + + const STFileSet *fset1; + TARRAY2_FOREACH(src, fset1) { + STFileSet *fset2; + code = tsdbTFileSetInitDup(fs->tsdb, fset1, &fset2); + if (code) return code; + code = TARRAY2_APPEND(dst, fset2); + if (code) return code; + } + + return 0; +} + +static int32_t open_fs(STFileSystem *fs, int8_t rollback) { + int32_t code = 0; + int32_t lino = 0; + STsdb *pTsdb = fs->tsdb; + + char fCurrent[TSDB_FILENAME_LEN]; + char cCurrent[TSDB_FILENAME_LEN]; + char mCurrent[TSDB_FILENAME_LEN]; + + current_fname(pTsdb, fCurrent, TSDB_FCURRENT); + current_fname(pTsdb, cCurrent, TSDB_FCURRENT_C); + current_fname(pTsdb, mCurrent, TSDB_FCURRENT_M); + + if (taosCheckExistFile(fCurrent)) { // current.json exists + code = load_fs(pTsdb, fCurrent, fs->fSetArr); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosCheckExistFile(cCurrent)) { + // current.c.json exists + + fs->etype = TSDB_FEDIT_COMMIT; + if (rollback) { + code = abort_edit(fs); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = load_fs(pTsdb, cCurrent, fs->fSetArrTmp); + TSDB_CHECK_CODE(code, lino, _exit); + + code = commit_edit(fs); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else if (taosCheckExistFile(mCurrent)) { + // current.m.json exists + fs->etype = TSDB_FEDIT_MERGE; + code = abort_edit(fs); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbFSDupState(fs); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFSScanAndFix(fs); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = save_fs(fs->fSetArr, fCurrent); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d %s success", TD_VID(pTsdb->pVnode), __func__); + } + return 0; +} + +static int32_t close_file_system(STFileSystem *fs) { + TARRAY2_CLEAR(fs->fSetArr, tsdbTFileSetClear); + TARRAY2_CLEAR(fs->fSetArrTmp, tsdbTFileSetClear); + // TODO + return 0; +} + +static int32_t apply_edit(STFileSystem *pFS) { + int32_t code = 0; + ASSERTS(0, "TODO: Not implemented yet"); + return code; +} + +static int32_t fset_cmpr_fn(const struct STFileSet *pSet1, const struct STFileSet *pSet2) { + if (pSet1->fid < pSet2->fid) { + return -1; + } else if (pSet1->fid > pSet2->fid) { + return 1; + } + return 0; +} + +static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFSDupState(fs); + if (code) return code; + + TFileSetArray *fsetArray = fs->fSetArrTmp; + STFileSet *fset = NULL; + const STFileOp *op; + TARRAY2_FOREACH_PTR(opArray, op) { + if (!fset || fset->fid != op->fid) { + STFileSet tfset = {.fid = op->fid}; + fset = &tfset; + STFileSet **fsetPtr = TARRAY2_SEARCH(fsetArray, &fset, tsdbTFileSetCmprFn, TD_EQ); + fset = (fsetPtr == NULL) ? NULL : *fsetPtr; + + if (!fset) { + code = tsdbTFileSetInit(op->fid, &fset); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_SORT_INSERT(fsetArray, fset, tsdbTFileSetCmprFn); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + code = tsdbTFileSetEdit(fs->tsdb, fset, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // remove empty file set + int32_t i = 0; + while (i < TARRAY2_SIZE(fsetArray)) { + fset = TARRAY2_GET(fsetArray, i); + if (tsdbTFileSetIsEmpty(fset)) { + TARRAY2_REMOVE(fsetArray, i, tsdbTFileSetClear); + } else { + i++; + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(fs->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbOpenFS(STsdb *pTsdb, STFileSystem **fs, int8_t rollback) { + int32_t code; + int32_t lino; + + code = tsdbCheckAndUpgradeFileSystem(pTsdb, rollback); + TSDB_CHECK_CODE(code, lino, _exit); + + code = create_fs(pTsdb, fs); + TSDB_CHECK_CODE(code, lino, _exit); + + code = open_fs(fs[0], rollback); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + destroy_fs(fs); + } else { + tsdbInfo("vgId:%d %s success", TD_VID(pTsdb->pVnode), __func__); + } + return 0; +} + +static void tsdbDoWaitBgTask(STFileSystem *fs, STFSBgTask *task) { + task->numWait++; + taosThreadCondWait(task->done, fs->mutex); + task->numWait--; + + if (task->numWait == 0) { + taosThreadCondDestroy(task->done); + if (task->free) { + task->free(task->arg); + } + taosMemoryFree(task); + } +} + +static void tsdbDoDoneBgTask(STFileSystem *fs, STFSBgTask *task) { + if (task->numWait > 0) { + taosThreadCondBroadcast(task->done); + } else { + taosThreadCondDestroy(task->done); + if (task->free) { + task->free(task->arg); + } + taosMemoryFree(task); + } +} + +int32_t tsdbCloseFS(STFileSystem **fs) { + if (fs[0] == NULL) return 0; + + taosThreadMutexLock(fs[0]->mutex); + fs[0]->stop = true; + + if (fs[0]->bgTaskRunning) { + tsdbDoWaitBgTask(fs[0], fs[0]->bgTaskRunning); + } + taosThreadMutexUnlock(fs[0]->mutex); + + close_file_system(fs[0]); + destroy_fs(fs); + return 0; +} + +int64_t tsdbFSAllocEid(STFileSystem *fs) { + taosThreadRwlockRdlock(&fs->tsdb->rwLock); + int64_t cid = ++fs->neid; + taosThreadRwlockUnlock(&fs->tsdb->rwLock); + return cid; +} + +int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype) { + int32_t code = 0; + int32_t lino; + char current_t[TSDB_FILENAME_LEN]; + + switch (etype) { + case TSDB_FEDIT_COMMIT: + current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C); + break; + case TSDB_FEDIT_MERGE: + current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M); + break; + default: + ASSERT(0); + } + + tsem_wait(&fs->canEdit); + fs->etype = etype; + + // edit + code = edit_fs(fs, opArray); + TSDB_CHECK_CODE(code, lino, _exit); + + // save fs + code = save_fs(fs->fSetArrTmp, current_t); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, lino, + tstrerror(code), etype); + } else { + tsdbInfo("vgId:%d %s done, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, etype); + } + return code; +} + +int32_t tsdbFSEditCommit(STFileSystem *fs) { + int32_t code = 0; + int32_t lino = 0; + + // commit + code = commit_edit(fs); + TSDB_CHECK_CODE(code, lino, _exit); + + // schedule merge + if (fs->tsdb->pVnode->config.sttTrigger != 1) { + STFileSet *fset; + TARRAY2_FOREACH_REVERSE(fs->fSetArr, fset) { + if (TARRAY2_SIZE(fset->lvlArr) == 0) continue; + + SSttLvl *lvl = TARRAY2_FIRST(fset->lvlArr); + if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < fs->tsdb->pVnode->config.sttTrigger) continue; + + code = tsdbFSScheduleBgTask(fs, TSDB_BG_TASK_MERGER, tsdbMerge, NULL, fs->tsdb, NULL); + TSDB_CHECK_CODE(code, lino, _exit); + + break; + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(fs->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); + tsem_post(&fs->canEdit); + } + return code; +} + +int32_t tsdbFSEditAbort(STFileSystem *fs) { + int32_t code = abort_edit(fs); + tsem_post(&fs->canEdit); + return code; +} + +int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset) { + STFileSet tfset = {.fid = fid}; + STFileSet *pset = &tfset; + STFileSet **fsetPtr = TARRAY2_SEARCH(fs->fSetArr, &pset, tsdbTFileSetCmprFn, TD_EQ); + fset[0] = (fsetPtr == NULL) ? NULL : fsetPtr[0]; + return 0; +} + +int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr) { + int32_t code = 0; + STFileSet *fset; + STFileSet *fset1; + + fsetArr[0] = taosMemoryMalloc(sizeof(TFileSetArray)); + if (fsetArr == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + TARRAY2_INIT(fsetArr[0]); + + taosThreadRwlockRdlock(&fs->tsdb->rwLock); + TARRAY2_FOREACH(fs->fSetArr, fset) { + code = tsdbTFileSetInitDup(fs->tsdb, fset, &fset1); + if (code) break; + + code = TARRAY2_APPEND(fsetArr[0], fset1); + if (code) break; + } + taosThreadRwlockUnlock(&fs->tsdb->rwLock); + + if (code) { + TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear); + taosMemoryFree(fsetArr[0]); + fsetArr[0] = NULL; + } + return code; +} + +int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr) { + if (fsetArr[0]) { + TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear); + taosMemoryFree(fsetArr[0]); + fsetArr[0] = NULL; + } + return 0; +} + +int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr) { + int32_t code = 0; + STFileSet *fset, *fset1; + + fsetArr[0] = taosMemoryCalloc(1, sizeof(*fsetArr[0])); + if (fsetArr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + taosThreadRwlockRdlock(&fs->tsdb->rwLock); + TARRAY2_FOREACH(fs->fSetArr, fset) { + code = tsdbTFileSetInitRef(fs->tsdb, fset, &fset1); + if (code) break; + + code = TARRAY2_APPEND(fsetArr[0], fset1); + if (code) break; + } + taosThreadRwlockUnlock(&fs->tsdb->rwLock); + + if (code) { + TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear); + fsetArr[0] = NULL; + } + return code; +} + +int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { + if (fsetArr[0]) { + TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear); + taosMemoryFreeClear(fsetArr[0]); + fsetArr[0] = NULL; + } + return 0; +} + +const char *gFSBgTaskName[] = {NULL, "MERGE", "RETENTION", "COMPACT"}; + +static int32_t tsdbFSRunBgTask(void *arg) { + STFileSystem *fs = (STFileSystem *)arg; + + ASSERT(fs->bgTaskRunning != NULL); + + fs->bgTaskRunning->launchTime = taosGetTimestampMs(); + fs->bgTaskRunning->run(fs->bgTaskRunning->arg); + fs->bgTaskRunning->finishTime = taosGetTimestampMs(); + + tsdbDebug("vgId:%d bg task:%s task id:%" PRId64 " finished, schedule time:%" PRId64 " launch time:%" PRId64 + " finish time:%" PRId64, + TD_VID(fs->tsdb->pVnode), gFSBgTaskName[fs->bgTaskRunning->type], fs->bgTaskRunning->taskid, + fs->bgTaskRunning->scheduleTime, fs->bgTaskRunning->launchTime, fs->bgTaskRunning->finishTime); + + taosThreadMutexLock(fs->mutex); + + // free last + tsdbDoDoneBgTask(fs, fs->bgTaskRunning); + fs->bgTaskRunning = NULL; + + // schedule next + if (fs->bgTaskNum > 0) { + if (fs->stop) { + while (fs->bgTaskNum > 0) { + STFSBgTask *task = fs->bgTaskQueue->next; + task->prev->next = task->next; + task->next->prev = task->prev; + fs->bgTaskNum--; + tsdbDoDoneBgTask(fs, task); + } + } else { + // pop task from head + fs->bgTaskRunning = fs->bgTaskQueue->next; + fs->bgTaskRunning->prev->next = fs->bgTaskRunning->next; + fs->bgTaskRunning->next->prev = fs->bgTaskRunning->prev; + fs->bgTaskNum--; + vnodeScheduleTaskEx(1, tsdbFSRunBgTask, arg); + } + } + + taosThreadMutexUnlock(fs->mutex); + return 0; +} + +static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), + void (*destroy)(void *), void *arg, int64_t *taskid) { + if (fs->stop) { + if (destroy) { + destroy(arg); + } + return 0; // TODO: use a better error code + } + + for (STFSBgTask *task = fs->bgTaskQueue->next; task != fs->bgTaskQueue; task = task->next) { + if (task->type == type) { + if (destroy) { + destroy(arg); + } + return 0; + } + } + + // do schedule task + STFSBgTask *task = taosMemoryCalloc(1, sizeof(STFSBgTask)); + if (task == NULL) return TSDB_CODE_OUT_OF_MEMORY; + taosThreadCondInit(task->done, NULL); + + task->type = type; + task->run = run; + task->free = destroy; + task->arg = arg; + task->scheduleTime = taosGetTimestampMs(); + task->taskid = ++fs->taskid; + + if (fs->bgTaskRunning == NULL && fs->bgTaskNum == 0) { + // launch task directly + fs->bgTaskRunning = task; + vnodeScheduleTaskEx(1, tsdbFSRunBgTask, fs); + } else { + // add to the queue tail + fs->bgTaskNum++; + task->next = fs->bgTaskQueue; + task->prev = fs->bgTaskQueue->prev; + task->prev->next = task; + task->next->prev = task; + } + + if (taskid) *taskid = task->taskid; + return 0; +} + +int32_t tsdbFSScheduleBgTask(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void (*free)(void *), void *arg, + int64_t *taskid) { + taosThreadMutexLock(fs->mutex); + int32_t code = tsdbFSScheduleBgTaskImpl(fs, type, run, free, arg, taskid); + taosThreadMutexUnlock(fs->mutex); + return code; +} + +int32_t tsdbFSWaitBgTask(STFileSystem *fs, int64_t taskid) { + STFSBgTask *task = NULL; + + taosThreadMutexLock(fs->mutex); + + if (fs->bgTaskRunning && fs->bgTaskRunning->taskid == taskid) { + task = fs->bgTaskRunning; + } else { + for (STFSBgTask *taskt = fs->bgTaskQueue->next; taskt != fs->bgTaskQueue; taskt = taskt->next) { + if (taskt->taskid == taskid) { + task = taskt; + break; + } + } + } + + if (task) { + tsdbDoWaitBgTask(fs, task); + } + + taosThreadMutexUnlock(fs->mutex); + return 0; +} + +int32_t tsdbFSWaitAllBgTask(STFileSystem *fs) { + taosThreadMutexLock(fs->mutex); + + while (fs->bgTaskRunning) { + taosThreadCondWait(fs->bgTaskRunning->done, fs->mutex); + } + + taosThreadMutexUnlock(fs->mutex); + return 0; +} + +static int32_t tsdbFSDoDisableBgTask(STFileSystem *fs) { + fs->stop = true; + + if (fs->bgTaskRunning) { + tsdbDoWaitBgTask(fs, fs->bgTaskRunning); + } + return 0; +} + +int32_t tsdbFSDisableBgTask(STFileSystem *fs) { + taosThreadMutexLock(fs->mutex); + int32_t code = tsdbFSDoDisableBgTask(fs); + taosThreadMutexUnlock(fs->mutex); + return code; +} + +int32_t tsdbFSEnableBgTask(STFileSystem *fs) { + taosThreadMutexLock(fs->mutex); + fs->stop = false; + taosThreadMutexUnlock(fs->mutex); + return 0; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h new file mode 100644 index 0000000000000000000000000000000000000000..e814ab2fffb2a49b76fa2642efd309db0ddd7f2b --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFSet2.h" + +#ifndef _TSDB_FILE_SYSTEM_H +#define _TSDB_FILE_SYSTEM_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Exposed Handle */ +typedef struct STFileSystem STFileSystem; +typedef struct STFSBgTask STFSBgTask; +// typedef TARRAY2(STFileSet *) TFileSetArray; + +typedef enum { + TSDB_FEDIT_COMMIT = 1, // + TSDB_FEDIT_MERGE +} EFEditT; + +typedef enum { + TSDB_BG_TASK_MERGER = 1, + TSDB_BG_TASK_RETENTION, + TSDB_BG_TASK_COMPACT, +} EFSBgTaskT; + +typedef enum { + TSDB_FCURRENT = 1, + TSDB_FCURRENT_C, // for commit + TSDB_FCURRENT_M, // for merge +} EFCurrentT; + +/* Exposed APIs */ +// open/close +int32_t tsdbOpenFS(STsdb *pTsdb, STFileSystem **fs, int8_t rollback); +int32_t tsdbCloseFS(STFileSystem **fs); +// snapshot +int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr); +int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr); +int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr); +int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr); +// txn +int64_t tsdbFSAllocEid(STFileSystem *fs); +int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype); +int32_t tsdbFSEditCommit(STFileSystem *fs); +int32_t tsdbFSEditAbort(STFileSystem *fs); +// background task +int32_t tsdbFSScheduleBgTask(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void (*free)(void *), void *arg, + int64_t *taskid); +int32_t tsdbFSWaitBgTask(STFileSystem *fs, int64_t taskid); +int32_t tsdbFSWaitAllBgTask(STFileSystem *fs); +int32_t tsdbFSDisableBgTask(STFileSystem *fs); +int32_t tsdbFSEnableBgTask(STFileSystem *fs); +// other +int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset); + +struct STFSBgTask { + EFSBgTaskT type; + int32_t (*run)(void *arg); + void (*free)(void *arg); + void *arg; + + TdThreadCond done[1]; + int32_t numWait; + + int64_t taskid; + int64_t scheduleTime; + int64_t launchTime; + int64_t finishTime; + + struct STFSBgTask *prev; + struct STFSBgTask *next; +}; + +/* Exposed Structs */ +struct STFileSystem { + STsdb *tsdb; + tsem_t canEdit; + int32_t state; + int64_t neid; + EFEditT etype; + TFileSetArray fSetArr[1]; + TFileSetArray fSetArrTmp[1]; + + // background task queue + TdThreadMutex mutex[1]; + bool stop; + int64_t taskid; + int32_t bgTaskNum; + STFSBgTask bgTaskQueue[1]; + STFSBgTask *bgTaskRunning; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_FILE_SYSTEM_H*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c new file mode 100644 index 0000000000000000000000000000000000000000..7bc9743ecb726c9305572ba0ab0db184355bbc08 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -0,0 +1,542 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFSet2.h" + +int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl) { + if (!(lvl[0] = taosMemoryMalloc(sizeof(SSttLvl)))) return TSDB_CODE_OUT_OF_MEMORY; + lvl[0]->level = level; + TARRAY2_INIT(lvl[0]->fobjArr); + return 0; +} + +static void tsdbSttLvlClearFObj(void *data) { tsdbTFileObjUnref(*(STFileObj **)data); } + +int32_t tsdbSttLvlClear(SSttLvl **lvl) { + if (lvl[0] != NULL) { + TARRAY2_DESTROY(lvl[0]->fobjArr, tsdbSttLvlClearFObj); + taosMemoryFree(lvl[0]); + lvl[0] = NULL; + } + return 0; +} + +static int32_t tsdbSttLvlInitEx(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl **lvl) { + int32_t code = tsdbSttLvlInit(lvl1->level, lvl); + if (code) return code; + + const STFileObj *fobj1; + TARRAY2_FOREACH(lvl1->fobjArr, fobj1) { + STFileObj *fobj; + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj); + if (code) { + tsdbSttLvlClear(lvl); + return code; + } + + TARRAY2_APPEND(lvl[0]->fobjArr, fobj); + } + return 0; +} + +static int32_t tsdbSttLvlInitRef(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl **lvl) { + int32_t code = tsdbSttLvlInit(lvl1->level, lvl); + if (code) return code; + + STFileObj *fobj1; + TARRAY2_FOREACH(lvl1->fobjArr, fobj1) { + tsdbTFileObjRef(fobj1); + code = TARRAY2_APPEND(lvl[0]->fobjArr, fobj1); + if (code) return code; + } + return 0; +} + +static void tsdbSttLvlRemoveFObj(void *data) { tsdbTFileObjRemove(*(STFileObj **)data); } +static void tsdbSttLvlRemove(SSttLvl **lvl) { + TARRAY2_DESTROY(lvl[0]->fobjArr, tsdbSttLvlRemoveFObj); + taosMemoryFree(lvl[0]); + lvl[0] = NULL; +} + +static int32_t tsdbSttLvlApplyEdit(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl *lvl2) { + int32_t code = 0; + + ASSERT(lvl1->level == lvl2->level); + + int32_t i1 = 0, i2 = 0; + while (i1 < TARRAY2_SIZE(lvl1->fobjArr) || i2 < TARRAY2_SIZE(lvl2->fobjArr)) { + STFileObj *fobj1 = i1 < TARRAY2_SIZE(lvl1->fobjArr) ? TARRAY2_GET(lvl1->fobjArr, i1) : NULL; + STFileObj *fobj2 = i2 < TARRAY2_SIZE(lvl2->fobjArr) ? TARRAY2_GET(lvl2->fobjArr, i2) : NULL; + + if (fobj1 && fobj2) { + if (fobj1->f->cid < fobj2->f->cid) { + // create a file obj + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj2); + if (code) return code; + code = TARRAY2_APPEND(lvl2->fobjArr, fobj2); + if (code) return code; + i1++; + i2++; + } else if (fobj1->f->cid > fobj2->f->cid) { + // remove a file obj + TARRAY2_REMOVE(lvl2->fobjArr, i2, tsdbSttLvlRemoveFObj); + } else { + if (tsdbIsSameTFile(fobj1->f, fobj2->f)) { + if (tsdbIsTFileChanged(fobj1->f, fobj2->f)) { + fobj2->f[0] = fobj1->f[0]; + } + } else { + TARRAY2_REMOVE(lvl2->fobjArr, i2, tsdbSttLvlRemoveFObj); + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj2); + if (code) return code; + code = TARRAY2_SORT_INSERT(lvl2->fobjArr, fobj2, tsdbTFileObjCmpr); + if (code) return code; + } + i1++; + i2++; + } + } else if (fobj1) { + // create a file obj + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj2); + if (code) return code; + code = TARRAY2_APPEND(lvl2->fobjArr, fobj2); + if (code) return code; + i1++; + i2++; + } else { + // remove a file obj + TARRAY2_REMOVE(lvl2->fobjArr, i2, tsdbSttLvlRemoveFObj); + } + } + return 0; +} + +static int32_t tsdbSttLvlCmprFn(const SSttLvl **lvl1, const SSttLvl **lvl2) { + if (lvl1[0]->level < lvl2[0]->level) return -1; + if (lvl1[0]->level > lvl2[0]->level) return 1; + return 0; +} + +static int32_t tsdbSttLvlToJson(const SSttLvl *lvl, cJSON *json) { + if (cJSON_AddNumberToObject(json, "level", lvl->level) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + cJSON *ajson = cJSON_AddArrayToObject(json, "files"); + if (ajson == NULL) return TSDB_CODE_OUT_OF_MEMORY; + const STFileObj *fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + cJSON *item = cJSON_CreateObject(); + if (item == NULL) return TSDB_CODE_OUT_OF_MEMORY; + cJSON_AddItemToArray(ajson, item); + + int32_t code = tsdbTFileToJson(fobj->f, item); + if (code) return code; + } + + return 0; +} + +static int32_t tsdbJsonToSttLvl(STsdb *pTsdb, const cJSON *json, SSttLvl **lvl) { + const cJSON *item1, *item2; + int32_t level; + + item1 = cJSON_GetObjectItem(json, "level"); + if (cJSON_IsNumber(item1)) { + level = item1->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + int32_t code = tsdbSttLvlInit(level, lvl); + if (code) return code; + + item1 = cJSON_GetObjectItem(json, "files"); + if (!cJSON_IsArray(item1)) { + tsdbSttLvlClear(lvl); + return TSDB_CODE_FILE_CORRUPTED; + } + + cJSON_ArrayForEach(item2, item1) { + STFile tf; + code = tsdbJsonToTFile(item2, TSDB_FTYPE_STT, &tf); + if (code) { + tsdbSttLvlClear(lvl); + return code; + } + + STFileObj *fobj; + code = tsdbTFileObjInit(pTsdb, &tf, &fobj); + if (code) { + tsdbSttLvlClear(lvl); + return code; + } + + TARRAY2_APPEND(lvl[0]->fobjArr, fobj); + } + return 0; +} + +int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json) { + int32_t code = 0; + cJSON *item1, *item2; + + // fid + if (cJSON_AddNumberToObject(json, "fid", fset->fid) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset->farr[ftype] == NULL) continue; + + code = tsdbTFileToJson(fset->farr[ftype]->f, json); + if (code) return code; + } + + // each level + item1 = cJSON_AddArrayToObject(json, "stt lvl"); + if (item1 == NULL) return TSDB_CODE_OUT_OF_MEMORY; + const SSttLvl *lvl; + TARRAY2_FOREACH(fset->lvlArr, lvl) { + item2 = cJSON_CreateObject(); + if (!item2) return TSDB_CODE_OUT_OF_MEMORY; + cJSON_AddItemToArray(item1, item2); + + code = tsdbSttLvlToJson(lvl, item2); + if (code) return code; + } + + return 0; +} + +int32_t tsdbJsonToTFileSet(STsdb *pTsdb, const cJSON *json, STFileSet **fset) { + int32_t code; + const cJSON *item1, *item2; + int32_t fid; + STFile tf; + + // fid + item1 = cJSON_GetObjectItem(json, "fid"); + if (cJSON_IsNumber(item1)) { + fid = item1->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + code = tsdbTFileSetInit(fid, fset); + if (code) return code; + + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + code = tsdbJsonToTFile(json, ftype, &tf); + if (code == TSDB_CODE_NOT_FOUND) { + continue; + } else if (code) { + tsdbTFileSetClear(fset); + return code; + } else { + code = tsdbTFileObjInit(pTsdb, &tf, &(*fset)->farr[ftype]); + if (code) return code; + } + } + + // each level + item1 = cJSON_GetObjectItem(json, "stt lvl"); + if (cJSON_IsArray(item1)) { + cJSON_ArrayForEach(item2, item1) { + SSttLvl *lvl; + code = tsdbJsonToSttLvl(pTsdb, item2, &lvl); + if (code) { + tsdbTFileSetClear(fset); + return code; + } + + TARRAY2_APPEND((*fset)->lvlArr, lvl); + } + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + return 0; +} + +// NOTE: the api does not remove file, only do memory operation +int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) { + int32_t code = 0; + + if (op->optype == TSDB_FOP_CREATE) { + // create a new file + STFileObj *fobj; + code = tsdbTFileObjInit(pTsdb, &op->nf, &fobj); + if (code) return code; + + if (fobj->f->type == TSDB_FTYPE_STT) { + SSttLvl *lvl = tsdbTFileSetGetSttLvl(fset, fobj->f->stt->level); + if (!lvl) { + code = tsdbSttLvlInit(fobj->f->stt->level, &lvl); + if (code) return code; + + code = TARRAY2_SORT_INSERT(fset->lvlArr, lvl, tsdbSttLvlCmprFn); + if (code) return code; + } + + code = TARRAY2_SORT_INSERT(lvl->fobjArr, fobj, tsdbTFileObjCmpr); + if (code) return code; + } else { + ASSERT(fset->farr[fobj->f->type] == NULL); + fset->farr[fobj->f->type] = fobj; + } + } else if (op->optype == TSDB_FOP_REMOVE) { + // delete a file + if (op->of.type == TSDB_FTYPE_STT) { + SSttLvl *lvl = tsdbTFileSetGetSttLvl(fset, op->of.stt->level); + ASSERT(lvl); + + STFileObj tfobj = {.f[0] = {.cid = op->of.cid}}; + STFileObj *tfobjp = &tfobj; + int32_t idx = TARRAY2_SEARCH_IDX(lvl->fobjArr, &tfobjp, tsdbTFileObjCmpr, TD_EQ); + ASSERT(idx >= 0); + TARRAY2_REMOVE(lvl->fobjArr, idx, tsdbSttLvlClearFObj); + + if (TARRAY2_SIZE(lvl->fobjArr) == 0) { + // TODO: remove the stt level if no file exists anymore + // TARRAY2_REMOVE(&fset->lvlArr, lvl - fset->lvlArr.data, tsdbSttLvlClear); + } + } else { + ASSERT(tsdbIsSameTFile(&op->of, fset->farr[op->of.type]->f)); + tsdbTFileObjUnref(fset->farr[op->of.type]); + fset->farr[op->of.type] = NULL; + } + } else { + if (op->nf.type == TSDB_FTYPE_STT) { + SSttLvl *lvl = tsdbTFileSetGetSttLvl(fset, op->of.stt->level); + ASSERT(lvl); + + STFileObj tfobj = {.f[0] = {.cid = op->of.cid}}, *tfobjp = &tfobj; + STFileObj **fobjPtr = TARRAY2_SEARCH(lvl->fobjArr, &tfobjp, tsdbTFileObjCmpr, TD_EQ); + tfobjp = (fobjPtr ? *fobjPtr : NULL); + + ASSERT(tfobjp); + + tfobjp->f[0] = op->nf; + } else { + fset->farr[op->nf.type]->f[0] = op->nf; + } + } + + return 0; +} + +int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *fset2) { + int32_t code = 0; + + ASSERT(fset1->fid == fset2->fid); + + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (!fset1->farr[ftype] && !fset2->farr[ftype]) continue; + + STFileObj *fobj1 = fset1->farr[ftype]; + STFileObj *fobj2 = fset2->farr[ftype]; + + if (fobj1 && fobj2) { + if (tsdbIsSameTFile(fobj1->f, fobj2->f)) { + if (tsdbIsTFileChanged(fobj1->f, fobj2->f)) { + fobj2->f[0] = fobj1->f[0]; + } + } else { + tsdbTFileObjRemove(fobj2); + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fset2->farr[ftype]); + if (code) return code; + } + } else if (fobj1) { + // create a new file + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fset2->farr[ftype]); + if (code) return code; + } else { + // remove the file + tsdbTFileObjRemove(fobj2); + fset2->farr[ftype] = NULL; + } + } + + // stt part + int32_t i1 = 0, i2 = 0; + while (i1 < TARRAY2_SIZE(fset1->lvlArr) || i2 < TARRAY2_SIZE(fset2->lvlArr)) { + SSttLvl *lvl1 = i1 < TARRAY2_SIZE(fset1->lvlArr) ? TARRAY2_GET(fset1->lvlArr, i1) : NULL; + SSttLvl *lvl2 = i2 < TARRAY2_SIZE(fset2->lvlArr) ? TARRAY2_GET(fset2->lvlArr, i2) : NULL; + + if (lvl1 && lvl2) { + if (lvl1->level < lvl2->level) { + // add a new stt level + code = tsdbSttLvlInitEx(pTsdb, lvl1, &lvl2); + if (code) return code; + code = TARRAY2_SORT_INSERT(fset2->lvlArr, lvl2, tsdbSttLvlCmprFn); + if (code) return code; + i1++; + i2++; + } else if (lvl1->level > lvl2->level) { + // remove the stt level + TARRAY2_REMOVE(fset2->lvlArr, i2, tsdbSttLvlRemove); + } else { + // apply edit on stt level + code = tsdbSttLvlApplyEdit(pTsdb, lvl1, lvl2); + if (code) return code; + i1++; + i2++; + } + } else if (lvl1) { + // add a new stt level + code = tsdbSttLvlInitEx(pTsdb, lvl1, &lvl2); + if (code) return code; + code = TARRAY2_SORT_INSERT(fset2->lvlArr, lvl2, tsdbSttLvlCmprFn); + if (code) return code; + i1++; + i2++; + } else { + // remove the stt level + TARRAY2_REMOVE(fset2->lvlArr, i2, tsdbSttLvlRemove); + } + } + + return 0; +} + +int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) { + fset[0] = taosMemoryCalloc(1, sizeof(STFileSet)); + if (fset[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + fset[0]->fid = fid; + TARRAY2_INIT(fset[0]->lvlArr); + return 0; +} + +int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset) { + int32_t code = tsdbTFileSetInit(fset1->fid, fset); + if (code) return code; + + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset1->farr[ftype] == NULL) continue; + + code = tsdbTFileObjInit(pTsdb, fset1->farr[ftype]->f, &fset[0]->farr[ftype]); + if (code) { + tsdbTFileSetClear(fset); + return code; + } + } + + const SSttLvl *lvl1; + TARRAY2_FOREACH(fset1->lvlArr, lvl1) { + SSttLvl *lvl; + code = tsdbSttLvlInitEx(pTsdb, lvl1, &lvl); + if (code) { + tsdbTFileSetClear(fset); + return code; + } + + code = TARRAY2_APPEND(fset[0]->lvlArr, lvl); + if (code) return code; + } + + return 0; +} + +int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset) { + int32_t code = tsdbTFileSetInit(fset1->fid, fset); + if (code) return code; + + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset1->farr[ftype] == NULL) continue; + + tsdbTFileObjRef(fset1->farr[ftype]); + fset[0]->farr[ftype] = fset1->farr[ftype]; + } + + const SSttLvl *lvl1; + TARRAY2_FOREACH(fset1->lvlArr, lvl1) { + SSttLvl *lvl; + code = tsdbSttLvlInitRef(pTsdb, lvl1, &lvl); + if (code) { + tsdbTFileSetClear(fset); + return code; + } + + code = TARRAY2_APPEND(fset[0]->lvlArr, lvl); + if (code) return code; + } + + return 0; +} + +int32_t tsdbTFileSetClear(STFileSet **fset) { + if (!fset[0]) return 0; + + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset[0]->farr[ftype] == NULL) continue; + tsdbTFileObjUnref(fset[0]->farr[ftype]); + } + + TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlClear); + + taosMemoryFree(fset[0]); + fset[0] = NULL; + + return 0; +} + +int32_t tsdbTFileSetRemove(STFileSet **fset) { + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset[0]->farr[ftype] == NULL) continue; + tsdbTFileObjRemove(fset[0]->farr[ftype]); + } + + TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlRemove); + taosMemoryFree(fset[0]); + fset[0] = NULL; + return 0; +} + +SSttLvl *tsdbTFileSetGetSttLvl(STFileSet *fset, int32_t level) { + SSttLvl sttLvl = {.level = level}; + SSttLvl *lvl = &sttLvl; + SSttLvl **lvlPtr = TARRAY2_SEARCH(fset->lvlArr, &lvl, tsdbSttLvlCmprFn, TD_EQ); + return lvlPtr ? lvlPtr[0] : NULL; +} + +int32_t tsdbTFileSetCmprFn(const STFileSet **fset1, const STFileSet **fset2) { + if (fset1[0]->fid < fset2[0]->fid) return -1; + if (fset1[0]->fid > fset2[0]->fid) return 1; + return 0; +} + +int64_t tsdbTFileSetMaxCid(const STFileSet *fset) { + int64_t maxCid = 0; + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset->farr[ftype] == NULL) continue; + maxCid = TMAX(maxCid, fset->farr[ftype]->f->cid); + } + const SSttLvl *lvl; + const STFileObj *fobj; + TARRAY2_FOREACH(fset->lvlArr, lvl) { + TARRAY2_FOREACH(lvl->fobjArr, fobj) { maxCid = TMAX(maxCid, fobj->f->cid); } + } + return maxCid; +} + +bool tsdbTFileSetIsEmpty(const STFileSet *fset) { + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset->farr[ftype] != NULL) return false; + } + return TARRAY2_SIZE(fset->lvlArr) == 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h new file mode 100644 index 0000000000000000000000000000000000000000..d7b3c1fc8cb3478b8ad6fe85d94ec7a992dad0c8 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFile2.h" + +#ifndef _TSDB_FILE_SET2_H +#define _TSDB_FILE_SET2_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct STFileSet STFileSet; +typedef struct STFileOp STFileOp; +typedef struct SSttLvl SSttLvl; +typedef TARRAY2(STFileObj *) TFileObjArray; +typedef TARRAY2(SSttLvl *) TSttLvlArray; +typedef TARRAY2(STFileOp) TFileOpArray; + +typedef enum { + TSDB_FOP_NONE = 0, + TSDB_FOP_CREATE, + TSDB_FOP_REMOVE, + TSDB_FOP_MODIFY, +} tsdb_fop_t; + +#define TFILE_SET(fid_) \ + (STFileSet) { .fid = (fid_) } + +// init/clear +int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset); +int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset); +int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset); +int32_t tsdbTFileSetClear(STFileSet **fset); +int32_t tsdbTFileSetRemove(STFileSet **fset); +// to/from json +int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json); +int32_t tsdbJsonToTFileSet(STsdb *pTsdb, const cJSON *json, STFileSet **fset); +// cmpr +int32_t tsdbTFileSetCmprFn(const STFileSet **fset1, const STFileSet **fset2); +// edit +int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op); +int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *fset); +// max commit id +int64_t tsdbTFileSetMaxCid(const STFileSet *fset); +// get +SSttLvl *tsdbTFileSetGetSttLvl(STFileSet *fset, int32_t level); +// is empty +bool tsdbTFileSetIsEmpty(const STFileSet *fset); + +struct STFileOp { + tsdb_fop_t optype; + int32_t fid; + STFile of; // old file state + STFile nf; // new file state +}; + +struct SSttLvl { + int32_t level; + TFileObjArray fobjArr[1]; +}; + +struct STFileSet { + int32_t fid; + STFileObj *farr[TSDB_FTYPE_MAX]; // file array + TSttLvlArray lvlArr[1]; // level array +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_FILE_SET2_H*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c new file mode 100644 index 0000000000000000000000000000000000000000..83ae8c24291542f179a53fe9d7e2215c8f4ca8ca --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFSetRW.h" + +// SFSetWriter ================================================== +struct SFSetWriter { + SFSetWriterConfig config[1]; + + SSkmInfo skmTb[1]; + SSkmInfo skmRow[1]; + uint8_t *bufArr[10]; + + struct { + TABLEID tbid[1]; + } ctx[1]; + + // writer + SBlockData blockData[2]; + int32_t blockDataIdx; + SDataFileWriter *dataWriter; + SSttFileWriter *sttWriter; +}; + +static int32_t tsdbFSetWriteTableDataBegin(SFSetWriter *writer, const TABLEID *tbid) { + int32_t code = 0; + int32_t lino = 0; + + writer->ctx->tbid->suid = tbid->suid; + writer->ctx->tbid->uid = tbid->uid; + + code = tsdbUpdateSkmTb(writer->config->tsdb, writer->ctx->tbid, writer->skmTb); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->blockDataIdx = 0; + for (int32_t i = 0; i < ARRAY_SIZE(writer->blockData); i++) { + code = tBlockDataInit(&writer->blockData[i], writer->ctx->tbid, writer->skmTb->pTSchema, NULL, 0); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbFSetWriteTableDataEnd(SFSetWriter *writer) { + if (writer->ctx->tbid->uid == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + int32_t cidx = writer->blockDataIdx; + int32_t pidx = ((cidx + 1) & 1); + int32_t numRow = ((writer->blockData[pidx].nRow + writer->blockData[cidx].nRow) >> 1); + + if (writer->blockData[pidx].nRow > 0 && numRow >= writer->config->minRow) { + ASSERT(writer->blockData[pidx].nRow == writer->config->maxRow); + + SRowInfo row = { + .suid = writer->ctx->tbid->suid, + .uid = writer->ctx->tbid->uid, + .row = tsdbRowFromBlockData(writer->blockData + pidx, 0), + }; + + for (int32_t i = 0; i < numRow; i++) { + row.row.iRow = i; + + code = tsdbDataFileWriteRow(writer->dataWriter, &row); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDataFileFlush(writer->dataWriter); + TSDB_CHECK_CODE(code, lino, _exit); + + for (int32_t i = numRow; i < writer->blockData[pidx].nRow; i++) { + row.row.iRow = i; + code = tsdbDataFileWriteRow(writer->dataWriter, &row); + TSDB_CHECK_CODE(code, lino, _exit); + } + + row.row = tsdbRowFromBlockData(writer->blockData + cidx, 0); + for (int32_t i = 0; i < writer->blockData[cidx].nRow; i++) { + row.row.iRow = i; + code = tsdbDataFileWriteRow(writer->dataWriter, &row); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + // pidx + if (writer->blockData[pidx].nRow > 0) { + code = tsdbDataFileWriteBlockData(writer->dataWriter, &writer->blockData[pidx]); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // cidx + if (writer->blockData[cidx].nRow < writer->config->minRow) { + code = tsdbSttFileWriteBlockData(writer->sttWriter, &writer->blockData[cidx]); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDataFileWriteBlockData(writer->dataWriter, &writer->blockData[cidx]); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + for (int32_t i = 0; i < ARRAY_SIZE(writer->blockData); i++) { + tBlockDataReset(&writer->blockData[i]); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) { + int32_t code = 0; + int32_t lino = 0; + + writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); + if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->config[0] = config[0]; + + // data writer + if (!config->toSttOnly) { + SDataFileWriterConfig dataWriterConfig = { + .tsdb = config->tsdb, + .cmprAlg = config->cmprAlg, + .maxRow = config->maxRow, + .szPage = config->szPage, + .fid = config->fid, + .cid = config->cid, + .did = config->did, + .compactVersion = config->compactVersion, + .skmTb = writer[0]->skmTb, + .skmRow = writer[0]->skmRow, + .bufArr = writer[0]->bufArr, + }; + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) { + dataWriterConfig.files[ftype].exist = config->files[ftype].exist; + dataWriterConfig.files[ftype].file = config->files[ftype].file; + } + + code = tsdbDataFileWriterOpen(&dataWriterConfig, &writer[0]->dataWriter); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // stt writer + SSttFileWriterConfig sttWriterConfig = { + .tsdb = config->tsdb, + .maxRow = config->maxRow, + .szPage = config->szPage, + .cmprAlg = config->cmprAlg, + .compactVersion = config->compactVersion, + .did = config->did, + .fid = config->fid, + .cid = config->cid, + .level = config->level, + .skmTb = writer[0]->skmTb, + .skmRow = writer[0]->skmRow, + .bufArr = writer[0]->bufArr, + }; + code = tsdbSttFileWriterOpen(&sttWriterConfig, &writer[0]->sttWriter); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFSetWriterClose(SFSetWriter **writer, bool abort, TFileOpArray *fopArr) { + if (writer[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + STsdb *tsdb = writer[0]->config->tsdb; + + // end + if (!writer[0]->config->toSttOnly) { + code = tsdbFSetWriteTableDataEnd(writer[0]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriterClose(&writer[0]->dataWriter, abort, fopArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbSttFileWriterClose(&writer[0]->sttWriter, abort, fopArr); + TSDB_CHECK_CODE(code, lino, _exit); + + // free + for (int32_t i = 0; i < ARRAY_SIZE(writer[0]->blockData); i++) { + tBlockDataDestroy(&writer[0]->blockData[i]); + } + for (int32_t i = 0; i < ARRAY_SIZE(writer[0]->bufArr); i++) { + tFree(writer[0]->bufArr[i]); + } + tDestroyTSchema(writer[0]->skmRow->pTSchema); + tDestroyTSchema(writer[0]->skmTb->pTSchema); + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFSetWriteRow(SFSetWriter *writer, SRowInfo *row) { + int32_t code = 0; + int32_t lino = 0; + + if (writer->config->toSttOnly) { + code = tsdbSttFileWriteRow(writer->sttWriter, row); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + if (writer->ctx->tbid->uid != row->uid) { + code = tsdbFSetWriteTableDataEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFSetWriteTableDataBegin(writer, (TABLEID *)row); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (row->row.type == TSDBROW_ROW_FMT) { + code = tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, TSDBROW_SVERSION(&row->row), writer->skmRow); + TSDB_CHECK_CODE(code, lino, _exit); + } + + TSDBKEY key = TSDBROW_KEY(&row->row); + if (key.version <= writer->config->compactVersion // + && writer->blockData[writer->blockDataIdx].nRow > 0 // + && key.ts == writer->blockData[writer->blockDataIdx].aTSKEY[writer->blockData[writer->blockDataIdx].nRow - 1]) { + code = tBlockDataUpdateRow(&writer->blockData[writer->blockDataIdx], &row->row, writer->skmRow->pTSchema); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + if (writer->blockData[writer->blockDataIdx].nRow >= writer->config->maxRow) { + int32_t idx = ((writer->blockDataIdx + 1) & 1); + if (writer->blockData[idx].nRow >= writer->config->maxRow) { + code = tsdbDataFileWriteBlockData(writer->dataWriter, &writer->blockData[idx]); + TSDB_CHECK_CODE(code, lino, _exit); + + tBlockDataClear(&writer->blockData[idx]); + } + writer->blockDataIdx = idx; + } + + code = + tBlockDataAppendRow(&writer->blockData[writer->blockDataIdx], &row->row, writer->skmRow->pTSchema, row->uid); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFSetWriteTombRecord(SFSetWriter *writer, const STombRecord *tombRecord) { + int32_t code = 0; + int32_t lino = 0; + + if (writer->config->toSttOnly || tsdbSttFileWriterIsOpened(writer->sttWriter)) { + code = tsdbSttFileWriteTombRecord(writer->sttWriter, tombRecord); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDataFileWriteTombRecord(writer->dataWriter, tombRecord); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.h b/source/dnode/vnode/src/tsdb/tsdbFSetRW.h new file mode 100644 index 0000000000000000000000000000000000000000..b5710407cfe40e28736c2949d3f3421e131c6624 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDataFileRW.h" +#include "tsdbSttFileRW.h" + +#ifndef _TSDB_FSET_RW_H +#define _TSDB_FSET_RW_H + +#ifdef __cplusplus +extern "C" { +#endif + +// +typedef struct SFSetWriter SFSetWriter; +typedef struct { + STsdb *tsdb; + bool toSttOnly; + int64_t compactVersion; + int32_t minRow; + int32_t maxRow; + int32_t szPage; + int8_t cmprAlg; + int32_t fid; + int64_t cid; + SDiskID did; + int32_t level; + struct { + bool exist; + STFile file; + } files[TSDB_FTYPE_MAX]; +} SFSetWriterConfig; + +int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer); +int32_t tsdbFSetWriterClose(SFSetWriter **writer, bool abort, TFileOpArray *fopArr); +int32_t tsdbFSetWriteRow(SFSetWriter *writer, SRowInfo *row); +int32_t tsdbFSetWriteTombRecord(SFSetWriter *writer, const STombRecord *tombRecord); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_FSET_RW_H*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index d91475376b240812b8942227fa85e37cb5671dec..62b37cd0a66a9a9d2f1515a17d1ce8f6f11cbdde 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -14,6 +14,7 @@ */ #include "tsdb.h" +#include "vnd.h" int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile) { int32_t n = 0; @@ -111,7 +112,10 @@ static char* getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t p += titoa(TD_VID(pTsdb->pVnode), 10, p); *(p++) = 'f'; - p += titoa(fid, 10, p); + if (fid < 0) { + *(p++) = '-'; + } + p += titoa((fid < 0) ? -fid : fid, 10, p); memcpy(p, "ver", 3); p += 3; @@ -282,8 +286,13 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { // SDelFile =============================================== void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]) { - snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%dver%" PRId64 "%s", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), - TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), pFile->commitID, ".del"); + int32_t offset = 0; + SVnode *pVnode = pTsdb->pVnode; + + vnodeGetPrimaryDir(pTsdb->path, pVnode->diskPrimary, pVnode->pTfs, fname, TSDB_FILENAME_LEN); + offset = strlen(fname); + snprintf((char *)fname + offset, TSDB_FILENAME_LEN - offset - 1, "%sv%dver%" PRId64 ".del", TD_DIRSEP, + TD_VID(pTsdb->pVnode), pFile->commitID); } int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c new file mode 100644 index 0000000000000000000000000000000000000000..be021169cd890676e8a90bf0588d8617b97343f2 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFile2.h" + +// to_json +static int32_t head_to_json(const STFile *file, cJSON *json); +static int32_t data_to_json(const STFile *file, cJSON *json); +static int32_t sma_to_json(const STFile *file, cJSON *json); +static int32_t tomb_to_json(const STFile *file, cJSON *json); +static int32_t stt_to_json(const STFile *file, cJSON *json); + +// from_json +static int32_t head_from_json(const cJSON *json, STFile *file); +static int32_t data_from_json(const cJSON *json, STFile *file); +static int32_t sma_from_json(const cJSON *json, STFile *file); +static int32_t tomb_from_json(const cJSON *json, STFile *file); +static int32_t stt_from_json(const cJSON *json, STFile *file); + +static const struct { + const char *suffix; + int32_t (*to_json)(const STFile *file, cJSON *json); + int32_t (*from_json)(const cJSON *json, STFile *file); +} g_tfile_info[] = { + [TSDB_FTYPE_HEAD] = {"head", head_to_json, head_from_json}, + [TSDB_FTYPE_DATA] = {"data", data_to_json, data_from_json}, + [TSDB_FTYPE_SMA] = {"sma", sma_to_json, sma_from_json}, + [TSDB_FTYPE_TOMB] = {"tomb", tomb_to_json, tomb_from_json}, + [TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json}, +}; + +static void remove_file(const char *fname) { + taosRemoveFile(fname); + tsdbInfo("file:%s is removed", fname); +} + +static int32_t tfile_to_json(const STFile *file, cJSON *json) { + /* did.level */ + if (cJSON_AddNumberToObject(json, "did.level", file->did.level) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + /* did.id */ + if (cJSON_AddNumberToObject(json, "did.id", file->did.id) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + /* fid */ + if (cJSON_AddNumberToObject(json, "fid", file->fid) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + /* cid */ + if (cJSON_AddNumberToObject(json, "cid", file->cid) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + /* size */ + if (cJSON_AddNumberToObject(json, "size", file->size) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return 0; +} + +static int32_t tfile_from_json(const cJSON *json, STFile *file) { + const cJSON *item; + + /* did.level */ + item = cJSON_GetObjectItem(json, "did.level"); + if (cJSON_IsNumber(item)) { + file->did.level = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + /* did.id */ + item = cJSON_GetObjectItem(json, "did.id"); + if (cJSON_IsNumber(item)) { + file->did.id = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + /* fid */ + item = cJSON_GetObjectItem(json, "fid"); + if (cJSON_IsNumber(item)) { + file->fid = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + /* cid */ + item = cJSON_GetObjectItem(json, "cid"); + if (cJSON_IsNumber(item)) { + file->cid = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + /* size */ + item = cJSON_GetObjectItem(json, "size"); + if (cJSON_IsNumber(item)) { + file->size = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + return 0; +} + +static int32_t head_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } +static int32_t data_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } +static int32_t sma_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } +static int32_t tomb_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } +static int32_t stt_to_json(const STFile *file, cJSON *json) { + int32_t code = tfile_to_json(file, json); + if (code) return code; + + /* lvl */ + if (cJSON_AddNumberToObject(json, "level", file->stt->level) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return 0; +} + +static int32_t head_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } +static int32_t data_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } +static int32_t sma_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } +static int32_t tomb_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } +static int32_t stt_from_json(const cJSON *json, STFile *file) { + int32_t code = tfile_from_json(json, file); + if (code) return code; + + const cJSON *item; + + /* lvl */ + item = cJSON_GetObjectItem(json, "level"); + if (cJSON_IsNumber(item)) { + file->stt->level = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + return 0; +} + +int32_t tsdbTFileToJson(const STFile *file, cJSON *json) { + if (file->type == TSDB_FTYPE_STT) { + return g_tfile_info[file->type].to_json(file, json); + } else { + cJSON *item = cJSON_AddObjectToObject(json, g_tfile_info[file->type].suffix); + if (item == NULL) return TSDB_CODE_OUT_OF_MEMORY; + return g_tfile_info[file->type].to_json(file, item); + } +} + +int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f) { + f[0] = (STFile){.type = ftype}; + + if (ftype == TSDB_FTYPE_STT) { + int32_t code = g_tfile_info[ftype].from_json(json, f); + if (code) return code; + } else { + const cJSON *item = cJSON_GetObjectItem(json, g_tfile_info[ftype].suffix); + if (cJSON_IsObject(item)) { + int32_t code = g_tfile_info[ftype].from_json(item, f); + if (code) return code; + } else { + return TSDB_CODE_NOT_FOUND; + } + } + + return 0; +} + +int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) { + fobj[0] = taosMemoryMalloc(sizeof(*fobj[0])); + if (!fobj[0]) return TSDB_CODE_OUT_OF_MEMORY; + + taosThreadMutexInit(&fobj[0]->mutex, NULL); + fobj[0]->f[0] = f[0]; + fobj[0]->state = TSDB_FSTATE_LIVE; + fobj[0]->ref = 1; + tsdbTFileName(pTsdb, f, fobj[0]->fname); + return 0; +} + +int32_t tsdbTFileObjRef(STFileObj *fobj) { + int32_t nRef; + taosThreadMutexLock(&fobj->mutex); + ASSERT(fobj->ref > 0 && fobj->state == TSDB_FSTATE_LIVE); + nRef = ++fobj->ref; + taosThreadMutexUnlock(&fobj->mutex); + tsdbTrace("ref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); + return 0; +} + +int32_t tsdbTFileObjUnref(STFileObj *fobj) { + taosThreadMutexLock(&fobj->mutex); + int32_t nRef = --fobj->ref; + taosThreadMutexUnlock(&fobj->mutex); + ASSERT(nRef >= 0); + tsdbTrace("unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); + if (nRef == 0) { + if (fobj->state == TSDB_FSTATE_DEAD) { + remove_file(fobj->fname); + } + taosMemoryFree(fobj); + } + + return 0; +} + +int32_t tsdbTFileObjRemove(STFileObj *fobj) { + taosThreadMutexLock(&fobj->mutex); + ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0); + fobj->state = TSDB_FSTATE_DEAD; + int32_t nRef = --fobj->ref; + taosThreadMutexUnlock(&fobj->mutex); + tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); + if (nRef == 0) { + remove_file(fobj->fname); + taosMemoryFree(fobj); + } + return 0; +} + +int32_t tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]) { + SVnode *pVnode = pTsdb->pVnode; + STfs *pTfs = pVnode->pTfs; + + if (pTfs) { + snprintf(fname, // + TSDB_FILENAME_LEN, // + "%s%s%s%sv%df%dver%" PRId64 ".%s", // + tfsGetDiskPath(pTfs, f->did), // + TD_DIRSEP, // + pTsdb->path, // + TD_DIRSEP, // + TD_VID(pVnode), // + f->fid, // + f->cid, // + g_tfile_info[f->type].suffix); + } else { + snprintf(fname, // + TSDB_FILENAME_LEN, // + "%s%sv%df%dver%" PRId64 ".%s", // + pTsdb->path, // + TD_DIRSEP, // + TD_VID(pVnode), // + f->fid, // + f->cid, // + g_tfile_info[f->type].suffix); + } + return 0; +} + +bool tsdbIsSameTFile(const STFile *f1, const STFile *f2) { + if (f1->type != f2->type) return false; + if (f1->did.level != f2->did.level) return false; + if (f1->did.id != f2->did.id) return false; + if (f1->fid != f2->fid) return false; + if (f1->cid != f2->cid) return false; + return true; +} + +bool tsdbIsTFileChanged(const STFile *f1, const STFile *f2) { + if (f1->size != f2->size) return true; + // if (f1->type == TSDB_FTYPE_STT && f1->stt->nseg != f2->stt->nseg) return true; + return false; +} + +int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2) { + if (fobj1[0]->f->cid < fobj2[0]->f->cid) { + return -1; + } else if (fobj1[0]->f->cid > fobj2[0]->f->cid) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.h b/source/dnode/vnode/src/tsdb/tsdbFile2.h new file mode 100644 index 0000000000000000000000000000000000000000..11d08e45e667ddea6f8150239b472c86426f841a --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDef.h" + +#ifndef _TSDB_FILE_H +#define _TSDB_FILE_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct STFile STFile; +typedef struct STFileObj STFileObj; + +typedef enum { + TSDB_FTYPE_HEAD = 0, // .head + TSDB_FTYPE_DATA, // .data + TSDB_FTYPE_SMA, // .sma + TSDB_FTYPE_TOMB, // .tomb + TSDB_FTYPE_STT = TSDB_FTYPE_TOMB + 2, // .stt +} tsdb_ftype_t; + +enum { + TSDB_FSTATE_LIVE = 1, + TSDB_FSTATE_DEAD, +}; + +#define TSDB_FTYPE_MIN TSDB_FTYPE_HEAD +#define TSDB_FTYPE_MAX (TSDB_FTYPE_TOMB + 1) + +// STFile +int32_t tsdbTFileToJson(const STFile *f, cJSON *json); +int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f); +int32_t tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]); +bool tsdbIsSameTFile(const STFile *f1, const STFile *f2); +bool tsdbIsTFileChanged(const STFile *f1, const STFile *f2); + +// STFileObj +int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj); +int32_t tsdbTFileObjRef(STFileObj *fobj); +int32_t tsdbTFileObjUnref(STFileObj *fobj); +int32_t tsdbTFileObjRemove(STFileObj *fobj); +int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2); + +struct STFile { + tsdb_ftype_t type; + SDiskID did; // disk id + int32_t fid; // file id + int64_t cid; // commit id + int64_t size; + union { + struct { + int32_t level; + } stt[1]; + }; +}; + +struct STFileObj { + TdThreadMutex mutex; + STFile f[1]; + int32_t state; + int32_t ref; + char fname[TSDB_FILENAME_LEN]; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_FILE_H*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbIter.c b/source/dnode/vnode/src/tsdb/tsdbIter.c new file mode 100644 index 0000000000000000000000000000000000000000..9780cc6be63661869576bdfe7f1af544c59f1d59 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbIter.c @@ -0,0 +1,780 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbIter.h" + +// STsdbIter ================ +struct STsdbIter { + EIterType type; + bool noMoreData; + bool filterByVersion; + int64_t range[2]; + union { + SRowInfo row[1]; + STombRecord record[1]; + }; + SRBTreeNode node[1]; + union { + struct { + SSttFileReader *reader; + const TSttBlkArray *sttBlkArray; + int32_t sttBlkArrayIdx; + SBlockData blockData[1]; + int32_t blockDataIdx; + } sttData[1]; + struct { + SDataFileReader *reader; + const TBrinBlkArray *brinBlkArray; + int32_t brinBlkArrayIdx; + SBrinBlock brinBlock[1]; + int32_t brinBlockIdx; + SBlockData blockData[1]; + int32_t blockDataIdx; + } dataData[1]; + struct { + SMemTable *memt; + TSDBKEY from[1]; + SRBTreeIter iter[1]; + STbData *tbData; + STbDataIter tbIter[1]; + } memtData[1]; + struct { + SSttFileReader *reader; + const TTombBlkArray *tombBlkArray; + int32_t tombBlkArrayIdx; + STombBlock tombBlock[1]; + int32_t tombBlockIdx; + } sttTomb[1]; + struct { + SDataFileReader *reader; + const TTombBlkArray *tombBlkArray; + int32_t tombBlkArrayIdx; + STombBlock tombBlock[1]; + int32_t tombBlockIdx; + } dataTomb[1]; + struct { + SMemTable *memt; + SRBTreeIter rbtIter[1]; + STbData *tbData; + SDelData *delData; + } memtTomb[1]; + }; +}; + +static int32_t tsdbSttIterNext(STsdbIter *iter, const TABLEID *tbid) { + while (!iter->noMoreData) { + for (; iter->sttData->blockDataIdx < iter->sttData->blockData->nRow; iter->sttData->blockDataIdx++) { + int64_t version = iter->sttData->blockData->aVersion[iter->sttData->blockDataIdx]; + + if (iter->filterByVersion && (version < iter->range[0] || version > iter->range[1])) { + continue; + } + + iter->row->suid = iter->sttData->blockData->suid; + iter->row->uid = iter->sttData->blockData->uid ? iter->sttData->blockData->uid + : iter->sttData->blockData->aUid[iter->sttData->blockDataIdx]; + + if (tbid && iter->row->suid == tbid->suid && iter->row->uid == tbid->uid) { + continue; + } + + iter->row->row = tsdbRowFromBlockData(iter->sttData->blockData, iter->sttData->blockDataIdx); + iter->sttData->blockDataIdx++; + goto _exit; + } + + if (iter->sttData->sttBlkArrayIdx >= TARRAY2_SIZE(iter->sttData->sttBlkArray)) { + iter->noMoreData = true; + break; + } + + for (; iter->sttData->sttBlkArrayIdx < TARRAY2_SIZE(iter->sttData->sttBlkArray); iter->sttData->sttBlkArrayIdx++) { + const SSttBlk *sttBlk = TARRAY2_GET_PTR(iter->sttData->sttBlkArray, iter->sttData->sttBlkArrayIdx); + + if (iter->filterByVersion && (sttBlk->maxVer < iter->range[0] || sttBlk->minVer > iter->range[1])) { + continue; + } + + if (tbid && tbid->suid == sttBlk->suid && tbid->uid == sttBlk->minUid && tbid->uid == sttBlk->maxUid) { + continue; + } + + int32_t code = tsdbSttFileReadBlockData(iter->sttData->reader, sttBlk, iter->sttData->blockData); + if (code) return code; + + iter->sttData->blockDataIdx = 0; + iter->sttData->sttBlkArrayIdx++; + break; + } + } + +_exit: + return 0; +} + +static int32_t tsdbDataIterNext(STsdbIter *iter, const TABLEID *tbid) { + int32_t code; + + while (!iter->noMoreData) { + for (;;) { + // SBlockData + for (; iter->dataData->blockDataIdx < iter->dataData->blockData->nRow; iter->dataData->blockDataIdx++) { + int64_t version = iter->dataData->blockData->aVersion[iter->dataData->blockDataIdx]; + if (iter->filterByVersion && (version < iter->range[0] || version > iter->range[1])) { + continue; + } + + if (tbid && tbid->suid == iter->dataData->blockData->suid && tbid->uid == iter->dataData->blockData->uid) { + iter->dataData->blockDataIdx = iter->dataData->blockData->nRow; + break; + } + + iter->row->row = tsdbRowFromBlockData(iter->dataData->blockData, iter->dataData->blockDataIdx); + iter->dataData->blockDataIdx++; + goto _exit; + } + + // SBrinBlock + if (iter->dataData->brinBlockIdx >= BRIN_BLOCK_SIZE(iter->dataData->brinBlock)) { + break; + } + + for (; iter->dataData->brinBlockIdx < BRIN_BLOCK_SIZE(iter->dataData->brinBlock); + iter->dataData->brinBlockIdx++) { + SBrinRecord record[1]; + tBrinBlockGet(iter->dataData->brinBlock, iter->dataData->brinBlockIdx, record); + + if (iter->filterByVersion && (record->maxVer < iter->range[0] || record->minVer > iter->range[1])) { + continue; + } + + if (tbid && tbid->suid == record->suid && tbid->uid == record->uid) { + continue; + } + + iter->row->suid = record->suid; + iter->row->uid = record->uid; + + code = tsdbDataFileReadBlockData(iter->dataData->reader, record, iter->dataData->blockData); + if (code) return code; + + iter->dataData->blockDataIdx = 0; + iter->dataData->brinBlockIdx++; + break; + } + } + + if (iter->dataData->brinBlkArrayIdx >= TARRAY2_SIZE(iter->dataData->brinBlkArray)) { + iter->noMoreData = true; + break; + } + + for (; iter->dataData->brinBlkArrayIdx < TARRAY2_SIZE(iter->dataData->brinBlkArray); + iter->dataData->brinBlkArrayIdx++) { + const SBrinBlk *brinBlk = TARRAY2_GET_PTR(iter->dataData->brinBlkArray, iter->dataData->brinBlkArrayIdx); + + if (iter->filterByVersion && (brinBlk->maxVer < iter->range[0] || brinBlk->minVer > iter->range[1])) { + continue; + } + + if (tbid && tbid->uid == brinBlk->minTbid.uid && tbid->uid == brinBlk->maxTbid.uid) { + continue; + } + + code = tsdbDataFileReadBrinBlock(iter->dataData->reader, brinBlk, iter->dataData->brinBlock); + if (code) return code; + + iter->dataData->brinBlockIdx = 0; + iter->dataData->brinBlkArrayIdx++; + break; + } + } + +_exit: + return 0; +} + +static int32_t tsdbMemTableIterNext(STsdbIter *iter, const TABLEID *tbid) { + SRBTreeNode *node; + + while (!iter->noMoreData) { + for (TSDBROW *row; iter->memtData->tbData && (row = tsdbTbDataIterGet(iter->memtData->tbIter));) { + if (tbid && tbid->suid == iter->memtData->tbData->suid && tbid->uid == iter->memtData->tbData->uid) { + iter->memtData->tbData = NULL; + break; + } + + if (iter->filterByVersion) { + int64_t version = TSDBROW_VERSION(row); + if (version < iter->range[0] || version > iter->range[1]) { + continue; + } + } + + iter->row->row = row[0]; + + tsdbTbDataIterNext(iter->memtData->tbIter); + goto _exit; + } + + for (;;) { + node = tRBTreeIterNext(iter->memtData->iter); + if (!node) { + iter->noMoreData = true; + goto _exit; + } + + iter->memtData->tbData = TCONTAINER_OF(node, STbData, rbtn); + if (tbid && tbid->suid == iter->memtData->tbData->suid && tbid->uid == iter->memtData->tbData->uid) { + continue; + } else { + iter->row->suid = iter->memtData->tbData->suid; + iter->row->uid = iter->memtData->tbData->uid; + tsdbTbDataIterOpen(iter->memtData->tbData, iter->memtData->from, 0, iter->memtData->tbIter); + break; + } + } + } + +_exit: + return 0; +} + +static int32_t tsdbDataTombIterNext(STsdbIter *iter, const TABLEID *tbid) { + while (!iter->noMoreData) { + for (; iter->dataTomb->tombBlockIdx < TOMB_BLOCK_SIZE(iter->dataTomb->tombBlock); iter->dataTomb->tombBlockIdx++) { + iter->record->suid = TARRAY2_GET(iter->dataTomb->tombBlock->suid, iter->dataTomb->tombBlockIdx); + iter->record->uid = TARRAY2_GET(iter->dataTomb->tombBlock->uid, iter->dataTomb->tombBlockIdx); + iter->record->version = TARRAY2_GET(iter->dataTomb->tombBlock->version, iter->dataTomb->tombBlockIdx); + + if (iter->filterByVersion && (iter->record->version < iter->range[0] || iter->record->version > iter->range[1])) { + continue; + } + + if (tbid && iter->record->suid == tbid->suid && iter->record->uid == tbid->uid) { + continue; + } + + iter->record->skey = TARRAY2_GET(iter->dataTomb->tombBlock->skey, iter->dataTomb->tombBlockIdx); + iter->record->ekey = TARRAY2_GET(iter->dataTomb->tombBlock->ekey, iter->dataTomb->tombBlockIdx); + iter->dataTomb->tombBlockIdx++; + goto _exit; + } + + if (iter->dataTomb->tombBlkArrayIdx >= TARRAY2_SIZE(iter->dataTomb->tombBlkArray)) { + iter->noMoreData = true; + goto _exit; + } + + for (; iter->dataTomb->tombBlkArrayIdx < TARRAY2_SIZE(iter->dataTomb->tombBlkArray); + iter->dataTomb->tombBlkArrayIdx++) { + const STombBlk *tombBlk = TARRAY2_GET_PTR(iter->dataTomb->tombBlkArray, iter->dataTomb->tombBlkArrayIdx); + + if (tbid && tbid->suid == tombBlk->minTbid.suid && tbid->uid == tombBlk->minTbid.uid && + tbid->suid == tombBlk->maxTbid.suid && tbid->uid == tombBlk->maxTbid.uid) { + continue; + } + + int32_t code = tsdbDataFileReadTombBlock(iter->dataTomb->reader, tombBlk, iter->dataTomb->tombBlock); + if (code) return code; + + iter->dataTomb->tombBlockIdx = 0; + iter->dataTomb->tombBlkArrayIdx++; + break; + } + } + +_exit: + return 0; +} + +static int32_t tsdbMemTombIterNext(STsdbIter *iter, const TABLEID *tbid) { + while (!iter->noMoreData) { + for (; iter->memtTomb->delData;) { + if (tbid && tbid->uid == iter->memtTomb->tbData->uid) { + iter->memtTomb->delData = NULL; + break; + } + + if (iter->filterByVersion && + (iter->memtTomb->delData->version < iter->range[0] || iter->memtTomb->delData->version > iter->range[1])) { + continue; + } + + iter->record->suid = iter->memtTomb->tbData->suid; + iter->record->uid = iter->memtTomb->tbData->uid; + iter->record->version = iter->memtTomb->delData->version; + iter->record->skey = iter->memtTomb->delData->sKey; + iter->record->ekey = iter->memtTomb->delData->eKey; + + iter->memtTomb->delData = iter->memtTomb->delData->pNext; + goto _exit; + } + + for (;;) { + SRBTreeNode *node = tRBTreeIterNext(iter->memtTomb->rbtIter); + if (node == NULL) { + iter->noMoreData = true; + goto _exit; + } + + iter->memtTomb->tbData = TCONTAINER_OF(node, STbData, rbtn); + if (tbid && tbid->uid == iter->memtTomb->tbData->uid) { + continue; + } else { + iter->memtTomb->delData = iter->memtTomb->tbData->pHead; + break; + } + } + } + +_exit: + return 0; +} + +static int32_t tsdbSttIterOpen(STsdbIter *iter) { + int32_t code; + + code = tsdbSttFileReadSttBlk(iter->sttData->reader, &iter->sttData->sttBlkArray); + if (code) return code; + + if (TARRAY2_SIZE(iter->sttData->sttBlkArray) == 0) { + iter->noMoreData = true; + return 0; + } + + iter->sttData->sttBlkArrayIdx = 0; + tBlockDataCreate(iter->sttData->blockData); + iter->sttData->blockDataIdx = 0; + + return tsdbSttIterNext(iter, NULL); +} + +static int32_t tsdbDataIterOpen(STsdbIter *iter) { + int32_t code; + + // SBrinBlk + code = tsdbDataFileReadBrinBlk(iter->dataData->reader, &iter->dataData->brinBlkArray); + if (code) return code; + + if (TARRAY2_SIZE(iter->dataData->brinBlkArray) == 0) { + iter->noMoreData = true; + return 0; + } + + iter->dataData->brinBlkArrayIdx = 0; + + // SBrinBlock + tBrinBlockInit(iter->dataData->brinBlock); + iter->dataData->brinBlockIdx = 0; + + // SBlockData + tBlockDataCreate(iter->dataData->blockData); + iter->dataData->blockDataIdx = 0; + + return tsdbDataIterNext(iter, NULL); +} + +static int32_t tsdbMemTableIterOpen(STsdbIter *iter) { + if (iter->memtData->memt->nRow == 0) { + iter->noMoreData = true; + return 0; + } + + iter->memtData->iter[0] = tRBTreeIterCreate(iter->memtData->memt->tbDataTree, 1); + return tsdbMemTableIterNext(iter, NULL); +} + +static int32_t tsdbSttIterClose(STsdbIter *iter) { + tBlockDataDestroy(iter->sttData->blockData); + return 0; +} + +static int32_t tsdbDataTombIterOpen(STsdbIter *iter) { + int32_t code; + + code = tsdbDataFileReadTombBlk(iter->dataTomb->reader, &iter->dataTomb->tombBlkArray); + if (code) return code; + + if (TARRAY2_SIZE(iter->dataTomb->tombBlkArray) == 0) { + iter->noMoreData = true; + return 0; + } + iter->dataTomb->tombBlkArrayIdx = 0; + + tTombBlockInit(iter->dataTomb->tombBlock); + iter->dataTomb->tombBlockIdx = 0; + + return tsdbDataTombIterNext(iter, NULL); +} + +static int32_t tsdbMemTombIterOpen(STsdbIter *iter) { + int32_t code; + + if (iter->memtTomb->memt->nDel == 0) { + iter->noMoreData = true; + return 0; + } + + iter->memtTomb->rbtIter[0] = tRBTreeIterCreate(iter->memtTomb->memt->tbDataTree, 1); + return tsdbMemTombIterNext(iter, NULL); +} + +static int32_t tsdbDataIterClose(STsdbIter *iter) { + tBrinBlockDestroy(iter->dataData->brinBlock); + tBlockDataDestroy(iter->dataData->blockData); + return 0; +} + +static int32_t tsdbMemTableIterClose(STsdbIter *iter) { return 0; } + +static int32_t tsdbSttTombIterNext(STsdbIter *iter, const TABLEID *tbid) { + while (!iter->noMoreData) { + for (; iter->sttTomb->tombBlockIdx < TOMB_BLOCK_SIZE(iter->sttTomb->tombBlock); iter->sttTomb->tombBlockIdx++) { + iter->record->suid = TARRAY2_GET(iter->sttTomb->tombBlock->suid, iter->sttTomb->tombBlockIdx); + iter->record->uid = TARRAY2_GET(iter->sttTomb->tombBlock->uid, iter->sttTomb->tombBlockIdx); + iter->record->version = TARRAY2_GET(iter->sttTomb->tombBlock->version, iter->sttTomb->tombBlockIdx); + + if (iter->filterByVersion && (iter->record->version < iter->range[0] || iter->record->version > iter->range[1])) { + continue; + } + + if (tbid && iter->record->suid == tbid->suid && iter->record->uid == tbid->uid) { + continue; + } + + iter->record->skey = TARRAY2_GET(iter->sttTomb->tombBlock->skey, iter->sttTomb->tombBlockIdx); + iter->record->ekey = TARRAY2_GET(iter->sttTomb->tombBlock->ekey, iter->sttTomb->tombBlockIdx); + iter->sttTomb->tombBlockIdx++; + goto _exit; + } + + if (iter->sttTomb->tombBlkArrayIdx >= TARRAY2_SIZE(iter->sttTomb->tombBlkArray)) { + iter->noMoreData = true; + goto _exit; + } + + for (; iter->sttTomb->tombBlkArrayIdx < TARRAY2_SIZE(iter->sttTomb->tombBlkArray); + iter->sttTomb->tombBlkArrayIdx++) { + const STombBlk *tombBlk = TARRAY2_GET_PTR(iter->sttTomb->tombBlkArray, iter->sttTomb->tombBlkArrayIdx); + + if (iter->filterByVersion && (tombBlk->maxVer < iter->range[0] || tombBlk->minVer > iter->range[1])) { + continue; + } + + if (tbid && tbid->suid == tombBlk->minTbid.suid && tbid->uid == tombBlk->minTbid.uid && + tbid->suid == tombBlk->maxTbid.suid && tbid->uid == tombBlk->maxTbid.uid) { + continue; + } + + int32_t code = tsdbSttFileReadTombBlock(iter->sttTomb->reader, tombBlk, iter->sttTomb->tombBlock); + if (code) return code; + + iter->sttTomb->tombBlockIdx = 0; + iter->sttTomb->tombBlkArrayIdx++; + break; + } + } + +_exit: + return 0; +} + +static int32_t tsdbSttTombIterOpen(STsdbIter *iter) { + int32_t code; + + code = tsdbSttFileReadTombBlk(iter->sttTomb->reader, &iter->sttTomb->tombBlkArray); + if (code) return code; + + if (TARRAY2_SIZE(iter->sttTomb->tombBlkArray) == 0) { + iter->noMoreData = true; + return 0; + } + + iter->sttTomb->tombBlkArrayIdx = 0; + tTombBlockInit(iter->sttTomb->tombBlock); + iter->sttTomb->tombBlockIdx = 0; + + return tsdbSttTombIterNext(iter, NULL); +} + +int32_t tsdbIterOpen(const STsdbIterConfig *config, STsdbIter **iter) { + int32_t code; + + iter[0] = taosMemoryCalloc(1, sizeof(*iter[0])); + if (iter[0] == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + iter[0]->type = config->type; + iter[0]->noMoreData = false; + iter[0]->filterByVersion = config->filterByVersion; + if (iter[0]->filterByVersion) { + iter[0]->range[0] = config->verRange[0]; + iter[0]->range[1] = config->verRange[1]; + } + + switch (config->type) { + case TSDB_ITER_TYPE_STT: + iter[0]->sttData->reader = config->sttReader; + code = tsdbSttIterOpen(iter[0]); + break; + case TSDB_ITER_TYPE_DATA: + iter[0]->dataData->reader = config->dataReader; + code = tsdbDataIterOpen(iter[0]); + break; + case TSDB_ITER_TYPE_MEMT: + iter[0]->memtData->memt = config->memt; + iter[0]->memtData->from[0] = config->from[0]; + code = tsdbMemTableIterOpen(iter[0]); + break; + case TSDB_ITER_TYPE_STT_TOMB: + iter[0]->sttTomb->reader = config->sttReader; + code = tsdbSttTombIterOpen(iter[0]); + break; + case TSDB_ITER_TYPE_DATA_TOMB: + iter[0]->dataTomb->reader = config->dataReader; + code = tsdbDataTombIterOpen(iter[0]); + break; + case TSDB_ITER_TYPE_MEMT_TOMB: + iter[0]->memtTomb->memt = config->memt; + code = tsdbMemTombIterOpen(iter[0]); + break; + default: + code = TSDB_CODE_INVALID_PARA; + ASSERTS(false, "Not implemented"); + } + + if (code) { + taosMemoryFree(iter[0]); + iter[0] = NULL; + } + return code; +} + +static int32_t tsdbSttTombIterClose(STsdbIter *iter) { + tTombBlockDestroy(iter->sttTomb->tombBlock); + return 0; +} + +static int32_t tsdbDataTombIterClose(STsdbIter *iter) { + tTombBlockDestroy(iter->dataTomb->tombBlock); + return 0; +} + +int32_t tsdbIterClose(STsdbIter **iter) { + switch (iter[0]->type) { + case TSDB_ITER_TYPE_STT: + tsdbSttIterClose(iter[0]); + break; + case TSDB_ITER_TYPE_DATA: + tsdbDataIterClose(iter[0]); + break; + case TSDB_ITER_TYPE_MEMT: + tsdbMemTableIterClose(iter[0]); + break; + case TSDB_ITER_TYPE_STT_TOMB: + tsdbSttTombIterClose(iter[0]); + break; + case TSDB_ITER_TYPE_DATA_TOMB: + tsdbDataTombIterClose(iter[0]); + break; + case TSDB_ITER_TYPE_MEMT_TOMB: + break; + default: + ASSERT(false); + } + taosMemoryFree(iter[0]); + iter[0] = NULL; + return 0; +} + +int32_t tsdbIterNext(STsdbIter *iter) { + switch (iter->type) { + case TSDB_ITER_TYPE_STT: + return tsdbSttIterNext(iter, NULL); + case TSDB_ITER_TYPE_DATA: + return tsdbDataIterNext(iter, NULL); + case TSDB_ITER_TYPE_MEMT: + return tsdbMemTableIterNext(iter, NULL); + case TSDB_ITER_TYPE_STT_TOMB: + return tsdbSttTombIterNext(iter, NULL); + case TSDB_ITER_TYPE_DATA_TOMB: + return tsdbDataTombIterNext(iter, NULL); + case TSDB_ITER_TYPE_MEMT_TOMB: + return tsdbMemTombIterNext(iter, NULL); + default: + ASSERT(false); + } + return 0; +} + +static int32_t tsdbIterSkipTableData(STsdbIter *iter, const TABLEID *tbid) { + switch (iter->type) { + case TSDB_ITER_TYPE_STT: + return tsdbSttIterNext(iter, tbid); + case TSDB_ITER_TYPE_DATA: + return tsdbDataIterNext(iter, tbid); + case TSDB_ITER_TYPE_MEMT: + return tsdbMemTableIterNext(iter, tbid); + case TSDB_ITER_TYPE_STT_TOMB: + return tsdbSttTombIterNext(iter, tbid); + case TSDB_ITER_TYPE_DATA_TOMB: + return tsdbDataTombIterNext(iter, tbid); + case TSDB_ITER_TYPE_MEMT_TOMB: + return tsdbMemTombIterNext(iter, tbid); + default: + ASSERT(false); + } + return 0; +} + +static int32_t tsdbIterCmprFn(const SRBTreeNode *n1, const SRBTreeNode *n2) { + STsdbIter *iter1 = TCONTAINER_OF(n1, STsdbIter, node); + STsdbIter *iter2 = TCONTAINER_OF(n2, STsdbIter, node); + return tRowInfoCmprFn(&iter1->row, &iter2->row); +} + +static int32_t tsdbTombIterCmprFn(const SRBTreeNode *n1, const SRBTreeNode *n2) { + STsdbIter *iter1 = TCONTAINER_OF(n1, STsdbIter, node); + STsdbIter *iter2 = TCONTAINER_OF(n2, STsdbIter, node); + + if (iter1->record->suid < iter2->record->suid) { + return -1; + } else if (iter1->record->suid > iter2->record->suid) { + return 1; + } + + if (iter1->record->uid < iter2->record->uid) { + return -1; + } else if (iter1->record->uid > iter2->record->uid) { + return 1; + } + + if (iter1->record->version < iter2->record->version) { + return -1; + } else if (iter1->record->version > iter2->record->version) { + return 1; + } + + return 0; +} + +// SIterMerger ================ +struct SIterMerger { + bool isTomb; + STsdbIter *iter; + SRBTree iterTree[1]; +}; + +int32_t tsdbIterMergerOpen(const TTsdbIterArray *iterArray, SIterMerger **merger, bool isTomb) { + STsdbIter *iter; + SRBTreeNode *node; + + merger[0] = taosMemoryCalloc(1, sizeof(*merger[0])); + if (merger[0] == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + merger[0]->isTomb = isTomb; + if (isTomb) { + tRBTreeCreate(merger[0]->iterTree, tsdbTombIterCmprFn); + } else { + tRBTreeCreate(merger[0]->iterTree, tsdbIterCmprFn); + } + TARRAY2_FOREACH(iterArray, iter) { + if (iter->noMoreData) continue; + node = tRBTreePut(merger[0]->iterTree, iter->node); + ASSERT(node); + } + + return tsdbIterMergerNext(merger[0]); +} + +int32_t tsdbIterMergerClose(SIterMerger **merger) { + if (merger[0]) { + taosMemoryFree(merger[0]); + merger[0] = NULL; + } + return 0; +} + +int32_t tsdbIterMergerNext(SIterMerger *merger) { + int32_t code; + int32_t c; + SRBTreeNode *node; + + if (merger->iter) { + code = tsdbIterNext(merger->iter); + if (code) return code; + + if (merger->iter->noMoreData) { + merger->iter = NULL; + } else if ((node = tRBTreeMin(merger->iterTree))) { + c = merger->iterTree->cmprFn(merger->iter->node, node); + ASSERT(c); + if (c > 0) { + node = tRBTreePut(merger->iterTree, merger->iter->node); + ASSERT(node); + merger->iter = NULL; + } + } + } + + if (merger->iter == NULL && (node = tRBTreeDropMin(merger->iterTree))) { + merger->iter = TCONTAINER_OF(node, STsdbIter, node); + } + + return 0; +} + +SRowInfo *tsdbIterMergerGetData(SIterMerger *merger) { + ASSERT(!merger->isTomb); + return merger->iter ? merger->iter->row : NULL; +} + +STombRecord *tsdbIterMergerGetTombRecord(SIterMerger *merger) { + ASSERT(merger->isTomb); + return merger->iter ? merger->iter->record : NULL; +} + +int32_t tsdbIterMergerSkipTableData(SIterMerger *merger, const TABLEID *tbid) { + int32_t code; + int32_t c; + SRBTreeNode *node; + + while (merger->iter && tbid->suid == merger->iter->row->suid && tbid->uid == merger->iter->row->uid) { + int32_t code = tsdbIterSkipTableData(merger->iter, tbid); + if (code) return code; + + if (merger->iter->noMoreData) { + merger->iter = NULL; + } else if ((node = tRBTreeMin(merger->iterTree))) { + c = merger->iterTree->cmprFn(merger->iter->node, node); + ASSERT(c); + if (c > 0) { + node = tRBTreePut(merger->iterTree, merger->iter->node); + ASSERT(node); + merger->iter = NULL; + } + } + + if (!merger->iter && (node = tRBTreeDropMin(merger->iterTree))) { + merger->iter = TCONTAINER_OF(node, STsdbIter, node); + } + } + + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbIter.h b/source/dnode/vnode/src/tsdb/tsdbIter.h new file mode 100644 index 0000000000000000000000000000000000000000..367901bd848df5b752a3439249dba95f76369b45 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbIter.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "trbtree.h" +#include "tsdbDataFileRW.h" +#include "tsdbDef.h" +#include "tsdbSttFileRW.h" + +#ifndef _TSDB_ITER_H_ +#define _TSDB_ITER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SIterMerger SIterMerger; +typedef struct STsdbIter STsdbIter; +typedef TARRAY2(STsdbIter *) TTsdbIterArray; + +typedef enum { + TSDB_ITER_TYPE_STT = 1, + TSDB_ITER_TYPE_DATA, + TSDB_ITER_TYPE_MEMT, + TSDB_ITER_TYPE_STT_TOMB, + TSDB_ITER_TYPE_DATA_TOMB, + TSDB_ITER_TYPE_MEMT_TOMB, +} EIterType; + +typedef struct { + EIterType type; + union { + SSttFileReader *sttReader; // TSDB_ITER_TYPE_STT || TSDB_ITER_TYPE_STT_TOMB + SDataFileReader *dataReader; // TSDB_ITER_TYPE_DATA || TSDB_ITER_TYPE_DATA_TOMB + struct { + SMemTable *memt; // TSDB_ITER_TYPE_MEMT_TOMB + TSDBKEY from[1]; + }; // TSDB_ITER_TYPE_MEMT + }; + bool filterByVersion; + int64_t verRange[2]; +} STsdbIterConfig; + +// STsdbIter =============== +int32_t tsdbIterOpen(const STsdbIterConfig *config, STsdbIter **iter); +int32_t tsdbIterClose(STsdbIter **iter); +int32_t tsdbIterNext(STsdbIter *iter); + +// SIterMerger =============== +int32_t tsdbIterMergerOpen(const TTsdbIterArray *iterArray, SIterMerger **merger, bool isTomb); +int32_t tsdbIterMergerClose(SIterMerger **merger); +int32_t tsdbIterMergerNext(SIterMerger *merger); +int32_t tsdbIterMergerSkipTableData(SIterMerger *merger, const TABLEID *tbid); + +SRowInfo *tsdbIterMergerGetData(SIterMerger *merger); +STombRecord *tsdbIterMergerGetTombRecord(SIterMerger *merger); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_ITER_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 97b648201c82cb92fd6673b2313653d4b2e61b12..ee3abf7559ecae536ab6dc522d541ee611477237 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -38,6 +38,16 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, SSubmitTbData *pSubmitTbData, int32_t *affectedRows); +static int32_t tTbDataCmprFn(const SRBTreeNode *n1, const SRBTreeNode *n2) { + STbData *tbData1 = TCONTAINER_OF(n1, STbData, rbtn); + STbData *tbData2 = TCONTAINER_OF(n2, STbData, rbtn); + if (tbData1->suid < tbData2->suid) return -1; + if (tbData1->suid > tbData2->suid) return 1; + if (tbData1->uid < tbData2->uid) return -1; + if (tbData1->uid > tbData2->uid) return 1; + return 0; +} + int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { int32_t code = 0; SMemTable *pMemTable = NULL; @@ -66,6 +76,7 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { goto _err; } vnodeBufPoolRef(pMemTable->pPool); + tRBTreeCreate(pMemTable->tbDataTree, tTbDataCmprFn); *ppMemTable = pMemTable; return code; @@ -190,9 +201,9 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid tsdbCacheDeleteLast(pTsdb->lruCache, pTbData->uid, eKey); } */ - if (eKey >= pTbData->maxKey && sKey <= pTbData->maxKey) { - tsdbCacheDel(pTsdb, suid, uid, sKey, eKey); - } + // if (eKey >= pTbData->maxKey && sKey <= pTbData->maxKey) { + tsdbCacheDel(pTsdb, suid, uid, sKey, eKey); + //} tsdbTrace("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 " at version %" PRId64, @@ -302,12 +313,12 @@ int64_t tsdbCountTbDataRows(STbData *pTbData) { return rowsNum; } -void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj* pTableMap, int64_t *rowsNum) { +void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj *pTableMap, int64_t *rowsNum) { taosRLockLatch(&pMemTable->latch); for (int32_t i = 0; i < pMemTable->nBucket; ++i) { STbData *pTbData = pMemTable->aBucket[i]; while (pTbData) { - void* p = tSimpleHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid)); + void *p = tSimpleHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid)); if (p == NULL) { pTbData = pTbData->next; continue; @@ -406,6 +417,8 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid pMemTable->aBucket[idx] = pTbData; pMemTable->nTbData++; + tRBTreePut(pMemTable->tbDataTree, pTbData->rbtn); + taosWUnLockLatch(&pMemTable->latch); _exit: @@ -673,7 +686,10 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, if (key.ts >= pTbData->maxKey) { pTbData->maxKey = key.ts; } - tsdbCacheUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, &lRow); + + if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, &lRow); + } // SMemTable pMemTable->minKey = TMIN(pMemTable->minKey, pTbData->minKey); @@ -734,7 +750,9 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, if (key.ts >= pTbData->maxKey) { pTbData->maxKey = key.ts; } - tsdbCacheUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, &lRow); + if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, &lRow); + } // SMemTable pMemTable->minKey = TMIN(pMemTable->minKey, pTbData->minKey); diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c new file mode 100644 index 0000000000000000000000000000000000000000..ec0ea3c60ffa1c8ef0a5564dcdb2e2f59bd6935a --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -0,0 +1,454 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbMerge.h" + +typedef struct { + STsdb *tsdb; + TFileSetArray *fsetArr; + + int32_t sttTrigger; + int32_t maxRow; + int32_t minRow; + int32_t szPage; + int8_t cmprAlg; + int64_t compactVersion; + int64_t cid; + + // context + struct { + bool opened; + int64_t now; + STFileSet *fset; + bool toData; + int32_t level; + SSttLvl *lvl; + TABLEID tbid[1]; + } ctx[1]; + + TFileOpArray fopArr[1]; + + // reader + TSttFileReaderArray sttReaderArr[1]; + // iter + TTsdbIterArray dataIterArr[1]; + SIterMerger *dataIterMerger; + TTsdbIterArray tombIterArr[1]; + SIterMerger *tombIterMerger; + // writer + SFSetWriter *writer; +} SMerger; + +static int32_t tsdbMergerOpen(SMerger *merger) { + merger->ctx->now = taosGetTimestampSec(); + merger->maxRow = merger->tsdb->pVnode->config.tsdbCfg.maxRows; + merger->minRow = merger->tsdb->pVnode->config.tsdbCfg.minRows; + merger->szPage = merger->tsdb->pVnode->config.tsdbPageSize; + merger->cmprAlg = merger->tsdb->pVnode->config.tsdbCfg.compression; + merger->compactVersion = INT64_MAX; + merger->cid = tsdbFSAllocEid(merger->tsdb->pFS); + merger->ctx->opened = true; + return 0; +} + +static int32_t tsdbMergerClose(SMerger *merger) { + int32_t code = 0; + int32_t lino = 0; + SVnode *pVnode = merger->tsdb->pVnode; + + // edit file system + code = tsdbFSEditBegin(merger->tsdb->pFS, merger->fopArr, TSDB_FEDIT_MERGE); + TSDB_CHECK_CODE(code, lino, _exit); + + taosThreadRwlockWrlock(&merger->tsdb->rwLock); + code = tsdbFSEditCommit(merger->tsdb->pFS); + if (code) { + taosThreadRwlockUnlock(&merger->tsdb->rwLock); + TSDB_CHECK_CODE(code, lino, _exit); + } + taosThreadRwlockUnlock(&merger->tsdb->rwLock); + + ASSERT(merger->writer == NULL); + ASSERT(merger->dataIterMerger == NULL); + ASSERT(merger->tombIterMerger == NULL); + ASSERT(TARRAY2_SIZE(merger->dataIterArr) == 0); + ASSERT(TARRAY2_SIZE(merger->tombIterArr) == 0); + ASSERT(TARRAY2_SIZE(merger->sttReaderArr) == 0); + + // clear the merge + TARRAY2_DESTROY(merger->tombIterArr, NULL); + TARRAY2_DESTROY(merger->dataIterArr, NULL); + TARRAY2_DESTROY(merger->sttReaderArr, NULL); + TARRAY2_DESTROY(merger->fopArr, NULL); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pVnode), lino, code); + } + return code; +} + +static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) { + int32_t code = 0; + int32_t lino = 0; + + merger->ctx->toData = true; + merger->ctx->level = 0; + + // TODO: optimize merge strategy + for (int32_t i = 0;; ++i) { + if (i >= TARRAY2_SIZE(merger->ctx->fset->lvlArr)) { + merger->ctx->lvl = NULL; + break; + } + + merger->ctx->lvl = TARRAY2_GET(merger->ctx->fset->lvlArr, i); + if (merger->ctx->lvl->level != merger->ctx->level || + TARRAY2_SIZE(merger->ctx->lvl->fobjArr) + 1 < merger->sttTrigger) { + merger->ctx->toData = false; + merger->ctx->lvl = NULL; + break; + } + + merger->ctx->level++; + + STFileObj *fobj; + int32_t numFile = 0; + TARRAY2_FOREACH(merger->ctx->lvl->fobjArr, fobj) { + if (numFile == merger->sttTrigger) { + break; + } + + STFileOp op = { + .optype = TSDB_FOP_REMOVE, + .fid = merger->ctx->fset->fid, + .of = fobj->f[0], + }; + code = TARRAY2_APPEND(merger->fopArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + SSttFileReader *reader; + SSttFileReaderConfig config = { + .tsdb = merger->tsdb, + .szPage = merger->szPage, + .file[0] = fobj->f[0], + }; + + code = tsdbSttFileReaderOpen(fobj->fname, &config, &reader); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(merger->sttReaderArr, reader); + TSDB_CHECK_CODE(code, lino, _exit); + + numFile++; + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbMergeFileSetBeginOpenIter(SMerger *merger) { + int32_t code = 0; + int32_t lino = 0; + int32_t vid = TD_VID(merger->tsdb->pVnode); + + SSttFileReader *sttReader; + TARRAY2_FOREACH(merger->sttReaderArr, sttReader) { + STsdbIter *iter; + STsdbIterConfig config = {0}; + + // data iter + config.type = TSDB_ITER_TYPE_STT; + config.sttReader = sttReader; + + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(merger->dataIterArr, iter); + TSDB_CHECK_CODE(code, lino, _exit); + + // tomb iter + config.type = TSDB_ITER_TYPE_STT_TOMB; + config.sttReader = sttReader; + + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(merger->tombIterArr, iter); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbIterMergerOpen(merger->dataIterArr, &merger->dataIterMerger, false); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbIterMergerOpen(merger->tombIterArr, &merger->tombIterMerger, true); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(vid, lino, code); + } + return code; +} + +static int32_t tsdbMergeFileSetBeginOpenWriter(SMerger *merger) { + int32_t code = 0; + int32_t lino = 0; + int32_t vid = TD_VID(merger->tsdb->pVnode); + + SDiskID did; + int32_t level = tsdbFidLevel(merger->ctx->fset->fid, &merger->tsdb->keepCfg, merger->ctx->now); + if (tfsAllocDisk(merger->tsdb->pVnode->pTfs, level, &did) < 0) { + code = TSDB_CODE_FS_NO_VALID_DISK; + TSDB_CHECK_CODE(code, lino, _exit); + } + tfsMkdirRecurAt(merger->tsdb->pVnode->pTfs, merger->tsdb->path, did); + SFSetWriterConfig config = { + .tsdb = merger->tsdb, + .toSttOnly = true, + .compactVersion = merger->compactVersion, + .minRow = merger->minRow, + .maxRow = merger->maxRow, + .szPage = merger->szPage, + .cmprAlg = merger->cmprAlg, + .fid = merger->ctx->fset->fid, + .cid = merger->cid, + .did = did, + .level = merger->ctx->level, + }; + + if (merger->ctx->toData) { + config.toSttOnly = false; + + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (merger->ctx->fset->farr[ftype]) { + config.files[ftype].exist = true; + config.files[ftype].file = merger->ctx->fset->farr[ftype]->f[0]; + } else { + config.files[ftype].exist = false; + } + } + } + + code = tsdbFSetWriterOpen(&config, &merger->writer); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(vid, lino, code); + } + return code; +} + +static int32_t tsdbMergeFileSetBegin(SMerger *merger) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(TARRAY2_SIZE(merger->sttReaderArr) == 0); + ASSERT(TARRAY2_SIZE(merger->dataIterArr) == 0); + ASSERT(merger->dataIterMerger == NULL); + ASSERT(merger->writer == NULL); + + merger->ctx->tbid->suid = 0; + merger->ctx->tbid->uid = 0; + + // open reader + code = tsdbMergeFileSetBeginOpenReader(merger); + TSDB_CHECK_CODE(code, lino, _exit); + + // open iterator + code = tsdbMergeFileSetBeginOpenIter(merger); + TSDB_CHECK_CODE(code, lino, _exit); + + // open writer + code = tsdbMergeFileSetBeginOpenWriter(merger); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbMergeFileSetEndCloseWriter(SMerger *merger) { + return tsdbFSetWriterClose(&merger->writer, 0, merger->fopArr); +} + +static int32_t tsdbMergeFileSetEndCloseIter(SMerger *merger) { + tsdbIterMergerClose(&merger->tombIterMerger); + TARRAY2_CLEAR(merger->tombIterArr, tsdbIterClose); + tsdbIterMergerClose(&merger->dataIterMerger); + TARRAY2_CLEAR(merger->dataIterArr, tsdbIterClose); + return 0; +} + +static int32_t tsdbMergeFileSetEndCloseReader(SMerger *merger) { + TARRAY2_CLEAR(merger->sttReaderArr, tsdbSttFileReaderClose); + return 0; +} + +static int32_t tsdbMergeFileSetEnd(SMerger *merger) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbMergeFileSetEndCloseWriter(merger); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbMergeFileSetEndCloseIter(merger); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbMergeFileSetEndCloseReader(merger); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbMergeFileSet(SMerger *merger, STFileSet *fset) { + int32_t code = 0; + int32_t lino = 0; + + merger->ctx->fset = fset; + code = tsdbMergeFileSetBegin(merger); + TSDB_CHECK_CODE(code, lino, _exit); + + // data + SMetaInfo info; + SRowInfo *row; + merger->ctx->tbid->suid = 0; + merger->ctx->tbid->uid = 0; + while ((row = tsdbIterMergerGetData(merger->dataIterMerger)) != NULL) { + if (row->uid != merger->ctx->tbid->uid) { + merger->ctx->tbid->uid = row->uid; + merger->ctx->tbid->suid = row->suid; + + if (metaGetInfo(merger->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) { + code = tsdbIterMergerSkipTableData(merger->dataIterMerger, merger->ctx->tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; + } + } + + code = tsdbFSetWriteRow(merger->writer, row); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbIterMergerNext(merger->dataIterMerger); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // tomb + merger->ctx->tbid->suid = 0; + merger->ctx->tbid->uid = 0; + for (STombRecord *record; (record = tsdbIterMergerGetTombRecord(merger->tombIterMerger)) != NULL;) { + if (record->uid != merger->ctx->tbid->uid) { + merger->ctx->tbid->uid = record->uid; + merger->ctx->tbid->suid = record->suid; + + if (metaGetInfo(merger->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) { + code = tsdbIterMergerSkipTableData(merger->tombIterMerger, merger->ctx->tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; + } + } + code = tsdbFSetWriteTombRecord(merger->writer, record); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbIterMergerNext(merger->tombIterMerger); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbMergeFileSetEnd(merger); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(merger->tsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(merger->tsdb->pVnode), __func__, fset->fid); + } + return code; +} + +static int32_t tsdbDoMerge(SMerger *merger) { + int32_t code = 0; + int32_t lino = 0; + + STFileSet *fset; + TARRAY2_FOREACH(merger->fsetArr, fset) { + if (TARRAY2_SIZE(fset->lvlArr) == 0) continue; + + SSttLvl *lvl = TARRAY2_FIRST(fset->lvlArr); + + if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < merger->sttTrigger) continue; + + if (!merger->ctx->opened) { + code = tsdbMergerOpen(merger); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbMergeFileSet(merger, fset); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (merger->ctx->opened) { + code = tsdbMergerClose(merger); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done", TD_VID(merger->tsdb->pVnode), __func__); + } + return code; +} + +int32_t tsdbMerge(void *arg) { + int32_t code = 0; + int32_t lino = 0; + STsdb *tsdb = (STsdb *)arg; + + SMerger merger[1] = {{ + .tsdb = tsdb, + .sttTrigger = tsdb->pVnode->config.sttTrigger, + }}; + + ASSERT(merger->sttTrigger > 1); + + code = tsdbFSCreateCopySnapshot(tsdb->pFS, &merger->fsetArr); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDoMerge(merger); + TSDB_CHECK_CODE(code, lino, _exit); + + tsdbFSDestroyCopySnapshot(&merger->fsetArr); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else if (merger->ctx->opened) { + tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); + } + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.h b/source/dnode/vnode/src/tsdb/tsdbMerge.h new file mode 100644 index 0000000000000000000000000000000000000000..69d802fd2776eddba8d65090dfe5717ba4bb76bc --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDataFileRW.h" +#include "tsdbFS2.h" +#include "tsdbFSetRW.h" +#include "tsdbIter.h" +#include "tsdbSttFileRW.h" +#include "tsdbUtil2.h" + +#ifndef _TD_TSDB_MERGE_H_ +#define _TD_TSDB_MERGE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Exposed Handle */ + +/* Exposed APIs */ + +/* Exposed Structs */ + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TSDB_MERGE_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 79f4a17f65430c3dc0efc8177c7f131db60934e3..ce6ee4345eafb49a29c4708c2b5383057e6d6f47 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -14,6 +14,12 @@ */ #include "tsdb.h" +#include "tsdbFSet2.h" +#include "tsdbMerge.h" +#include "tsdbReadUtil.h" +#include "tsdbSttFileRW.h" + +static void tLDataIterClose2(SLDataIter *pIter); // SLDataIter ================================================= SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, @@ -24,8 +30,6 @@ SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, return NULL; } - pLoadInfo->numOfStt = numOfSttTrigger; - for (int32_t i = 0; i < numOfSttTrigger; ++i) { pLoadInfo[i].blockIndex[0] = -1; pLoadInfo[i].blockIndex[1] = -1; @@ -50,8 +54,37 @@ SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, return pLoadInfo; } +SSttBlockLoadInfo *tCreateOneLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols) { + SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(1, sizeof(SSttBlockLoadInfo)); + if (pLoadInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pLoadInfo->blockIndex[0] = -1; + pLoadInfo->blockIndex[1] = -1; + pLoadInfo->currentLoadBlockIndex = 1; + + int32_t code = tBlockDataCreate(&pLoadInfo->blockData[0]); + if (code) { + terrno = code; + } + + code = tBlockDataCreate(&pLoadInfo->blockData[1]); + if (code) { + terrno = code; + } + + pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk)); + pLoadInfo->pSchema = pSchema; + pLoadInfo->colIds = colList; + pLoadInfo->numOfCols = numOfCols; + + return pLoadInfo; +} + void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { - for (int32_t i = 0; i < pLoadInfo->numOfStt; ++i) { + for (int32_t i = 0; i < 1; ++i) { pLoadInfo[i].currentLoadBlockIndex = 1; pLoadInfo[i].blockIndex[0] = -1; pLoadInfo[i].blockIndex[1] = -1; @@ -65,18 +98,24 @@ void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { } void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el) { - for (int32_t i = 0; i < pLoadInfo->numOfStt; ++i) { + for (int32_t i = 0; i < 1; ++i) { *el += pLoadInfo[i].elapsedTime; *blocks += pLoadInfo[i].loadBlocks; } } +static void freeTombBlock(void *param) { + STombBlock **pTombBlock = (STombBlock **)param; + tTombBlockDestroy(*pTombBlock); + taosMemoryFree(*pTombBlock); +} + void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { if (pLoadInfo == NULL) { return NULL; } - for (int32_t i = 0; i < pLoadInfo->numOfStt; ++i) { + for (int32_t i = 0; i < 1; ++i) { pLoadInfo[i].currentLoadBlockIndex = 1; pLoadInfo[i].blockIndex[0] = -1; pLoadInfo[i].blockIndex[1] = -1; @@ -91,6 +130,33 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { return NULL; } +static void destroyLDataIter(SLDataIter *pIter) { + tLDataIterClose2(pIter); + destroyLastBlockLoadInfo(pIter->pBlockLoadInfo); + taosMemoryFree(pIter); +} + +void *destroySttBlockReader(SArray *pLDataIterArray, int64_t *blocks, double *el) { + if (pLDataIterArray == NULL) { + return NULL; + } + + int32_t numOfLevel = taosArrayGetSize(pLDataIterArray); + for (int32_t i = 0; i < numOfLevel; ++i) { + SArray *pList = taosArrayGetP(pLDataIterArray, i); + for (int32_t j = 0; j < taosArrayGetSize(pList); ++j) { + SLDataIter *pIter = taosArrayGetP(pList, j); + *el += pIter->pBlockLoadInfo->elapsedTime; + *blocks += pIter->pBlockLoadInfo->loadBlocks; + destroyLDataIter(pIter); + } + taosArrayDestroy(pList); + } + + taosArrayDestroy(pLDataIterArray); + return NULL; +} + static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { int32_t code = 0; @@ -122,20 +188,8 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { int64_t st = taosGetTimestampUs(); SBlockData *pBlock = &pInfo->blockData[pInfo->currentLoadBlockIndex]; - - TABLEID id = {0}; - if (pIter->pSttBlk->suid != 0) { - id.suid = pIter->pSttBlk->suid; - } else { - id.uid = pIter->uid; - } - - code = tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } - - code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlock); + code = tsdbSttFileReadBlockDataByColumn(pIter->pReader, pIter->pSttBlk, pBlock, pInfo->pSchema, &pInfo->colIds[1], + pInfo->numOfCols - 1); if (code != TSDB_CODE_SUCCESS) { goto _exit; } @@ -255,74 +309,190 @@ static int32_t binarySearchForStartRowIndex(uint64_t *uidList, int32_t num, uint int32_t tLDataIterOpen(struct SLDataIter *pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo *pBlockLoadInfo, const char *idStr, bool strictTimeRange) { + return 0; +} + +static int32_t extractSttBlockInfo(SLDataIter *pIter, const TSttBlkArray *pArray, SSttBlockLoadInfo *pBlockLoadInfo, + uint64_t suid) { + if (TARRAY2_SIZE(pArray) <= 0) { + return TSDB_CODE_SUCCESS; + } + + SSttBlk *pStart = &pArray->data[0]; + SSttBlk *pEnd = &pArray->data[TARRAY2_SIZE(pArray) - 1]; + + // all identical + if (pStart->suid == pEnd->suid) { + if (pStart->suid != suid) { // no qualified stt block existed + taosArrayClear(pBlockLoadInfo->aSttBlk); + pIter->iSttBlk = -1; + return TSDB_CODE_SUCCESS; + } else { // all blocks are qualified + taosArrayClear(pBlockLoadInfo->aSttBlk); + taosArrayAddBatch(pBlockLoadInfo->aSttBlk, pArray->data, pArray->size); + } + } else { + SArray *pTmp = taosArrayInit(TARRAY2_SIZE(pArray), sizeof(SSttBlk)); + for (int32_t i = 0; i < TARRAY2_SIZE(pArray); ++i) { + SSttBlk *p = &pArray->data[i]; + if (p->suid < suid) { + continue; + } + + if (p->suid == suid) { + taosArrayPush(pTmp, p); + } else if (p->suid > suid) { + break; + } + } + + taosArrayDestroy(pBlockLoadInfo->aSttBlk); + pBlockLoadInfo->aSttBlk = pTmp; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t suidComparFn(const void *target, const void *p2) { + const uint64_t *targetUid = target; + const uint64_t *uid2 = p2; + if (*uid2 == (*targetUid)) { + return 0; + } else { + return (*targetUid) < (*uid2) ? -1 : 1; + } +} + +static bool existsFromSttBlkStatis(const TStatisBlkArray *pStatisBlkArray, uint64_t suid, uint64_t uid, + SSttFileReader *pReader) { + if (TARRAY2_SIZE(pStatisBlkArray) <= 0) { + return true; + } + + int32_t i = 0; + for (i = 0; i < TARRAY2_SIZE(pStatisBlkArray); ++i) { + SStatisBlk *p = &pStatisBlkArray->data[i]; + if (p->minTbid.suid <= suid && p->maxTbid.suid >= suid) { + break; + } + } + + if (i >= TARRAY2_SIZE(pStatisBlkArray)) { + return false; + } + + while (i < TARRAY2_SIZE(pStatisBlkArray)) { + SStatisBlk *p = &pStatisBlkArray->data[i]; + if (p->minTbid.suid > suid) { + return false; + } + + STbStatisBlock block = {0}; + tsdbSttFileReadStatisBlock(pReader, p, &block); + + int32_t index = tarray2SearchIdx(block.suid, &suid, sizeof(int64_t), suidComparFn, TD_EQ); + if (index == -1) { + tStatisBlockDestroy(&block); + return false; + } + + int32_t j = index; + if (block.uid->data[j] == uid) { + tStatisBlockDestroy(&block); + return true; + } else if (block.uid->data[j] > uid) { + while (j >= 0 && block.suid->data[j] == suid) { + if (block.uid->data[j] == uid) { + tStatisBlockDestroy(&block); + return true; + } else { + j -= 1; + } + } + } else { + j = index + 1; + while (j < block.suid->size && block.suid->data[j] == suid) { + if (block.uid->data[j] == uid) { + tStatisBlockDestroy(&block); + return true; + } else { + j += 1; + } + } + } + + tStatisBlockDestroy(&block); + i += 1; + } + + return false; +} + +int32_t tLDataIterOpen2(struct SLDataIter *pIter, SSttFileReader *pSttFileReader, int32_t iStt, int8_t backward, + uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, + SSttBlockLoadInfo *pBlockLoadInfo, const char *idStr, bool strictTimeRange, + _load_tomb_fn loadTombFn, void *pReader1) { int32_t code = TSDB_CODE_SUCCESS; pIter->uid = uid; - pIter->pReader = pReader; pIter->iStt = iStt; pIter->backward = backward; pIter->verRange.minVer = pRange->minVer; pIter->verRange.maxVer = pRange->maxVer; pIter->timeWindow.skey = pTimeWindow->skey; pIter->timeWindow.ekey = pTimeWindow->ekey; - + pIter->pReader = pSttFileReader; pIter->pBlockLoadInfo = pBlockLoadInfo; + if (pIter->pReader == NULL) { + tsdbError("stt file reader is null, %s", idStr); + pIter->pSttBlk = NULL; + pIter->iSttBlk = -1; + return TSDB_CODE_SUCCESS; + } + if (!pBlockLoadInfo->sttBlockLoaded) { int64_t st = taosGetTimestampUs(); + + const TSttBlkArray *pSttBlkArray = NULL; pBlockLoadInfo->sttBlockLoaded = true; - code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk); - if (code) { + // load the stt block info for each stt-block + code = tsdbSttFileReadSttBlk(pIter->pReader, &pSttBlkArray); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("load stt blk failed, code:%s, %s", tstrerror(code), idStr); return code; } - // only apply to the child tables, ordinary tables will not incur this filter procedure. - size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); - - if (size >= 1) { - SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); - SSttBlk *pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1); - - // all identical - if (pStart->suid == pEnd->suid) { - if (pStart->suid != suid) { - // no qualified stt block existed - taosArrayClear(pBlockLoadInfo->aSttBlk); - - pIter->iSttBlk = -1; - double el = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr); - return code; - } - } else { - SArray *pTmp = taosArrayInit(size, sizeof(SSttBlk)); - for (int32_t i = 0; i < size; ++i) { - SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); - uint64_t s = p->suid; - if (s < suid) { - continue; - } - - if (s == suid) { - taosArrayPush(pTmp, p); - } else if (s > suid) { - break; - } - } + code = extractSttBlockInfo(pIter, pSttBlkArray, pBlockLoadInfo, suid); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("load stt block info failed, code:%s, %s", tstrerror(code), idStr); + return code; + } - taosArrayDestroy(pBlockLoadInfo->aSttBlk); - pBlockLoadInfo->aSttBlk = pTmp; - } + // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file + code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray **)&pBlockLoadInfo->pSttStatisBlkArray); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), idStr); + return code; } + code = loadTombFn(pReader1, pIter->pReader, pIter->pBlockLoadInfo); + double el = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr); + tsdbDebug("load the stt file info completed, elapsed time:%.2fms, %s", el, idStr); } - size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + // bool exists = existsFromSttBlkStatis(pBlockLoadInfo->pSttStatisBlkArray, suid, uid, pIter->pReader); + // if (!exists) { + // pIter->iSttBlk = -1; + // pIter->pSttBlk = NULL; + // return TSDB_CODE_SUCCESS; + // } - // find the start block + // find the start block, actually we could load the position to avoid repeatly searching for the start position when + // the skey is updated. + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); pIter->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward); if (pIter->iSttBlk != -1) { pIter->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, pIter->iSttBlk); @@ -343,7 +513,10 @@ int32_t tLDataIterOpen(struct SLDataIter *pIter, SDataFReader *pReader, int32_t return code; } -void tLDataIterClose(SLDataIter *pIter) { /*taosMemoryFree(pIter); */} +void tLDataIterClose2(SLDataIter *pIter) { + tsdbSttFileReaderClose(&pIter->pReader); + pIter->pReader = NULL; +} void tLDataIterNextBlock(SLDataIter *pIter, const char *idStr) { int32_t step = pIter->backward ? -1 : 1; @@ -395,25 +568,23 @@ void tLDataIterNextBlock(SLDataIter *pIter, const char *idStr) { if (index != -1) { pIter->iSttBlk = index; pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, pIter->iSttBlk); - tsdbDebug("try next last file block:%d from %d, trigger by uid:%" PRIu64 ", file index:%d, %s", pIter->iSttBlk, - oldIndex, pIter->uid, pIter->iStt, idStr); + tsdbDebug("try next last file block:%d from stt fileIdx:%d, trigger by uid:%" PRIu64 ", file index:%d, %s", + pIter->iSttBlk, oldIndex, pIter->uid, pIter->iStt, idStr); } else { tsdbDebug("no more last block qualified, uid:%" PRIu64 ", file index:%d, %s", pIter->uid, oldIndex, idStr); } } static void findNextValidRow(SLDataIter *pIter, const char *idStr) { - int32_t step = pIter->backward ? -1 : 1; - bool hasVal = false; + int32_t step = pIter->backward ? -1 : 1; int32_t i = pIter->iRow; - SBlockData *pBlockData = loadLastBlock(pIter, idStr); + SBlockData *pData = loadLastBlock(pIter, idStr); // mostly we only need to find the start position for a given table - if ((((i == 0) && (!pIter->backward)) || (i == pBlockData->nRow - 1 && pIter->backward)) && - pBlockData->aUid != NULL) { - i = binarySearchForStartRowIndex((uint64_t *)pBlockData->aUid, pBlockData->nRow, pIter->uid, pIter->backward); + if ((((i == 0) && (!pIter->backward)) || (i == pData->nRow - 1 && pIter->backward)) && pData->aUid != NULL) { + i = binarySearchForStartRowIndex((uint64_t *)pData->aUid, pData->nRow, pIter->uid, pIter->backward); if (i == -1) { tsdbDebug("failed to find the data in pBlockData, uid:%" PRIu64 " , %s", pIter->uid, idStr); pIter->iRow = -1; @@ -421,20 +592,20 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) { } } - for (; i < pBlockData->nRow && i >= 0; i += step) { - if (pBlockData->aUid != NULL) { + for (; i < pData->nRow && i >= 0; i += step) { + if (pData->aUid != NULL) { if (!pIter->backward) { - if (pBlockData->aUid[i] > pIter->uid) { + if (pData->aUid[i] > pIter->uid) { break; } } else { - if (pBlockData->aUid[i] < pIter->uid) { + if (pData->aUid[i] < pIter->uid) { break; } } } - int64_t ts = pBlockData->aTSKEY[i]; + int64_t ts = pData->aTSKEY[i]; if (!pIter->backward) { // asc if (ts > pIter->timeWindow.ekey) { // no more data break; @@ -449,7 +620,7 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) { } } - int64_t ver = pBlockData->aVersion[i]; + int64_t ver = pData->aVersion[i]; if (ver < pIter->verRange.minVer) { continue; } @@ -485,7 +656,6 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { while (1) { bool skipBlock = false; - findNextValidRow(pIter, idStr); if (pIter->pBlockLoadInfo->checkRemainingRow) { @@ -570,7 +740,7 @@ static FORCE_INLINE int32_t tLDataIterDescCmprFn(const SRBTreeNode *p1, const SR int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, - bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter) { + bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter *pLDataIter) { int32_t code = TSDB_CODE_SUCCESS; pMTree->backward = backward; @@ -612,6 +782,101 @@ _end: return code; } +int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) { + int32_t code = TSDB_CODE_SUCCESS; + + pMTree->pIter = NULL; + pMTree->backward = pConf->backward; + pMTree->idStr = pConf->idstr; + + if (!pMTree->backward) { // asc + tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); + } else { // desc + tRBTreeCreate(&pMTree->rbt, tLDataIterDescCmprFn); + } + + pMTree->ignoreEarlierTs = false; + + int32_t size = ((STFileSet *)pConf->pCurrentFileset)->lvlArr->size; + if (size == 0) { + goto _end; + } + + // add the list/iter placeholder + while (taosArrayGetSize(pConf->pSttFileBlockIterArray) < size) { + SArray *pList = taosArrayInit(4, POINTER_BYTES); + taosArrayPush(pConf->pSttFileBlockIterArray, &pList); + } + + for (int32_t j = 0; j < size; ++j) { + SSttLvl *pSttLevel = ((STFileSet *)pConf->pCurrentFileset)->lvlArr->data[j]; + ASSERT(pSttLevel->level == j); + + SArray *pList = taosArrayGetP(pConf->pSttFileBlockIterArray, j); + int32_t numOfIter = taosArrayGetSize(pList); + + if (numOfIter < TARRAY2_SIZE(pSttLevel->fobjArr)) { + int32_t inc = TARRAY2_SIZE(pSttLevel->fobjArr) - numOfIter; + for (int32_t k = 0; k < inc; ++k) { + SLDataIter *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); + taosArrayPush(pList, &pIter); + } + } else if (numOfIter > TARRAY2_SIZE(pSttLevel->fobjArr)){ + int32_t inc = numOfIter - TARRAY2_SIZE(pSttLevel->fobjArr); + for (int i = 0; i < inc; ++i) { + SLDataIter *pIter = taosArrayPop(pList); + destroyLDataIter(pIter); + } + } + + for (int32_t i = 0; i < TARRAY2_SIZE(pSttLevel->fobjArr); ++i) { // open all last file + SLDataIter *pIter = taosArrayGetP(pList, i); + + SSttFileReader *pSttFileReader = pIter->pReader; + SSttBlockLoadInfo *pLoadInfo = pIter->pBlockLoadInfo; + + // open stt file reader if not + if (pSttFileReader == NULL) { + SSttFileReaderConfig conf = {.tsdb = pConf->pTsdb, .szPage = pConf->pTsdb->pVnode->config.tsdbPageSize}; + conf.file[0] = *pSttLevel->fobjArr->data[i]->f; + + code = tsdbSttFileReaderOpen(pSttLevel->fobjArr->data[i]->fname, &conf, &pSttFileReader); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("open stt file reader error. file name %s, code %s, %s", pSttLevel->fobjArr->data[i]->fname, + tstrerror(code), pMTree->idStr); + } + } + + if (pLoadInfo == NULL) { + pLoadInfo = tCreateOneLastBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); + } + + memset(pIter, 0, sizeof(SLDataIter)); + code = tLDataIterOpen2(pIter, pSttFileReader, i, pMTree->backward, pConf->suid, pConf->uid, &pConf->timewindow, + &pConf->verRange, pLoadInfo, pMTree->idStr, pConf->strictTimeRange, pConf->loadTombFn, + pConf->pReader); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + + bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr); + if (hasVal) { + tMergeTreeAddIter(pMTree, pIter); + } else { + if (!pMTree->ignoreEarlierTs) { + pMTree->ignoreEarlierTs = pIter->ignoreEarlierTs; + } + } + } + } + + return code; + +_end: + tMergeTreeClose(pMTree); + return code; +} + void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); } bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree) { return pMTree->ignoreEarlierTs; } diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 8901f644598ec4ca5343f4a35a7b063bf39096fd..c684ad51848d1a6af49ff2881f05b96eca6e51dc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -14,6 +14,7 @@ */ #include "tsdb.h" +#include "tsdbFS2.h" int32_t tsdbSetKeepCfg(STsdb *pTsdb, STsdbCfg *pCfg) { STsdbKeepCfg *pKeepCfg = &pTsdb->keepCfg; @@ -66,7 +67,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee } // open tsdb - if (tsdbFSOpen(pTsdb, rollback) < 0) { + if (tsdbOpenFS(pTsdb, &pTsdb->pFS, rollback) < 0) { goto _err; } @@ -94,7 +95,7 @@ int tsdbClose(STsdb **pTsdb) { taosThreadRwlockDestroy(&(*pTsdb)->rwLock); - tsdbFSClose(*pTsdb); + tsdbCloseFS(&(*pTsdb)->pFS); tsdbCloseCache(*pTsdb); taosMemoryFreeClear(*pTsdb); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c04a23d71f1649b7f1247f80109cd15b8d4951c9..2aa21bd86f31e8167d0f5f0fd9406dad9a50d513 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -17,7 +17,7 @@ #include "tsdb.h" #include "tsimplehash.h" -#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define getCurrentKeyInLastBlock(_r) ((_r)->currentKey) typedef enum { @@ -30,12 +30,12 @@ typedef enum { EXTERNAL_ROWS_MAIN = 0x2, EXTERNAL_ROWS_NEXT = 0x3, } EContentData; - +/* typedef enum { READ_MODE_COUNT_ONLY = 0x1, READ_MODE_ALL, } EReadMode; - +*/ typedef struct { STbDataIter* iter; int32_t index; @@ -166,7 +166,7 @@ typedef struct SReaderStatus { SDataBlockIter blockIter; SLDataIter* pLDataIter; SRowMerger merger; - SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data + SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data } SReaderStatus; typedef struct SBlockInfoBuf { @@ -210,6 +210,7 @@ struct STsdbReader { SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; SIOCostSummary cost; + SHashObj** pIgnoreTables; STSchema* pSchema; // the newest version schema SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema SDataFReader* pFileReader; // the file reader @@ -247,7 +248,7 @@ static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdb STbData* piMemTbData); static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr, int8_t* pLevel); -static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); +static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, const char* id); static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); static int32_t doBuildDataBlock(STsdbReader* pReader); static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); @@ -288,6 +289,10 @@ static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pC static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) { int32_t i = 0, j = 0; + if (j < pSupInfo->numOfCols && PRIMARYKEY_TIMESTAMP_COL_ID == pSupInfo->colId[j]) { + j += 1; + } + while (i < pSchema->numOfCols && j < pSupInfo->numOfCols) { STColumn* pTCol = &pSchema->columns[i]; if (pTCol->colId == pSupInfo->colId[j]) { @@ -405,7 +410,7 @@ static int32_t uidComparFunc(const void* p1, const void* p2) { // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model static SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, - STableUidList* pUidList, int32_t numOfTables) { + STableUidList* pUidList, int32_t numOfTables) { // allocate buffer in order to load data blocks from file // todo use simple hash instead, optimize the memory consumption SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); @@ -456,7 +461,7 @@ static SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBu } static void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { - void *p = NULL; + void* p = NULL; int32_t iter = 0; while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { @@ -500,7 +505,7 @@ static void clearBlockScanInfo(STableBlockScanInfo* p) { } static void destroyAllBlockScanInfo(SSHashObj* pTableMap) { - void* p = NULL; + void* p = NULL; int32_t iter = 0; while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { @@ -738,7 +743,8 @@ void tsdbReleaseDataBlock(STsdbReader* pReader) { } } -static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacity, SSDataBlock* pResBlock, SQueryTableDataCond* pCond) { +static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacity, SSDataBlock* pResBlock, + SQueryTableDataCond* pCond) { pResBlockInfo->capacity = capacity; pResBlockInfo->pResBlock = pResBlock; terrno = 0; @@ -753,7 +759,7 @@ static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacit return terrno; } -static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsdbReader** ppReader, int32_t capacity, +static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void** ppReader, int32_t capacity, SSDataBlock* pResBlock, const char* idstr) { int32_t code = 0; int8_t level = 0; @@ -774,7 +780,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->order = pCond->order; pReader->idStr = (idstr != NULL) ? taosStrdup(idstr) : NULL; - pReader->verRange = getQueryVerRange(pVnode, pCond, level); + pReader->verRange = getQueryVerRange(pVnode, pCond, idstr); pReader->type = pCond->type; pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket @@ -809,7 +815,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd if (pReader->suppInfo.colId[0] != PRIMARYKEY_TIMESTAMP_COL_ID) { tsdbError("the first column isn't primary timestamp, %d, %s", pReader->suppInfo.colId[0], pReader->idStr); - terrno = TSDB_CODE_INVALID_PARA; + code = TSDB_CODE_INVALID_PARA; goto _end; } @@ -818,7 +824,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd if (type != TSDB_DATA_TYPE_TIMESTAMP) { tsdbError("the first column isn't primary timestamp in result block, actual: %s, %s", tDataTypes[type].name, pReader->idStr); - terrno = TSDB_CODE_INVALID_PARA; + code = TSDB_CODE_INVALID_PARA; goto _end; } @@ -916,9 +922,9 @@ static void cleanupTableScanInfo(SReaderStatus* pStatus) { return; } - SSHashObj* pTableMap = pStatus->pTableMap; + SSHashObj* pTableMap = pStatus->pTableMap; STableBlockScanInfo** px = NULL; - int32_t iter = 0; + int32_t iter = 0; while (1) { px = tSimpleHashIterate(pTableMap, px, &iter); @@ -932,9 +938,10 @@ static void cleanupTableScanInfo(SReaderStatus* pStatus) { pStatus->mapDataCleaned = true; } -static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum, SArray* pTableScanInfoList) { - size_t sizeInDisk = 0; - size_t numOfTables = taosArrayGetSize(pIndexList); +static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum, + SArray* pTableScanInfoList) { + size_t sizeInDisk = 0; + size_t numOfTables = taosArrayGetSize(pIndexList); int64_t st = taosGetTimestampUs(); cleanupTableScanInfo(&pReader->status); @@ -1120,6 +1127,27 @@ static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData endPos = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, key, pReader->order); } + if ((pReader->verRange.maxVer >= pBlock->minVer && pReader->verRange.maxVer < pBlock->maxVer) || + (pReader->verRange.minVer <= pBlock->maxVer && pReader->verRange.minVer > pBlock->minVer)) { + int32_t i = endPos; + + if (asc) { + for (; i >= 0; --i) { + if (pBlockData->aVersion[i] <= pReader->verRange.maxVer) { + break; + } + } + } else { + for (; i < pBlock->nRow; ++i) { + if (pBlockData->aVersion[i] >= pReader->verRange.minVer) { + break; + } + } + } + + endPos = i; + } + return endPos; } @@ -1259,10 +1287,11 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { return 0; } + // row index of dump info remain the initial position, let's find the appropriate start position. if ((pDumpInfo->rowIndex == 0 && asc) || (pDumpInfo->rowIndex == pBlock->nRow - 1 && (!asc))) { - if (asc && pReader->window.skey <= pBlock->minKey.ts) { + if (asc && pReader->window.skey <= pBlock->minKey.ts && pReader->verRange.minVer <= pBlock->minVer) { // pDumpInfo->rowIndex = 0; - } else if (!asc && pReader->window.ekey >= pBlock->maxKey.ts) { + } else if (!asc && pReader->window.ekey >= pBlock->maxKey.ts && pReader->verRange.maxVer >= pBlock->maxVer) { // pDumpInfo->rowIndex = pBlock->nRow - 1; } else { // find the appropriate the start position in current block, and set it to be the current rowIndex int32_t pos = asc ? pBlock->nRow - 1 : 0; @@ -1278,6 +1307,29 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { pBlock->maxVer, pReader->idStr); return TSDB_CODE_INVALID_PARA; } + + ASSERT(pReader->verRange.minVer <= pBlock->maxVer && pReader->verRange.maxVer >= pBlock->minVer); + + // find the appropriate start position that satisfies the version requirement. + if ((pReader->verRange.maxVer >= pBlock->minVer && pReader->verRange.maxVer < pBlock->maxVer) || + (pReader->verRange.minVer <= pBlock->maxVer && pReader->verRange.minVer > pBlock->minVer)) { + int32_t i = pDumpInfo->rowIndex; + if (asc) { + for (; i < pBlock->nRow; ++i) { + if (pBlockData->aVersion[i] >= pReader->verRange.minVer) { + break; + } + } + } else { + for (; i >= 0; --i) { + if (pBlockData->aVersion[i] <= pReader->verRange.maxVer) { + break; + } + } + } + + pDumpInfo->rowIndex = i; + } } } @@ -1292,6 +1344,9 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { int32_t dumpedRows = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex); if (dumpedRows > pReader->resBlockInfo.capacity) { // output buffer check dumpedRows = pReader->resBlockInfo.capacity; + } else if (dumpedRows <= 0) { // no qualified rows in current data block, abort directly. + setBlockAllDumped(pDumpInfo, pReader->window.ekey, pReader->order); + return TSDB_CODE_SUCCESS; } int32_t i = 0; @@ -1509,7 +1564,8 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter, const char* idStr) return TSDB_CODE_SUCCESS; } -static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) { +static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, + SArray* pTableList) { bool asc = ASCENDING_TRAVERSE(pReader->order); SBlockOrderSupporter sup = {0}; @@ -1766,6 +1822,9 @@ static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SDa if (p->ts > pBlock->minKey.ts && index > 0) { index -= 1; } else { // find the first point that is smaller than the minKey.ts of dataBlock. + if (p->ts == pBlock->minKey.ts && p->version < pBlock->maxVer && index > 0) { + index -= 1; + } break; } } @@ -1911,13 +1970,14 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc } TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - int64_t key = pRow->pBlockData->aTSKEY[pRow->iRow]; - int64_t ver = pRow->pBlockData->aVersion[pRow->iRow]; + int64_t key = pRow->pBlockData->aTSKEY[pRow->iRow]; + int64_t ver = pRow->pBlockData->aVersion[pRow->iRow]; pLastBlockReader->currentKey = key; pScanInfo->lastKeyInStt = key; - if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order, pVerRange)) { + if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order, + pVerRange)) { return true; } } @@ -1974,7 +2034,7 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* } STSchema* ptr = NULL; - int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; @@ -2097,7 +2157,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return terrno; } - int32_t code = tsdbRowMergerAdd(pMerger, pRow, pSchema); + int32_t code = tsdbRowMergerAdd(pMerger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2152,7 +2212,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, bool mergeBlockData) { - SRowMerger* pMerger = &pReader->status.merger; + SRowMerger* pMerger = &pReader->status.merger; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); @@ -2162,9 +2222,10 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); // create local variable to hold the row value - TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; + TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; - tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", pRow->pBlockData, pRow->iRow, pLastBlockReader->uid, pReader->idStr); + tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", pRow->pBlockData, pRow->iRow, pLastBlockReader->uid, + pReader->idStr); // only last block exists if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { @@ -2184,7 +2245,8 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tsdbRowMergerAdd(pMerger, pRow1, NULL); - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->verRange, + pReader->idStr); code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { @@ -2234,7 +2296,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader* pLastBlockReader, int64_t key, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SRowMerger* pMerger = &pReader->status.merger; + SRowMerger* pMerger = &pReader->status.merger; // merge is not initialized yet, due to the fact that the pReader->pSchema is not initialized if (pMerger->pArray == NULL) { @@ -2260,7 +2322,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader if (key < ts) { // imem, mem are all empty, file blocks (data blocks and last block) exist return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key == ts) { - SRow* pTSRow = NULL; + SRow* pTSRow = NULL; int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2667,7 +2729,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc } else { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - SRow* pTSRow = NULL; + SRow* pTSRow = NULL; code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2781,25 +2843,31 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - bool asc = ASCENDING_TRAVERSE(pReader->order); - int64_t st = taosGetTimestampUs(); - int32_t step = asc ? 1 : -1; - double el = 0; + bool asc = ASCENDING_TRAVERSE(pReader->order); + int64_t st = taosGetTimestampUs(); + int32_t step = asc ? 1 : -1; + double el = 0; + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; STableBlockScanInfo* pBlockScanInfo = NULL; if (pBlockInfo != NULL) { + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { + setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); + return code; + } + pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); if (pBlockScanInfo == NULL) { goto _end; } - SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); - TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); + TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); // it is a clean block, load it directly if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && pBlock->nRow <= pReader->resBlockInfo.capacity) { - if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) { + if (asc || (!hasDataInLastBlock(pLastBlockReader) && (pBlock->maxKey.ts > keyInBuf.ts))) { code = copyBlockDataToSDataBlock(pReader); if (code) { goto _end; @@ -2812,10 +2880,14 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } } else { // file blocks not exist pBlockScanInfo = *pReader->status.pTableIter; + if (pReader->pIgnoreTables && + taosHashGet(*pReader->pIgnoreTables, &pBlockScanInfo->uid, sizeof(pBlockScanInfo->uid))) { + setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); + return code; + } } - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SBlockData* pBlockData = &pReader->status.fileBlockData; + SBlockData* pBlockData = &pReader->status.fileBlockData; while (1) { bool hasBlockData = false; @@ -2829,7 +2901,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { pDumpInfo->rowIndex += step; - SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); + pBlock = getCurrentBlock(&pReader->status.blockIter); if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) { pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); // NOTE: get the new block info @@ -2857,7 +2929,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { - SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); + pBlock = getCurrentBlock(&pReader->status.blockIter); setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); break; } @@ -2906,7 +2978,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* SDelIdx* pIdx = taosArraySearch(pReader->pDelIdx, &idx, tCmprDelIdx, TD_EQ); if (pIdx != NULL) { - code = tsdbReadDelData(pReader->pDelFReader, pIdx, pDelData); + code = tsdbReadDelDatav1(pReader->pDelFReader, pIdx, pDelData, pReader->verRange.maxVer); } if (code != TSDB_CODE_SUCCESS) { goto _err; @@ -2917,7 +2989,10 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* if (pMemTbData != NULL) { p = pMemTbData->pHead; while (p) { - taosArrayPush(pDelData, p); + if (p->version <= pReader->verRange.maxVer) { + taosArrayPush(pDelData, p); + } + p = p->pNext; } } @@ -2925,7 +3000,9 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* if (piMemTbData != NULL) { p = piMemTbData->pHead; while (p) { - taosArrayPush(pDelData, p); + if (p->version <= pReader->verRange.maxVer) { + taosArrayPush(pDelData, p); + } p = p->pNext; } } @@ -2998,6 +3075,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr // only check here, since the iterate data in memory is very fast. if (pReader->code != TSDB_CODE_SUCCESS) { tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + taosArrayDestroy(pIndexList); return pReader->code; } @@ -3101,6 +3179,28 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { // load the last data block of current table STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; + if (pScanInfo == NULL) { + tsdbError("table Iter is null, invalid pScanInfo, try next table %s", pReader->idStr); + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + + continue; + } + + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pScanInfo->uid, sizeof(pScanInfo->uid))) { + // reset the index in last block when handing a new file + doCleanupTableScanInfo(pScanInfo); + pStatus->mapDataCleaned = true; + + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + + continue; + } // reset the index in last block when handing a new file doCleanupTableScanInfo(pScanInfo); @@ -3155,14 +3255,19 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { } static int32_t doBuildDataBlock(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; - SDataBlk* pBlock = NULL; + int32_t code = TSDB_CODE_SUCCESS; SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; STableBlockScanInfo* pScanInfo = NULL; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + SDataBlk* pBlock = getCurrentBlock(pBlockIter); + + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock->maxKey.ts, pReader->order); + return code; + } if (pReader->code != TSDB_CODE_SUCCESS) { return pReader->code; @@ -3173,8 +3278,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return terrno; } - pBlock = getCurrentBlock(pBlockIter); - initLastBlockReader(pLastBlockReader, pScanInfo, pReader); TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); @@ -3251,7 +3354,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { } } - return (pReader->code != TSDB_CODE_SUCCESS)? pReader->code:code; + return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code; } static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { @@ -3406,6 +3509,15 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { } STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter; + if (pReader->pIgnoreTables && + taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) { + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + pBlockScanInfo = pStatus->pTableIter; + } + initMemDataIterator(*pBlockScanInfo, pReader); int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? INT64_MAX : INT64_MIN; @@ -3449,7 +3561,7 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { SBlockNumber num = {0}; - SArray* pTableList = taosArrayInit(40, POINTER_BYTES); + SArray* pTableList = taosArrayInit(40, POINTER_BYTES); int32_t code = moveToNextFile(pReader, &num, pTableList); if (code != TSDB_CODE_SUCCESS) { @@ -3494,7 +3606,7 @@ static ERetrieveType doReadDataFromLastFiles(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; SDataBlockIter* pBlockIter = &pReader->status.blockIter; - while(1) { + while (1) { terrno = 0; code = doLoadLastBlockSequentially(pReader); @@ -3517,7 +3629,7 @@ static ERetrieveType doReadDataFromLastFiles(STsdbReader* pReader) { return TSDB_READ_RETURN; } - if (pBlockIter->numOfBlocks > 0) { // there are data blocks existed. + if (pBlockIter->numOfBlocks > 0) { // there are data blocks existed. return TSDB_READ_CONTINUE; } else { // all blocks in data file are checked, let's check the data in last files resetTableListIndex(&pReader->status); @@ -3530,7 +3642,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { bool asc = ASCENDING_TRAVERSE(pReader->order); SDataBlockIter* pBlockIter = &pReader->status.blockIter; - SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; if (pBlockIter->numOfBlocks == 0) { // let's try to extract data from stt files. @@ -3631,7 +3743,7 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret return VND_TSDB(pVnode); } -SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) { +SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, const char* id) { int64_t startVer = (pCond->startVersion == -1) ? 0 : pCond->startVersion; int64_t endVer = 0; @@ -3642,10 +3754,14 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_ endVer = (pCond->endVersion > pVnode->state.applied) ? pVnode->state.applied : pCond->endVersion; } + tsdbDebug("queried verRange:%" PRId64 "-%" PRId64 ", revised query verRange:%" PRId64 "-%" PRId64 ", %s", + pCond->startVersion, pCond->endVersion, startVer, endVer, id); + return (SVersionRange){.minVer = startVer, .maxVer = endVer}; } -bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, SVersionRange* pVerRange) { +bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, + SVersionRange* pVerRange) { if (pDelList == NULL) { return false; } @@ -3663,8 +3779,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t return false; } else if (key == last->ts) { TSDBKEY* prev = taosArrayGet(pDelList, num - 2); - return (prev->version >= ver && prev->version <= pVerRange->maxVer && - prev->version >= pVerRange->minVer); + return (prev->version >= ver && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer); } } else { TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); @@ -3873,9 +3988,9 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SRowMerger* pMerger = &pReader->status.merger; - bool asc = ASCENDING_TRAVERSE(pReader->order); - int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; - int32_t step = asc ? 1 : -1; + bool asc = ASCENDING_TRAVERSE(pReader->order); + int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + int32_t step = asc ? 1 : -1; pDumpInfo->rowIndex += step; if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) { @@ -3972,14 +4087,14 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, return terrno; } - tsdbRowMergerAdd(&pReader->status.merger,pNextRow, pTSchema1); + tsdbRowMergerAdd(&pReader->status.merger, pNextRow, pTSchema1); } else { // let's merge rows in file block code = tsdbRowMergerAdd(&pReader->status.merger, ¤t, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - tsdbRowMergerAdd(&pReader->status.merger,pNextRow, NULL); + tsdbRowMergerAdd(&pReader->status.merger, pNextRow, NULL); } code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, pReader); @@ -4026,9 +4141,8 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tsdbRowMergerAdd(&pReader->status.merger,pRow, pSchema); - code = - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4267,7 +4381,7 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n int32_t size = tSimpleHashGetSize(pReader->status.pTableMap); STableBlockScanInfo** p = NULL; - int32_t iter = 0; + int32_t iter = 0; while ((p = tSimpleHashIterate(pReader->status.pTableMap, p, &iter)) != NULL) { clearBlockScanInfo(*p); @@ -4354,16 +4468,18 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { } static void freeSchemaFunc(void* param) { - void **p = (void **)param; + void** p = (void**)param; taosMemoryFreeClear(*p); } // ====================================== EXPOSED APIs ====================================== -int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, - SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr, bool countOnly) { +int32_t tsdbReaderOpen(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, + SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly, + SHashObj** pIgnoreTables) { STimeWindow window = pCond->twindows; + SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); - int32_t capacity = pVnode->config.tsdbCfg.maxRows; + int32_t capacity = pConf->tsdbCfg.maxRows; if (pResBlock != NULL) { blockDataEnsureCapacity(pResBlock, capacity); } @@ -4394,7 +4510,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } // here we only need one more row, so the capacity is set to be ONE. - code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, pResBlock, idstr); + code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[0], 1, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -4408,7 +4524,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } pCond->order = order; - code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, pResBlock, idstr); + code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[1], 1, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -4458,7 +4574,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL goto _err; } - pReader->status.pLDataIter = taosMemoryCalloc(pVnode->config.sttTrigger, sizeof(SLDataIter)); + pReader->status.pLDataIter = taosMemoryCalloc(pConf->sttTrigger, sizeof(SLDataIter)); if (pReader->status.pLDataIter == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -4470,12 +4586,18 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL pReader->readMode = READ_MODE_COUNT_ONLY; } - tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); + pReader->pIgnoreTables = pIgnoreTables; + + tsdbDebug("%p total numOfTable:%d, window:%" PRId64 " - %" PRId64 ", verRange:%" PRId64 " - %" PRId64 + " in this query %s", + pReader, numOfTables, pReader->window.skey, pReader->window.ekey, pReader->verRange.minVer, + pReader->verRange.maxVer, pReader->idStr); + return code; _err: tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); - tsdbReaderClose(pReader); + tsdbReaderClose(*ppReader); *ppReader = NULL; // reset the pointer value. return code; } @@ -4624,7 +4746,7 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) { // resetDataBlockScanInfo excluding lastKey STableBlockScanInfo** p = NULL; - int32_t iter = 0; + int32_t iter = 0; while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; @@ -4646,7 +4768,7 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) { } else { // resetDataBlockScanInfo excluding lastKey STableBlockScanInfo** p = NULL; - int32_t iter = 0; + int32_t iter = 0; while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; @@ -4845,8 +4967,9 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { *hasNext = false; - if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT || pReader->code != TSDB_CODE_SUCCESS) { - return (pReader->code != TSDB_CODE_SUCCESS)? pReader->code:code; + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT || + pReader->code != TSDB_CODE_SUCCESS) { + return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code; } SReaderStatus* pStatus = &pReader->status; @@ -4942,7 +5065,8 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { return code; } -static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { +static bool doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { + bool hasNullSMA = false; // do fill all null column value SMA info int32_t i = 0, j = 0; int32_t size = (int32_t)taosArrayGetSize(pSup->pColAgg); @@ -4962,6 +5086,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ taosArrayInsert(pSup->pColAgg, i, &nullColAgg); i += 1; size++; + hasNullSMA = true; } j += 1; } @@ -4972,12 +5097,15 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; taosArrayInsert(pSup->pColAgg, i, &nullColAgg); i += 1; + hasNullSMA = true; } j++; } + + return hasNullSMA; } -int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) { +int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool* hasNullSMA) { SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg; int32_t code = 0; @@ -5041,7 +5169,10 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, } // do fill all null column value SMA info - doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg); + if (doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg)) { + *hasNullSMA = true; + return TSDB_CODE_SUCCESS; + } size_t size = taosArrayGetSize(pSup->pColAgg); int32_t i = 0, j = 0; @@ -5083,9 +5214,9 @@ STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, c } static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { - SReaderStatus* pStatus = &pReader->status; - int32_t code = TSDB_CODE_SUCCESS; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); + SReaderStatus* pStatus = &pReader->status; + int32_t code = TSDB_CODE_SUCCESS; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); if (pReader->code != TSDB_CODE_SUCCESS) { return NULL; @@ -5208,6 +5339,9 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { } static int32_t getBucketIndex(int32_t startRow, int32_t bucketRange, int32_t numOfRows, int32_t numOfBucket) { + if (numOfRows < startRow) { + return 0; + } int32_t bucketIndex = ((numOfRows - startRow) / bucketRange); if (bucketIndex == numOfBucket) { bucketIndex -= 1; @@ -5342,10 +5476,10 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { return rows; } -int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { +int32_t tsdbGetTableSchema(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { SMetaReader mr = {0}; - metaReaderInit(&mr, pVnode->pMeta, 0); - int32_t code = metaGetTableEntryByUidCache(&mr, uid); + metaReaderDoInit(&mr, ((SVnode*)pVnode)->pMeta, 0); + int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; metaReaderClear(&mr); @@ -5358,7 +5492,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 if (mr.me.type == TSDB_CHILD_TABLE) { tDecoderClear(&mr.coder); *suid = mr.me.ctbEntry.suid; - code = metaGetTableEntryByUidCache(&mr, *suid); + code = metaReaderGetTableEntryByUidCache(&mr, *suid); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; metaReaderClear(&mr); @@ -5374,7 +5508,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 metaReaderClear(&mr); // get the newest table schema version - code = metaGetTbTSchemaEx(pVnode->pMeta, *suid, uid, -1, pSchema); + code = metaGetTbTSchemaEx(((SVnode*)pVnode)->pMeta, *suid, uid, -1, pSchema); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c new file mode 100644 index 0000000000000000000000000000000000000000..57a649d682827c3e42c208922e097a7d4a23634e --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -0,0 +1,4962 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "osDef.h" +#include "tsdb.h" +#include "tsdbDataFileRW.h" +#include "tsdbFS2.h" +#include "tsdbMerge.h" +#include "tsdbReadUtil.h" +#include "tsdbUtil2.h" +#include "tsimplehash.h" + +#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +#define getCurrentKeyInLastBlock(_r) ((_r)->currentKey) + +static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); +static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, + STsdbReader* pReader); +static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); +static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader); +static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, + SRowMerger* pMerger, SVersionRange* pVerRange, const char* id); +static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, STsdbReader* pReader); +static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, + STableBlockScanInfo* pScanInfo); +static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, + int32_t rowIndex); +static void setComposedBlockFlag(STsdbReader* pReader, bool composed); +static bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, + SVersionRange* pVerRange); + +static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, + TSDBROW* pResRow, STsdbReader* pReader, bool* freeTSRow); +static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, + STsdbReader* pReader, SRow** pTSRow); +static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, + STsdbReader* pReader); + +static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SCostSummary* pCost); +static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr, + int8_t* pLevel); +static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); +static int32_t doBuildDataBlock(STsdbReader* pReader); +static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); +static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo); +static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter); +static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order); + +static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } + +static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, + int32_t numOfCols) { + pSupInfo->smaValid = true; + pSupInfo->numOfCols = numOfCols; + pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t) * 2 + POINTER_BYTES)); + if (pSupInfo->colId == NULL) { + taosMemoryFree(pSupInfo->colId); + return TSDB_CODE_OUT_OF_MEMORY; + } + + pSupInfo->slotId = (int16_t*)((char*)pSupInfo->colId + (sizeof(int16_t) * numOfCols)); + pSupInfo->buildBuf = (char**)((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols)); + for (int32_t i = 0; i < numOfCols; ++i) { + pSupInfo->colId[i] = pCols[i].colId; + pSupInfo->slotId[i] = pSlotIdList[i]; + + if (IS_VAR_DATA_TYPE(pCols[i].type)) { + pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes); + } else { + pSupInfo->buildBuf[i] = NULL; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) { + int32_t i = 0, j = 0; + + while (i < pSchema->numOfCols && j < pSupInfo->numOfCols) { + STColumn* pTCol = &pSchema->columns[i]; + if (pTCol->colId == pSupInfo->colId[j]) { + if (!IS_BSMA_ON(pTCol)) { + pSupInfo->smaValid = false; + return TSDB_CODE_SUCCESS; + } + + i += 1; + j += 1; + } else if (pTCol->colId < pSupInfo->colId[j]) { // do nothing + i += 1; + } else { + return TSDB_CODE_INVALID_PARA; + } + } + + return TSDB_CODE_SUCCESS; +} + +static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) { return pWindow->skey > pWindow->ekey; } + +// Update the query time window according to the data time to live(TTL) information, in order to avoid to return +// the expired data to client, even it is queried already. +static STimeWindow updateQueryTimeWindow(STsdb* pTsdb, STimeWindow* pWindow) { + STsdbKeepCfg* pCfg = &pTsdb->keepCfg; + + int64_t now = taosGetTimestamp(pCfg->precision); + int64_t earilyTs = now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick + + STimeWindow win = *pWindow; + if (win.skey < earilyTs) { + win.skey = earilyTs; + } + + return win; +} + +// init file iterator +static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetArray, STsdbReader* pReader) { + size_t numOfFileset = TARRAY2_SIZE(pFileSetArray); + + pIter->index = ASCENDING_TRAVERSE(pReader->info.order) ? -1 : numOfFileset; + pIter->order = pReader->info.order; + pIter->pFilesetList = pFileSetArray; + pIter->numOfFiles = numOfFileset; + + if (pIter->pLastBlockReader == NULL) { + pIter->pLastBlockReader = taosMemoryCalloc(1, sizeof(struct SLastBlockReader)); + if (pIter->pLastBlockReader == NULL) { + int32_t code = TSDB_CODE_OUT_OF_MEMORY; + tsdbError("failed to prepare the last block iterator, since:%s %s", tstrerror(code), pReader->idStr); + return code; + } + } + + SLastBlockReader* pLReader = pIter->pLastBlockReader; + pLReader->order = pReader->info.order; + pLReader->window = pReader->info.window; + pLReader->verRange = pReader->info.verRange; + + pLReader->uid = 0; + tMergeTreeClose(&pLReader->mergeTree); + tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); + return TSDB_CODE_SUCCESS; +} + +static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bool* hasNext) { + bool asc = ASCENDING_TRAVERSE(pIter->order); + int32_t step = asc ? 1 : -1; + int32_t code = 0; + + pIter->index += step; + if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) { + *hasNext = false; + return TSDB_CODE_SUCCESS; + } + + SCostSummary* pSum = &pReader->cost; + + pIter->pLastBlockReader->uid = 0; + tMergeTreeClose(&pIter->pLastBlockReader->mergeTree); + + pReader->status.pLDataIterArray = + destroySttBlockReader(pReader->status.pLDataIterArray, &pSum->lastBlockLoad, &pSum->lastBlockLoadTime); + pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + + // check file the time range of coverage + STimeWindow win = {0}; + + while (1) { + if (pReader->pFileReader != NULL) { + tsdbDataFileReaderClose(&pReader->pFileReader); + } + + pReader->status.pCurrentFileset = pIter->pFilesetList->data[pIter->index]; + + STFileObj** pFileObj = pReader->status.pCurrentFileset->farr; + if (pFileObj[0] != NULL || pFileObj[3] != NULL) { + SDataFileReaderConfig conf = {.tsdb = pReader->pTsdb, .szPage = pReader->pTsdb->pVnode->config.tsdbPageSize}; + + const char* filesName[4] = {0}; + + if (pFileObj[0] != NULL) { + conf.files[0].file = *pFileObj[0]->f; + conf.files[0].exist = true; + filesName[0] = pFileObj[0]->fname; + + conf.files[1].file = *pFileObj[1]->f; + conf.files[1].exist = true; + filesName[1] = pFileObj[1]->fname; + + conf.files[2].file = *pFileObj[2]->f; + conf.files[2].exist = true; + filesName[2] = pFileObj[2]->fname; + } + + if (pFileObj[3] != NULL) { + conf.files[3].exist = true; + conf.files[3].file = *pFileObj[3]->f; + filesName[3] = pFileObj[3]->fname; + } + + code = tsdbDataFileReaderOpen(filesName, &conf, &pReader->pFileReader); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + pReader->cost.headFileLoad += 1; + } + + int32_t fid = pReader->status.pCurrentFileset->fid; + tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey); + + // current file are no longer overlapped with query time window, ignore remain files + if ((asc && win.skey > pReader->info.window.ekey) || (!asc && win.ekey < pReader->info.window.skey)) { + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pReader, + pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); + *hasNext = false; + return TSDB_CODE_SUCCESS; + } + + if ((asc && (win.ekey < pReader->info.window.skey)) || ((!asc) && (win.skey > pReader->info.window.ekey))) { + pIter->index += step; + if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) { + *hasNext = false; + return TSDB_CODE_SUCCESS; + } + continue; + } + + tsdbDebug("%p file found fid:%d for qrange:%" PRId64 "-%" PRId64 ", %s", pReader, fid, pReader->info.window.skey, + pReader->info.window.ekey, pReader->idStr); + *hasNext = true; + return TSDB_CODE_SUCCESS; + } + +_err: + *hasNext = false; + return code; +} + +static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order) { + pIter->order = order; + pIter->index = -1; + pIter->numOfBlocks = 0; + if (pIter->blockList == NULL) { + pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); + } else { + taosArrayClear(pIter->blockList); + } +} + +static void cleanupDataBlockIterator(SDataBlockIter* pIter) { taosArrayDestroy(pIter->blockList); } + +static void initReaderStatus(SReaderStatus* pStatus) { + pStatus->pTableIter = NULL; + pStatus->loadFromFile = true; +} + +static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity) { + SSDataBlock* pResBlock = createDataBlock(); + if (pResBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for (int32_t i = 0; i < pCond->numOfCols; ++i) { + SColumnInfoData colInfo = {0}; + colInfo.info = pCond->colList[i]; + blockDataAppendColInfo(pResBlock, &colInfo); + } + + int32_t code = blockDataEnsureCapacity(pResBlock, capacity); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + taosMemoryFree(pResBlock); + return NULL; + } + return pResBlock; +} + +static int32_t tsdbInitReaderLock(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexInit(&pReader->readerMutex, NULL); + + qTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +static int32_t tsdbUninitReaderLock(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexDestroy(&pReader->readerMutex); + + qTrace("tsdb/read: %p, post-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +static int32_t tsdbAcquireReader(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexLock(&pReader->readerMutex); + + qTrace("tsdb/read: %p, post-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +static int32_t tsdbTryAcquireReader(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexTryLock(&pReader->readerMutex); + + qTrace("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +static int32_t tsdbReleaseReader(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-untake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexUnlock(&pReader->readerMutex); + + qTrace("tsdb/read: %p, post-untake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +void tsdbReleaseDataBlock2(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + if (!pStatus->composedDataBlock) { + tsdbReleaseReader(pReader); + } +} + +static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacity, SSDataBlock* pResBlock, + SQueryTableDataCond* pCond) { + pResBlockInfo->capacity = capacity; + pResBlockInfo->pResBlock = pResBlock; + terrno = 0; + + if (pResBlockInfo->pResBlock == NULL) { + pResBlockInfo->freeBlock = true; + pResBlockInfo->pResBlock = createResBlock(pCond, pResBlockInfo->capacity); + } else { + pResBlockInfo->freeBlock = false; + } + + return terrno; +} + +static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void** ppReader, int32_t capacity, + SSDataBlock* pResBlock, const char* idstr) { + int32_t code = 0; + int8_t level = 0; + STsdbReader* pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader)); + if (pReader == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (VND_IS_TSMA(pVnode)) { + tsdbDebug("vgId:%d, tsma is selected to query, %s", TD_VID(pVnode), idstr); + } + + initReaderStatus(&pReader->status); + + pReader->pTsdb = getTsdbByRetentions(pVnode, pCond->twindows.skey, pVnode->config.tsdbCfg.retentions, idstr, &level); + pReader->info.suid = pCond->suid; + pReader->info.order = pCond->order; + + pReader->idStr = (idstr != NULL) ? taosStrdup(idstr) : NULL; + pReader->info.verRange = getQueryVerRange(pVnode, pCond, level); + pReader->type = pCond->type; + pReader->info.window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); + pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket + + code = initResBlockInfo(&pReader->resBlockInfo, capacity, pResBlock, pCond); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + + if (pCond->numOfCols <= 0) { + tsdbError("vgId:%d, invalid column number %d in query cond, %s", TD_VID(pVnode), pCond->numOfCols, idstr); + code = TSDB_CODE_INVALID_PARA; + goto _end; + } + + // allocate buffer in order to load data blocks from file + SBlockLoadSuppInfo* pSup = &pReader->suppInfo; + pSup->tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; + setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); + + code = tBlockDataCreate(&pReader->status.fileBlockData); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + goto _end; + } + + if (pReader->suppInfo.colId[0] != PRIMARYKEY_TIMESTAMP_COL_ID) { + tsdbError("the first column isn't primary timestamp, %d, %s", pReader->suppInfo.colId[0], pReader->idStr); + code = TSDB_CODE_INVALID_PARA; + goto _end; + } + + pReader->status.pPrimaryTsCol = taosArrayGet(pReader->resBlockInfo.pResBlock->pDataBlock, pSup->slotId[0]); + int32_t type = pReader->status.pPrimaryTsCol->info.type; + if (type != TSDB_DATA_TYPE_TIMESTAMP) { + tsdbError("the first column isn't primary timestamp in result block, actual: %s, %s", tDataTypes[type].name, + pReader->idStr); + code = TSDB_CODE_INVALID_PARA; + goto _end; + } + + tsdbInitReaderLock(pReader); + + *ppReader = pReader; + return code; + +_end: + tsdbReaderClose2(pReader); + *ppReader = NULL; + return code; +} + +static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileReader, SArray* pIndexList) { + int64_t st = taosGetTimestampUs(); + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + if (pFileReader == NULL) { + return TSDB_CODE_SUCCESS; + } + + const TBrinBlkArray* pBlkArray = NULL; + + int32_t code = tsdbDataFileReadBrinBlk(pFileReader, &pBlkArray); + +#if 0 + LRUHandle* handle = NULL; + + int32_t code = tsdbCacheGetBlockIdx(pFileReader->pTsdb->biCache, pFileReader, &handle); + if (code != TSDB_CODE_SUCCESS || handle == NULL) { + goto _end; + } + + + SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle); + size_t num = taosArrayGetSize(aBlockIdx); + if (num == 0) { + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); + return TSDB_CODE_SUCCESS; + } +#endif + + // todo binary search to the start position + int64_t et1 = taosGetTimestampUs(); + + SBrinBlk* pBrinBlk = NULL; + STableUidList* pList = &pReader->status.uidList; + + int32_t i = 0; + + while (i < TARRAY2_SIZE(pBlkArray)) { + pBrinBlk = &pBlkArray->data[i]; + if (pBrinBlk->maxTbid.suid < pReader->info.suid) { + i += 1; + continue; + } + + if (pBrinBlk->minTbid.suid > pReader->info.suid) { // not include the queried table/super table, quit the loop + break; + } + + ASSERT(pBrinBlk->minTbid.suid <= pReader->info.suid && pBrinBlk->maxTbid.suid >= pReader->info.suid); + if (pBrinBlk->maxTbid.suid == pReader->info.suid && pBrinBlk->maxTbid.uid < pList->tableUidList[0]) { + i += 1; + continue; + } + + if (pBrinBlk->minTbid.suid == pReader->info.suid && pBrinBlk->minTbid.uid > pList->tableUidList[numOfTables - 1]) { + break; + } + + taosArrayPush(pIndexList, pBrinBlk); + i += 1; + } + + int64_t et2 = taosGetTimestampUs(); + tsdbDebug("load block index for %d/%d tables completed, elapsed time:%.2f ms, set BrinBlk:%.2f ms, size:%.2f Kb %s", + numOfTables, (int32_t)pBlkArray->size, (et1 - st) / 1000.0, (et2 - et1) / 1000.0, + pBlkArray->size * sizeof(SBrinBlk) / 1024.0, pReader->idStr); + + pReader->cost.headFileLoadTime += (et1 - st) / 1000.0; + +_end: + // tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); + return code; +} + +static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum, + SArray* pTableScanInfoList) { + size_t sizeInDisk = 0; + int64_t st = taosGetTimestampUs(); + + // clear info for the new file + cleanupInfoFoxNextFileset(pReader->status.pTableMap); + + int32_t k = 0; + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; + STimeWindow w = pReader->info.window; + SBrinRecord* pRecord = NULL; + + SBrinRecordIter iter = {0}; + initBrinRecordIter(&iter, pReader->pFileReader, pIndexList); + + while (((pRecord = getNextBrinRecord(&iter)) != NULL)) { + if (pRecord->suid > pReader->info.suid) { + break; + } + + uint64_t uid = pReader->status.uidList.tableUidList[k]; + if (pRecord->suid < pReader->info.suid) { + continue; + } + + if (uid < pRecord->uid) { // forward the table uid index + while (k < numOfTables && pReader->status.uidList.tableUidList[k] < pRecord->uid) { + k += 1; + } + + if (k >= numOfTables) { + break; + } + + uid = pReader->status.uidList.tableUidList[k]; + } + + if (pRecord->uid < uid) { + continue; + } + + ASSERT(pRecord->suid == pReader->info.suid && uid == pRecord->uid); + + STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + if (ASCENDING_TRAVERSE(pReader->info.order)) { + w.skey = pScanInfo->lastKey + step; + } else { + w.ekey = pScanInfo->lastKey + step; + } + + if (isEmptyQueryTimeWindow(&w)) { + k += 1; + continue; + } + + // 1. time range check + if (pRecord->firstKey > w.ekey || pRecord->lastKey < w.skey) { + continue; + } + + // 2. version range check + if (pRecord->minVer > pReader->info.verRange.maxVer || pRecord->maxVer < pReader->info.verRange.minVer) { + continue; + } + + if (pScanInfo->pBlockList == NULL) { + pScanInfo->pBlockList = taosArrayInit(4, sizeof(SBrinRecord)); + } + + void* p1 = taosArrayPush(pScanInfo->pBlockList, pRecord); + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pBlockNum->numOfBlocks += 1; + if (taosArrayGetSize(pTableScanInfoList) == 0) { + taosArrayPush(pTableScanInfoList, &pScanInfo); + } else { + STableBlockScanInfo** p = taosArrayGetLast(pTableScanInfoList); + if ((*p)->uid != uid) { + taosArrayPush(pTableScanInfoList, &pScanInfo); + } + } + } + + clearBrinBlockIter(&iter); + + pBlockNum->numOfLastFiles = pReader->status.pCurrentFileset->lvlArr->size; + int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks; + + double el = (taosGetTimestampUs() - st) / 1000.0; + tsdbDebug( + "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " + "time:%.2f ms %s", + numOfTables, pBlockNum->numOfBlocks, (int32_t)taosArrayGetSize(pTableScanInfoList), pBlockNum->numOfLastFiles, + sizeInDisk / 1000.0, el, pReader->idStr); + + pReader->cost.numOfBlocks += total; + pReader->cost.headFileLoadTime += el; + + return TSDB_CODE_SUCCESS; +} + +static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, int64_t maxKey, int32_t order) { + int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1; + pDumpInfo->allDumped = true; + pDumpInfo->lastKey = maxKey + step; +} + +static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, + SBlockLoadSuppInfo* pSup) { + if (IS_VAR_DATA_TYPE(pColVal->type)) { + if (!COL_VAL_IS_VALUE(pColVal)) { + colDataSetNULL(pColInfoData, rowIndex); + } else { + varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData); + if (pColVal->value.nData > pColInfoData->info.bytes) { + tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, pColVal->value.nData, + pColInfoData->info.bytes); + return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + } + if (pColVal->value.nData > 0) { // pData may be null, if nData is 0 + memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); + } + + colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); + } + } else { + colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value, !COL_VAL_IS_VALUE(pColVal)); + } + + return TSDB_CODE_SUCCESS; +} + +static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { + size_t num = TARRAY_SIZE(pBlockIter->blockList); + if (num == 0) { + ASSERT(pBlockIter->numOfBlocks == num); + return NULL; + } + + SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); + return pBlockInfo; +} + +static int doBinarySearchKey(TSKEY* keyList, int num, int pos, TSKEY key, int order) { + // start end position + int s, e; + s = pos; + + // check + ASSERT(pos >= 0 && pos < num && num > 0); + if (order == TSDB_ORDER_ASC) { + // find the first position which is smaller than the key + e = num - 1; + if (key < keyList[pos]) return -1; + while (1) { + // check can return + if (key >= keyList[e]) return e; + if (key <= keyList[s]) return s; + if (e - s <= 1) return s; + + // change start or end position + int mid = s + (e - s + 1) / 2; + if (keyList[mid] > key) + e = mid; + else if (keyList[mid] < key) + s = mid; + else + return mid; + } + } else { // DESC + // find the first position which is bigger than the key + e = 0; + if (key > keyList[pos]) return -1; + while (1) { + // check can return + if (key <= keyList[e]) return e; + if (key >= keyList[s]) return s; + if (s - e <= 1) return s; + + // change start or end position + int mid = s - (s - e + 1) / 2; + if (keyList[mid] < key) + e = mid; + else if (keyList[mid] > key) + s = mid; + else + return mid; + } + } +} + +static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SBrinRecord* pRecord, int32_t pos) { + // NOTE: reverse the order to find the end position in data block + int32_t endPos = -1; + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + + if (asc && pReader->info.window.ekey >= pRecord->lastKey) { + endPos = pRecord->numRow - 1; + } else if (!asc && pReader->info.window.skey <= pRecord->firstKey) { + endPos = 0; + } else { + int64_t key = asc ? pReader->info.window.ekey : pReader->info.window.skey; + endPos = doBinarySearchKey(pBlockData->aTSKEY, pRecord->numRow, pos, key, pReader->info.order); + } + + if ((pReader->info.verRange.maxVer >= pRecord->minVer && pReader->info.verRange.maxVer < pRecord->maxVer) || + (pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.minVer > pRecord->minVer)) { + int32_t i = endPos; + + if (asc) { + for (; i >= 0; --i) { + if (pBlockData->aVersion[i] <= pReader->info.verRange.maxVer) { + break; + } + } + } else { + for (; i < pRecord->numRow; ++i) { + if (pBlockData->aVersion[i] >= pReader->info.verRange.minVer) { + break; + } + } + } + + endPos = i; + } + + return endPos; +} + +static void copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, + int32_t dumpedRows, bool asc) { + if (asc) { + memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t)); + } else { + int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1; + memcpy(pColData->pData, &pBlockData->aTSKEY[startIndex], dumpedRows * sizeof(int64_t)); + + // todo: opt perf by extract the loop + // reverse the array list + int32_t mid = dumpedRows >> 1u; + int64_t* pts = (int64_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + } +} + +// a faster version of copy procedure. +static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, + int32_t dumpedRows, bool asc) { + uint8_t* p = NULL; + if (asc) { + p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; + } else { + int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1; + p = pData->pData + tDataTypes[pData->type].bytes * startIndex; + } + + int32_t step = asc ? 1 : -1; + + // make sure it is aligned to 8bit, the allocated memory address is aligned to 256bit + // ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); + + // 1. copy data in a batch model + memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); + + // 2. reverse the array list in case of descending order scan data block + if (!asc) { + switch (pColData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: { + int32_t mid = dumpedRows >> 1u; + int64_t* pts = (int64_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: { + int32_t mid = dumpedRows >> 1u; + int8_t* pts = (int8_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int8_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: { + int32_t mid = dumpedRows >> 1u; + int16_t* pts = (int16_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: { + int32_t mid = dumpedRows >> 1u; + int32_t* pts = (int32_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int32_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + } + } + + // 3. if the null value exists, check items one-by-one + if (pData->flag != HAS_VALUE) { + int32_t rowIndex = 0; + + for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step, rowIndex++) { + uint8_t v = tColDataGetBitValue(pData, j); + if (v == 0 || v == 1) { + colDataSetNull_f(pColData->nullbitmap, rowIndex); + pColData->hasNull = true; + } + } + } +} + +static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + SBlockData* pBlockData = &pStatus->fileBlockData; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + int32_t numOfOutputCols = pSupInfo->numOfCols; + int32_t code = TSDB_CODE_SUCCESS; + + SColVal cv = {0}; + int64_t st = taosGetTimestampUs(); + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + int32_t step = asc ? 1 : -1; + + SBrinRecord* pRecord = &pBlockInfo->record; + + // no data exists, return directly. + if (pBlockData->nRow == 0 || pBlockData->aTSKEY == 0) { + tsdbWarn("%p no need to copy since no data in blockData, table uid:%" PRIu64 " has been dropped, %s", pReader, + pBlockInfo->uid, pReader->idStr); + pResBlock->info.rows = 0; + return 0; + } + + // row index of dump info remain the initial position, let's find the appropriate start position. + if ((pDumpInfo->rowIndex == 0 && asc) || (pDumpInfo->rowIndex == pRecord->numRow - 1 && (!asc))) { + if (asc && pReader->info.window.skey <= pRecord->firstKey && pReader->info.verRange.minVer <= pRecord->minVer) { + // pDumpInfo->rowIndex = 0; + } else if (!asc && pReader->info.window.ekey >= pRecord->lastKey && + pReader->info.verRange.maxVer >= pRecord->maxVer) { + // pDumpInfo->rowIndex = pRecord->numRow - 1; + } else { // find the appropriate the start position in current block, and set it to be the current rowIndex + int32_t pos = asc ? pRecord->numRow - 1 : 0; + int32_t order = asc ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; + int64_t key = asc ? pReader->info.window.skey : pReader->info.window.ekey; + pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pRecord->numRow, pos, key, order); + + if (pDumpInfo->rowIndex < 0) { + tsdbError( + "%p failed to locate the start position in current block, global index:%d, table index:%d, brange:%" PRId64 + "-%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 " %s", + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pRecord->firstKey, pRecord->lastKey, pRecord->minVer, + pRecord->maxVer, pReader->idStr); + return TSDB_CODE_INVALID_PARA; + } + + ASSERT(pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.maxVer >= pRecord->minVer); + + // find the appropriate start position that satisfies the version requirement. + if ((pReader->info.verRange.maxVer >= pRecord->minVer && pReader->info.verRange.maxVer < pRecord->maxVer) || + (pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.minVer > pRecord->minVer)) { + int32_t i = pDumpInfo->rowIndex; + if (asc) { + for (; i < pRecord->numRow; ++i) { + if (pBlockData->aVersion[i] >= pReader->info.verRange.minVer) { + break; + } + } + } else { + for (; i >= 0; --i) { + if (pBlockData->aVersion[i] <= pReader->info.verRange.maxVer) { + break; + } + } + } + + pDumpInfo->rowIndex = i; + } + } + } + + // time window check + int32_t endIndex = getEndPosInDataBlock(pReader, pBlockData, pRecord, pDumpInfo->rowIndex); + if (endIndex == -1) { + setBlockAllDumped(pDumpInfo, pReader->info.window.ekey, pReader->info.order); + return TSDB_CODE_SUCCESS; + } + + endIndex += step; + int32_t dumpedRows = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex); + if (dumpedRows > pReader->resBlockInfo.capacity) { // output buffer check + dumpedRows = pReader->resBlockInfo.capacity; + } else if (dumpedRows <= 0) { // no qualified rows in current data block, abort directly. + setBlockAllDumped(pDumpInfo, pReader->info.window.ekey, pReader->info.order); + return TSDB_CODE_SUCCESS; + } + + int32_t i = 0; + int32_t rowIndex = 0; + + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { + copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc); + i += 1; + } + + int32_t colIndex = 0; + int32_t num = pBlockData->nColData; + while (i < numOfOutputCols && colIndex < num) { + rowIndex = 0; + + SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex); + if (pData->cid < pSupInfo->colId[i]) { + colIndex += 1; + } else if (pData->cid == pSupInfo->colId[i]) { + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + + if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { + colDataSetNNULL(pColData, 0, dumpedRows); + } else { + if (IS_MATHABLE_TYPE(pColData->info.type)) { + copyNumericCols(pData, pDumpInfo, pColData, dumpedRows, asc); + } else { // varchar/nchar type + for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) { + tColDataGetValue(pData, j, &cv); + code = doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); + if (code) { + return code; + } + } + } + } + + colIndex += 1; + i += 1; + } else { // the specified column does not exist in file block, fill with null data + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + colDataSetNNULL(pColData, 0, dumpedRows); + i += 1; + } + } + + // fill the mis-matched columns with null value + while (i < numOfOutputCols) { + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + colDataSetNNULL(pColData, 0, dumpedRows); + i += 1; + } + + pResBlock->info.dataLoad = 1; + pResBlock->info.rows = dumpedRows; + pDumpInfo->rowIndex += step * dumpedRows; + + // check if current block are all handled + if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pRecord->numRow) { + int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + if (outOfTimeWindow(ts, + &pReader->info.window)) { // the remain data has out of query time window, ignore current block + setBlockAllDumped(pDumpInfo, ts, pReader->info.order); + } + } else { + int64_t ts = asc ? pRecord->lastKey : pRecord->firstKey; + setBlockAllDumped(pDumpInfo, ts, pReader->info.order); + } + + double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; + pReader->cost.blockLoadTime += elapsedTime; + + int32_t unDumpedRows = asc ? pRecord->numRow - pDumpInfo->rowIndex : pDumpInfo->rowIndex + 1; + tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 + ", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", uid:%" PRIu64 " elapsed time:%.2f ms, %s", + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pRecord->firstKey, pRecord->lastKey, dumpedRows, + unDumpedRows, pRecord->minVer, pRecord->maxVer, pBlockInfo->uid, elapsedTime, pReader->idStr); + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE STSchema* getTableSchemaImpl(STsdbReader* pReader, uint64_t uid) { + ASSERT(pReader->info.pSchema == NULL); + + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, -1, &pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS || pReader->info.pSchema == NULL) { + terrno = code; + tsdbError("failed to get table schema, uid:%" PRIu64 ", it may have been dropped, ver:-1, %s", uid, pReader->idStr); + return NULL; + } + + code = tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + tsdbError("failed to init merger, code:%s, %s", tstrerror(code), pReader->idStr); + return NULL; + } + + return pReader->info.pSchema; +} + +static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData, + uint64_t uid) { + int32_t code = 0; + STSchema* pSchema = pReader->info.pSchema; + int64_t st = taosGetTimestampUs(); + + tBlockDataReset(pBlockData); + + if (pReader->info.pSchema == NULL) { + pSchema = getTableSchemaImpl(pReader, uid); + if (pSchema == NULL) { + tsdbDebug("%p table uid:%" PRIu64 " has been dropped, no data existed, %s", pReader, uid, pReader->idStr); + return code; + } + } + + SBlockLoadSuppInfo* pSup = &pReader->suppInfo; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + SBrinRecord* pRecord = &pBlockInfo->record; + code = tsdbDataFileReadBlockDataByColumn(pReader->pFileReader, pRecord, pBlockData, pSchema, &pSup->colId[1], + pSup->numOfCols - 1); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 + ", rows:%d, code:%s %s", + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->record.firstKey, + pBlockInfo->record.lastKey, pBlockInfo->record.numRow, tstrerror(code), pReader->idStr); + return code; + } + + double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; + + tsdbDebug("%p load file block into buffer, global index:%d, index in table block list:%d, brange:%" PRId64 "-%" PRId64 + ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pRecord->firstKey, pRecord->lastKey, pRecord->numRow, + pRecord->minVer, pRecord->maxVer, elapsedTime, pReader->idStr); + + pReader->cost.blockLoadTime += elapsedTime; + pDumpInfo->allDumped = false; + + return TSDB_CODE_SUCCESS; +} + +/** + * This is an two rectangles overlap cases. + */ +static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SFileDataBlockInfo* pBlock) { + return (pWindow->ekey < pBlock->record.lastKey && pWindow->ekey >= pBlock->record.firstKey) || + (pWindow->skey > pBlock->record.firstKey && pWindow->skey <= pBlock->record.lastKey) || + (pVerRange->minVer > pBlock->record.minVer && pVerRange->minVer <= pBlock->record.maxVer) || + (pVerRange->maxVer < pBlock->record.maxVer && pVerRange->maxVer >= pBlock->record.minVer); +} + +static bool getNeighborBlockOfSameTable(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, + int32_t* nextIndex, int32_t order, SBrinRecord* pRecord) { + bool asc = ASCENDING_TRAVERSE(order); + if (asc && pBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockList) - 1) { + return false; + } + + if (!asc && pBlockInfo->tbBlockIdx == 0) { + return false; + } + + int32_t step = asc ? 1 : -1; + // *nextIndex = pBlockInfo->tbBlockIdx + step; + // *pBlockIndex = *(SBlockIndex*)taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); + SBrinRecord* p = taosArrayGet(pTableBlockScanInfo->pBlockList, pBlockInfo->tbBlockIdx + step); + memcpy(pRecord, p, sizeof(SBrinRecord)); + + *nextIndex = pBlockInfo->tbBlockIdx + step; + + // tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, pIndex->ordinalIndex, pBlock, tGetDataBlk); + return true; +} + +static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pFBlockInfo) { + int32_t step = ASCENDING_TRAVERSE(pBlockIter->order) ? 1 : -1; + int32_t index = pBlockIter->index; + + while (index < pBlockIter->numOfBlocks && index >= 0) { + SFileDataBlockInfo* pFBlock = taosArrayGet(pBlockIter->blockList, index); + if (pFBlock->uid == pFBlockInfo->uid && pFBlock->tbBlockIdx == pFBlockInfo->tbBlockIdx) { + return index; + } + + index += step; + } + + return -1; +} + +static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t index, int32_t step) { + if (index < 0 || index >= pBlockIter->numOfBlocks) { + return -1; + } + + SFileDataBlockInfo fblock = *(SFileDataBlockInfo*)taosArrayGet(pBlockIter->blockList, index); + pBlockIter->index += step; + + if (index != pBlockIter->index) { + taosArrayRemove(pBlockIter->blockList, index); + taosArrayInsert(pBlockIter->blockList, pBlockIter->index, &fblock); + + SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); + ASSERT(pBlockInfo->uid == fblock.uid && pBlockInfo->tbBlockIdx == fblock.tbBlockIdx); + } + + return TSDB_CODE_SUCCESS; +} + +// todo: this attribute could be acquired during extractin the global ordered block list. +static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order) { + // it is the last block in current file, no chance to overlap with neighbor blocks. + if (ASCENDING_TRAVERSE(order)) { + return pBlock->record.lastKey == pRec->firstKey; + } else { + return pBlock->record.firstKey == pRec->lastKey; + } +} + +static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SFileDataBlockInfo* pBlock) { + bool ascScan = ASCENDING_TRAVERSE(order); + + return (ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts <= pBlock->record.firstKey)) || + (!ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts >= pBlock->record.lastKey)); +} + +static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersionRange* pVerRange) { + return (key.ts >= pBlock->record.firstKey && key.ts <= pBlock->record.lastKey) && + (pBlock->record.maxVer >= pVerRange->minVer) && (pBlock->record.minVer <= pVerRange->maxVer); +} + +static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, + int32_t startIndex) { + size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); + + for (int32_t i = startIndex; i < num; i += 1) { + TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i); + if (p->ts >= pRecord->firstKey && p->ts <= pRecord->lastKey) { + if (p->version >= pRecord->minVer) { + return true; + } + } else if (p->ts < pRecord->firstKey) { // p->ts < pBlock->minKey.ts + if (p->version >= pRecord->minVer) { + if (i < num - 1) { + TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1); + if (pnext->ts >= pRecord->firstKey) { + return true; + } + } else { // it must be the last point + ASSERT(p->version == 0); + } + } + } else { // (p->ts > pBlock->maxKey.ts) { + return false; + } + } + + return false; +} + +static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order) { + if (pBlockScanInfo->delSkyline == NULL || (taosArrayGetSize(pBlockScanInfo->delSkyline) == 0)) { + return false; + } + + // ts is not overlap + TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0); + TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline); + if (pRecord->firstKey > pLast->ts || pRecord->lastKey < pFirst->ts) { + return false; + } + + // version is not overlap + if (ASCENDING_TRAVERSE(order)) { + return doCheckforDatablockOverlap(pBlockScanInfo, pRecord, pBlockScanInfo->fileDelIndex); + } else { + int32_t index = pBlockScanInfo->fileDelIndex; + while (1) { + TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index); + if (p->ts > pRecord->firstKey && index > 0) { + index -= 1; + } else { // find the first point that is smaller than the minKey.ts of dataBlock. + if (p->ts == pRecord->firstKey && p->version < pRecord->maxVer && index > 0) { + index -= 1; + } + break; + } + } + + return doCheckforDatablockOverlap(pBlockScanInfo, pRecord, index); + } +} + +typedef struct { + bool overlapWithNeighborBlock; + bool hasDupTs; + bool overlapWithDelInfo; + bool overlapWithLastBlock; + bool overlapWithKeyInBuf; + bool partiallyRequired; + bool moreThanCapcity; +} SDataBlockToLoadInfo; + +static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo, + STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader, + STsdbReader* pReader) { + int32_t neighborIndex = 0; + SBrinRecord rec = {0}; + + bool hasNeighbor = getNeighborBlockOfSameTable(pBlockInfo, pScanInfo, &neighborIndex, pReader->info.order, &rec); + + // overlap with neighbor + if (hasNeighbor) { + pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock2(pBlockInfo, &rec, pReader->info.order); + } + + // has duplicated ts of different version in this block + pInfo->hasDupTs = (pBlockInfo->record.numRow > pBlockInfo->record.count); + pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pBlockInfo->record, pReader->info.order); + + if (hasDataInLastBlock(pLastBlockReader)) { + int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + pInfo->overlapWithLastBlock = !(pBlockInfo->record.lastKey < tsLast || pBlockInfo->record.firstKey > tsLast); + } + + pInfo->moreThanCapcity = pBlockInfo->record.numRow > pReader->resBlockInfo.capacity; + pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->info.window, &pReader->info.verRange, pBlockInfo); + pInfo->overlapWithKeyInBuf = keyOverlapFileBlock(keyInBuf, pBlockInfo, &pReader->info.verRange); +} + +// 1. the version of all rows should be less than the endVersion +// 2. current block should not overlap with next neighbor block +// 3. current timestamp should not be overlap with each other +// 4. output buffer should be large enough to hold all rows in current block +// 5. delete info should not overlap with current block data +// 6. current block should not contain the duplicated ts +static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, + TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader) { + SDataBlockToLoadInfo info = {0}; + getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pLastBlockReader, pReader); + + bool loadDataBlock = + (info.overlapWithNeighborBlock || info.hasDupTs || info.partiallyRequired || info.overlapWithKeyInBuf || + info.moreThanCapcity || info.overlapWithDelInfo || info.overlapWithLastBlock); + + // log the reason why load the datablock for profile + if (loadDataBlock) { + tsdbDebug("%p uid:%" PRIu64 + " need to load the datablock, overlapneighbor:%d, hasDup:%d, partiallyRequired:%d, " + "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s", + pReader, pBlockInfo->uid, info.overlapWithNeighborBlock, info.hasDupTs, info.partiallyRequired, + info.overlapWithKeyInBuf, info.moreThanCapcity, info.overlapWithDelInfo, info.overlapWithLastBlock, + pReader->idStr); + } + + return loadDataBlock; +} + +static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, + TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader) { + SDataBlockToLoadInfo info = {0}; + getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pLastBlockReader, pReader); + bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf || + info.overlapWithDelInfo || info.overlapWithLastBlock); + return isCleanFileBlock; +} + +static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, int64_t endKey) { + if (!(pBlockScanInfo->iiter.hasVal || pBlockScanInfo->iter.hasVal)) { + return TSDB_CODE_SUCCESS; + } + + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + + int64_t st = taosGetTimestampUs(); + int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->resBlockInfo.capacity, pReader); + + blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotId[0]); + pBlock->info.id.uid = pBlockScanInfo->uid; + + setComposedBlockFlag(pReader, true); + + double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; + tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%" PRId64 ", brange:%" PRId64 + " - %" PRId64 ", uid:%" PRIu64 ", %s", + pReader, elapsedTime, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, + pBlockScanInfo->uid, pReader->idStr); + + pReader->cost.buildmemBlock += elapsedTime; + return code; +} + +static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pBlockData, int64_t key, + SFileBlockDumpInfo* pDumpInfo, bool* copied) { + // opt version + // 1. it is not a border point + // 2. the direct next point is not an duplicated timestamp + int32_t code = TSDB_CODE_SUCCESS; + + *copied = false; + bool asc = (pReader->info.order == TSDB_ORDER_ASC); + if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) { + int32_t step = pReader->info.order == TSDB_ORDER_ASC ? 1 : -1; + + int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; + if (nextKey != key) { // merge is not needed + code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); + if (code) { + return code; + } + pDumpInfo->rowIndex += step; + *copied = true; + } + } + + return code; +} + +static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, + SVersionRange* pVerRange) { + int32_t step = ASCENDING_TRAVERSE(pLastBlockReader->order) ? 1 : -1; + + while (1) { + bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); + if (!hasVal) { // the next value will be the accessed key in stt + pScanInfo->lastKeyInStt += step; + return false; + } + + TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + int64_t key = pRow->pBlockData->aTSKEY[pRow->iRow]; + int64_t ver = pRow->pBlockData->aVersion[pRow->iRow]; + + pLastBlockReader->currentKey = key; + pScanInfo->lastKeyInStt = key; + + if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order, + pVerRange)) { + return true; + } + } +} + +static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLastBlockReader, + STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader, + bool* copied) { + int32_t code = TSDB_CODE_SUCCESS; + + *copied = false; + + bool hasVal = nextRowFromLastBlocks(pLastBlockReader, pScanInfo, &pReader->info.verRange); + if (hasVal) { + int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); + if (next1 != ts) { + code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow); + if (code) { + return code; + } + + *copied = true; + return code; + } + } else { + code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow); + if (code) { + return code; + } + + *copied = true; + return code; + } + + return code; +} + +static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { + // always set the newest schema version in pReader->info.pSchema + if (pReader->info.pSchema == NULL) { + STSchema* ps = getTableSchemaImpl(pReader, uid); + if (ps == NULL) { + return NULL; + } + } + + if (pReader->info.pSchema && sversion == pReader->info.pSchema->version) { + return pReader->info.pSchema; + } + + void** p = tSimpleHashGet(pReader->pSchemaMap, &sversion, sizeof(sversion)); + if (p != NULL) { + return *(STSchema**)p; + } + + STSchema* ptr = NULL; + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, sversion, &ptr); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } else { + code = tSimpleHashPut(pReader->pSchemaMap, &sversion, sizeof(sversion), &ptr, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + return ptr; + } +} + +static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, + SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { + SRowMerger* pMerger = &pReader->status.merger; + SRow* pTSRow = NULL; + SBlockData* pBlockData = &pReader->status.fileBlockData; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + int64_t tsLast = INT64_MIN; + if (hasDataInLastBlock(pLastBlockReader)) { + tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + } + + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + + // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized + if (pMerger->pArray == NULL) { + ASSERT(pReader->info.pSchema == NULL); + STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); + if (ps == NULL) { + return terrno; + } + } + + int64_t minKey = 0; + if (pReader->info.order == TSDB_ORDER_ASC) { + minKey = INT64_MAX; // chosen the minimum value + if (minKey > tsLast && hasDataInLastBlock(pLastBlockReader)) { + minKey = tsLast; + } + + if (minKey > k.ts) { + minKey = k.ts; + } + + if (minKey > key && hasDataInFileBlock(pBlockData, pDumpInfo)) { + minKey = key; + } + } else { + minKey = INT64_MIN; + if (minKey < tsLast && hasDataInLastBlock(pLastBlockReader)) { + minKey = tsLast; + } + + if (minKey < k.ts) { + minKey = k.ts; + } + + if (minKey < key && hasDataInFileBlock(pBlockData, pDumpInfo)) { + minKey = key; + } + } + + // todo remove init + bool init = false; + + // ASC: file block ---> last block -----> imem -----> mem + // DESC: mem -----> imem -----> last block -----> file block + if (pReader->info.order == TSDB_ORDER_ASC) { + if (minKey == key) { + init = true; + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + } + + if (minKey == tsLast) { + TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + if (init) { + tsdbRowMergerAdd(pMerger, fRow1, NULL); + } else { + init = true; + int32_t code = tsdbRowMergerAdd(pMerger, fRow1, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, + pReader->idStr); + } + + if (minKey == k.ts) { + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return terrno; + } + if (init) { + tsdbRowMergerAdd(pMerger, pRow, pSchema); + } else { + init = true; + int32_t code = tsdbRowMergerAdd(pMerger, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + int32_t code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } else { + if (minKey == k.ts) { + init = true; + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return terrno; + } + + int32_t code = tsdbRowMergerAdd(pMerger, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS || pMerger->pTSchema == NULL) { + return code; + } + } + + if (minKey == tsLast) { + TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + if (init) { + tsdbRowMergerAdd(pMerger, fRow1, NULL); + } else { + init = true; + int32_t code = tsdbRowMergerAdd(pMerger, fRow1, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, + pReader->idStr); + } + + if (minKey == key) { + if (init) { + tsdbRowMergerAdd(pMerger, &fRow, NULL); + } else { + init = true; + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + } + } + + int32_t code = tsdbRowMergerGetRow(pMerger, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFree(pTSRow); + tsdbRowMergerClear(pMerger); + + return code; +} + +static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader, + STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, + bool mergeBlockData) { + SRowMerger* pMerger = &pReader->status.merger; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + bool copied = false; + int32_t code = TSDB_CODE_SUCCESS; + SRow* pTSRow = NULL; + TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + + // create local variable to hold the row value + TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; + + tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", pRow->pBlockData, pRow->iRow, pLastBlockReader->uid, + pReader->idStr); + + // only last block exists + if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { + code = tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); + if (code) { + return code; + } + + if (copied) { + pBlockScanInfo->lastKey = tsLastBlock; + return TSDB_CODE_SUCCESS; + } else { + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tsdbRowMergerAdd(pMerger, pRow1, NULL); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, + pReader->idStr); + + code = tsdbRowMergerGetRow(pMerger, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFree(pTSRow); + tsdbRowMergerClear(pMerger); + + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } else { // not merge block data + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, + pReader->idStr); + + // merge with block data if ts == key + if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + } + + code = tsdbRowMergerGetRow(pMerger, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFree(pTSRow); + tsdbRowMergerClear(pMerger); + + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader* pLastBlockReader, int64_t key, + STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + SRowMerger* pMerger = &pReader->status.merger; + + // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized + if (pMerger->pArray == NULL) { + ASSERT(pReader->info.pSchema == NULL); + STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); + if (ps == NULL) { + return terrno; + } + } + + if (hasDataInFileBlock(pBlockData, pDumpInfo)) { + // no last block available, only data block exists + if (!hasDataInLastBlock(pLastBlockReader)) { + return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); + } + + // row in last file block + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + if (ASCENDING_TRAVERSE(pReader->info.order)) { + if (key < tsLast) { + return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); + } else if (key > tsLast) { + return doMergeFileBlockAndLastBlock(pLastBlockReader, pReader, pBlockScanInfo, NULL, false); + } + } else { + if (key > tsLast) { + return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); + } else if (key < tsLast) { + return doMergeFileBlockAndLastBlock(pLastBlockReader, pReader, pBlockScanInfo, NULL, false); + } + } + // the following for key == tsLast + SRow* pTSRow = NULL; + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tsdbRowMergerAdd(pMerger, pRow1, NULL); + + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); + + code = tsdbRowMergerGetRow(pMerger, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFree(pTSRow); + tsdbRowMergerClear(pMerger); + return code; + + } else { // only last block exists + return doMergeFileBlockAndLastBlock(pLastBlockReader, pReader, pBlockScanInfo, NULL, false); + } +} + +static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, + SLastBlockReader* pLastBlockReader) { + SRowMerger* pMerger = &pReader->status.merger; + SRow* pTSRow = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + SArray* pDelList = pBlockScanInfo->delSkyline; + + TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader); + TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); + + int64_t tsLast = INT64_MIN; + if (hasDataInLastBlock(pLastBlockReader)) { + tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + } + + int64_t key = hasDataInFileBlock(pBlockData, pDumpInfo) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; + + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return code; + } + + STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + if (piSchema == NULL) { + return code; + } + + // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized + if (pMerger->pArray == NULL) { + ASSERT(pReader->info.pSchema == NULL); + STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); + if (ps == NULL) { + return terrno; + } + } + + int64_t minKey = 0; + if (ASCENDING_TRAVERSE(pReader->info.order)) { + minKey = INT64_MAX; // let's find the minimum + if (minKey > k.ts) { + minKey = k.ts; + } + + if (minKey > ik.ts) { + minKey = ik.ts; + } + + if (minKey > key && hasDataInFileBlock(pBlockData, pDumpInfo)) { + minKey = key; + } + + if (minKey > tsLast && hasDataInLastBlock(pLastBlockReader)) { + minKey = tsLast; + } + } else { + minKey = INT64_MIN; // let find the maximum ts value + if (minKey < k.ts) { + minKey = k.ts; + } + + if (minKey < ik.ts) { + minKey = ik.ts; + } + + if (minKey < key && hasDataInFileBlock(pBlockData, pDumpInfo)) { + minKey = key; + } + + if (minKey < tsLast && hasDataInLastBlock(pLastBlockReader)) { + minKey = tsLast; + } + } + + bool init = false; + + // ASC: file block -----> last block -----> imem -----> mem + // DESC: mem -----> imem -----> last block -----> file block + if (ASCENDING_TRAVERSE(pReader->info.order)) { + if (minKey == key) { + init = true; + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + } + + if (minKey == tsLast) { + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + if (init) { + tsdbRowMergerAdd(pMerger, pRow1, NULL); + } else { + init = true; + code = tsdbRowMergerAdd(pMerger, pRow1, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, + pReader->idStr); + } + + if (minKey == ik.ts) { + if (init) { + tsdbRowMergerAdd(pMerger, piRow, piSchema); + } else { + init = true; + code = tsdbRowMergerAdd(pMerger, piRow, piSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + if (minKey == k.ts) { + if (init) { + tsdbRowMergerAdd(pMerger, pRow, pSchema); + } else { + // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + code = tsdbRowMergerAdd(pMerger, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } else { + if (minKey == k.ts) { + init = true; + code = tsdbRowMergerAdd(pMerger, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + if (minKey == ik.ts) { + if (init) { + tsdbRowMergerAdd(pMerger, piRow, piSchema); + } else { + init = true; + code = tsdbRowMergerAdd(pMerger, piRow, piSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + if (minKey == tsLast) { + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + if (init) { + tsdbRowMergerAdd(pMerger, pRow1, NULL); + } else { + init = true; + code = tsdbRowMergerAdd(pMerger, pRow1, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, + pReader->idStr); + } + + if (minKey == key) { + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + if (!init) { + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + tsdbRowMergerAdd(pMerger, &fRow, NULL); + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + } + } + + code = tsdbRowMergerGetRow(pMerger, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFree(pTSRow); + tsdbRowMergerClear(pMerger); + return code; +} + +int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScanInfo* pBlockScanInfo, TSDBKEY* pKey, + SMemTable* pMem, SIterInfo* pIter, const char* type) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t backward = (!ASCENDING_TRAVERSE(pReader->info.order)); + pIter->hasVal = false; + + if (pMem != NULL) { + *pData = tsdbGetTbDataFromMemTable(pMem, pReader->info.suid, pBlockScanInfo->uid); + + if ((*pData) != NULL) { + code = tsdbTbDataIterCreate((*pData), pKey, backward, &pIter->iter); + if (code == TSDB_CODE_SUCCESS) { + pIter->hasVal = (tsdbTbDataIterGet(pIter->iter) != NULL); + + tsdbDebug("%p uid:%" PRIu64 ", check data in %s from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + "-%" PRId64 " %s", + pReader, pBlockScanInfo->uid, type, pKey->ts, pReader->info.order, (*pData)->minKey, (*pData)->maxKey, + pReader->idStr); + } else { + tsdbError("%p uid:%" PRIu64 ", failed to create iterator for %s, code:%s, %s", pReader, pBlockScanInfo->uid, + type, tstrerror(code), pReader->idStr); + return code; + } + } + } else { + tsdbDebug("%p uid:%" PRIu64 ", no data in %s, %s", pReader, pBlockScanInfo->uid, type, pReader->idStr); + } + + return code; +} + +static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { + if (pBlockScanInfo->iterInit) { + return TSDB_CODE_SUCCESS; + } + + STbData* d = NULL; + TSDBKEY startKey = {0}; + if (ASCENDING_TRAVERSE(pReader->info.order)) { + startKey = (TSDBKEY){.ts = pBlockScanInfo->lastKey + 1, .version = pReader->info.verRange.minVer}; + } else { + startKey = (TSDBKEY){.ts = pBlockScanInfo->lastKey - 1, .version = pReader->info.verRange.maxVer}; + } + + int32_t code = + doInitMemDataIter(pReader, &d, pBlockScanInfo, &startKey, pReader->pReadSnap->pMem, &pBlockScanInfo->iter, "mem"); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + STbData* di = NULL; + code = doInitMemDataIter(pReader, &di, pBlockScanInfo, &startKey, pReader->pReadSnap->pIMem, &pBlockScanInfo->iiter, + "imem"); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + loadMemTombData(&pBlockScanInfo->pMemDelData, d, di, pReader->info.verRange.maxVer); + + pBlockScanInfo->iterInit = true; + return TSDB_CODE_SUCCESS; +} + +static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, + STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { + // it is an multi-table data block + if (pBlockData->aUid != NULL) { + uint64_t uid = pBlockData->aUid[pDumpInfo->rowIndex]; + if (uid != pBlockScanInfo->uid) { // move to next row + return false; + } + } + + // check for version and time range + int64_t ver = pBlockData->aVersion[pDumpInfo->rowIndex]; + if (ver > pReader->info.verRange.maxVer || ver < pReader->info.verRange.minVer) { + return false; + } + + int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + if (ts > pReader->info.window.ekey || ts < pReader->info.window.skey) { + return false; + } + + if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, pReader->info.order, + &pReader->info.verRange)) { + return false; + } + + return true; +} + +static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { + // the last block reader has been initialized for this table. + if (pLBlockReader->uid == pScanInfo->uid) { + return hasDataInLastBlock(pLBlockReader); + } + + if (pLBlockReader->uid != 0) { + tMergeTreeClose(&pLBlockReader->mergeTree); + } + + pLBlockReader->uid = pScanInfo->uid; + + STimeWindow w = pLBlockReader->window; + if (ASCENDING_TRAVERSE(pLBlockReader->order)) { + w.skey = pScanInfo->lastKeyInStt; + } else { + w.ekey = pScanInfo->lastKeyInStt; + } + + int64_t st = taosGetTimestampUs(); + tsdbDebug("init last block reader, window:%" PRId64 "-%" PRId64 ", uid:%" PRIu64 ", %s", w.skey, w.ekey, + pScanInfo->uid, pReader->idStr); + + SMergeTreeConf conf = { + .uid = pScanInfo->uid, + .suid = pReader->info.suid, + .pTsdb = pReader->pTsdb, + .timewindow = w, + .verRange = pLBlockReader->verRange, + .strictTimeRange = false, + .pSchema = pReader->info.pSchema, + .pCurrentFileset = pReader->status.pCurrentFileset, + .backward = (pLBlockReader->order == TSDB_ORDER_DESC), + .pSttFileBlockIterArray = pReader->status.pLDataIterArray, + .pCols = pReader->suppInfo.colId, + .numOfCols = pReader->suppInfo.numOfCols, + .loadTombFn = loadSttTombDataForAll, + .pReader = pReader, + .idstr = pReader->idStr, + }; + + int32_t code = tMergeTreeOpen2(&pLBlockReader->mergeTree, &conf); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + initMemDataIterator(pScanInfo, pReader); + initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost); + + code = nextRowFromLastBlocks(pLBlockReader, pScanInfo, &pReader->info.verRange); + + int64_t el = taosGetTimestampUs() - st; + pReader->cost.initLastBlockReader += (el / 1000.0); + + tsdbDebug("init last block reader completed, elapsed time:%" PRId64 "us %s", el, pReader->idStr); + return code; +} + +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } + +bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) { + if ((pBlockData->nRow > 0) && (pBlockData->nRow != pDumpInfo->totalRows)) { + return false; // this is an invalid result. + } + return pBlockData->nRow > 0 && (!pDumpInfo->allDumped); +} + +int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, + STsdbReader* pReader) { + SRowMerger* pMerger = &pReader->status.merger; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + bool copied = false; + + int32_t code = tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo, &copied); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized + if (pMerger->pArray == NULL) { + ASSERT(pReader->info.pSchema == NULL); + STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); + if (ps == NULL) { + return terrno; + } + } + + if (copied) { + pBlockScanInfo->lastKey = key; + return TSDB_CODE_SUCCESS; + } else { + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + + SRow* pTSRow = NULL; + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + code = tsdbRowMergerGetRow(pMerger, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFree(pTSRow); + tsdbRowMergerClear(pMerger); + return code; + } +} + +static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, + SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + TSDBROW *pRow = NULL, *piRow = NULL; + int64_t key = (pBlockData->nRow > 0 && (!pDumpInfo->allDumped)) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : + (ASCENDING_TRAVERSE(pReader->info.order) ? INT64_MAX : INT64_MIN); + if (pBlockScanInfo->iter.hasVal) { + pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); + } + + if (pBlockScanInfo->iiter.hasVal) { + piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); + } + + // two levels of mem-table does contain the valid rows + if (pRow != NULL && piRow != NULL) { + return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); + } + + // imem + file + last block + if (pBlockScanInfo->iiter.hasVal) { + return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader); + } + + // mem + file + last block + if (pBlockScanInfo->iter.hasVal) { + return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pLastBlockReader); + } + + // files data blocks + last block + return mergeFileBlockAndLastBlock(pReader, pLastBlockReader, key, pBlockScanInfo, pBlockData); +} + +static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pBlockScanInfo, + STsdbReader* pReader, bool* loadNeighbor) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; + int32_t nextIndex = -1; + + *loadNeighbor = false; + + SBrinRecord rec = {0}; + bool hasNeighbor = getNeighborBlockOfSameTable(pBlockInfo, pBlockScanInfo, &nextIndex, pReader->info.order, &rec); + if (!hasNeighbor) { // do nothing + return code; + } + + if (overlapWithNeighborBlock2(pBlockInfo, &rec, pReader->info.order)) { // load next block + SReaderStatus* pStatus = &pReader->status; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + + // 1. find the next neighbor block in the scan block list + SFileDataBlockInfo fb = {.uid = pBlockInfo->uid, .tbBlockIdx = nextIndex}; + int32_t neighborIndex = findFileBlockInfoIndex(pBlockIter, &fb); + + // 2. remove it from the scan block list + setFileBlockActiveInBlockIter(pBlockIter, neighborIndex, step); + + // 3. load the neighbor block, and set it to be the currently accessed file data block + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pBlockInfo->uid); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // 4. check the data values + initBlockDumpInfo(pReader, pBlockIter); + *loadNeighbor = true; + } + + return code; +} + +static void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo) { + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + + pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; + pResBlock->info.dataLoad = 1; + blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); + + setComposedBlockFlag(pReader, true); + + pReader->cost.composedBlocks += 1; + pReader->cost.buildComposedBlockTime += el; +} + +static int32_t buildComposedDataBlock(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + int64_t st = taosGetTimestampUs(); + int32_t step = asc ? 1 : -1; + double el = 0; + SBrinRecord* pRecord = &pBlockInfo->record; + + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + STableBlockScanInfo* pBlockScanInfo = NULL; + if (pBlockInfo != NULL) { + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { + setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + return code; + } + + pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); + if (pBlockScanInfo == NULL) { + goto _end; + } + + pRecord = &pBlockInfo->record; + TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); + + // it is a clean block, load it directly + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf, pLastBlockReader) && + (pRecord->numRow <= pReader->resBlockInfo.capacity)) { + if (asc || (!hasDataInLastBlock(pLastBlockReader))) { + code = copyBlockDataToSDataBlock(pReader); + if (code) { + goto _end; + } + + // record the last key value + pBlockScanInfo->lastKey = asc ? pRecord->lastKey : pRecord->firstKey; + goto _end; + } + } + } else { // file blocks not exist + ASSERT(0); + pBlockScanInfo = *pReader->status.pTableIter; + if (pReader->pIgnoreTables && + taosHashGet(*pReader->pIgnoreTables, &pBlockScanInfo->uid, sizeof(pBlockScanInfo->uid))) { + // setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->info.order); + return code; + } + } + + SBlockData* pBlockData = &pReader->status.fileBlockData; + + while (1) { + bool hasBlockData = false; + { + while (pBlockData->nRow > 0 && pBlockData->uid == pBlockScanInfo->uid) { + // find the first qualified row in data block + if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { + hasBlockData = true; + break; + } + + pDumpInfo->rowIndex += step; + + if (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0) { + pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); // NOTE: get the new block info + + // continue check for the next file block if the last ts in the current block + // is overlapped with the next neighbor block + bool loadNeighbor = false; + code = loadNeighborIfOverlap(pBlockInfo, pBlockScanInfo, pReader, &loadNeighbor); + if ((!loadNeighbor) || (code != 0)) { + setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + break; + } + } + } + } + + // no data in last block and block, no need to proceed. + if (hasBlockData == false) { + break; + } + + code = buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); + if (code) { + goto _end; + } + + // currently loaded file data block is consumed + if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { + setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + break; + } + + if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) { + break; + } + } + +_end: + el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pBlockScanInfo); + + if (pResBlock->info.rows > 0) { + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", + pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, + pResBlock->info.rows, el, pReader->idStr); + } + + return code; +} + +void setComposedBlockFlag(STsdbReader* pReader, bool composed) { pReader->status.composedDataBlock = composed; } + +int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order) { + if (pDelSkyline == NULL) { + return 0; + } + + return ASCENDING_TRAVERSE(order) ? 0 : taosArrayGetSize(pDelSkyline) - 1; +} + +int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SCostSummary* pCost) { + int32_t code = 0; + int32_t newDelDataInFile = taosArrayGetSize(pBlockScanInfo->pfileDelData); + if (newDelDataInFile == 0 && + ((pBlockScanInfo->delSkyline != NULL) || (TARRAY_SIZE(pBlockScanInfo->pMemDelData) == 0))) { + return code; + } + + int64_t st = taosGetTimestampUs(); + + if (pBlockScanInfo->delSkyline != NULL) { + taosArrayClear(pBlockScanInfo->delSkyline); + } else { + pBlockScanInfo->delSkyline = taosArrayInit(4, sizeof(TSDBKEY)); + } + + SArray* pSource = pBlockScanInfo->pfileDelData; + if (pSource == NULL) { + pSource = pBlockScanInfo->pMemDelData; + } else { + taosArrayAddAll(pSource, pBlockScanInfo->pMemDelData); + } + + code = tsdbBuildDeleteSkyline(pSource, 0, taosArrayGetSize(pSource) - 1, pBlockScanInfo->delSkyline); + + taosArrayClear(pBlockScanInfo->pfileDelData); + int32_t index = getInitialDelIndex(pBlockScanInfo->delSkyline, order); + + pBlockScanInfo->iter.index = index; + pBlockScanInfo->iiter.index = index; + pBlockScanInfo->fileDelIndex = index; + pBlockScanInfo->lastBlockDelIndex = index; + + double el = taosGetTimestampUs() - st; + pCost->createSkylineIterTime = el / 1000.0; + + return code; +} + +TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}, ikey = {.ts = TSKEY_INITIAL_VAL}; + + bool hasKey = false, hasIKey = false; + TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); + if (pRow != NULL) { + hasKey = true; + key = TSDBROW_KEY(pRow); + } + + TSDBROW* pIRow = getValidMemRow(&pScanInfo->iiter, pScanInfo->delSkyline, pReader); + if (pIRow != NULL) { + hasIKey = true; + ikey = TSDBROW_KEY(pIRow); + } + + if (hasKey) { + if (hasIKey) { // has data in mem & imem + if (asc) { + return key.ts <= ikey.ts ? key : ikey; + } else { + return key.ts <= ikey.ts ? ikey : key; + } + } else { // no data in imem + return key; + } + } else { + // no data in mem & imem, return the initial value + // only imem has data, return ikey + return ikey; + } +} + +static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pTableList) { + SReaderStatus* pStatus = &pReader->status; + pBlockNum->numOfBlocks = 0; + pBlockNum->numOfLastFiles = 0; + + size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBrinBlk)); + + while (1) { + // only check here, since the iterate data in memory is very fast. + if (pReader->code != TSDB_CODE_SUCCESS) { + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + return pReader->code; + } + + bool hasNext = false; + int32_t code = filesetIteratorNext(&pStatus->fileIter, pReader, &hasNext); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pIndexList); + return code; + } + + if (!hasNext) { // no data files on disk + break; + } + + taosArrayClear(pIndexList); + code = doLoadBlockIndex(pReader, pReader->pFileReader, pIndexList); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pIndexList); + return code; + } + + if (taosArrayGetSize(pIndexList) > 0 || pReader->status.pCurrentFileset->lvlArr->size > 0) { + code = doLoadFileBlock(pReader, pIndexList, pBlockNum, pTableList); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pIndexList); + return code; + } + + if (pBlockNum->numOfBlocks + pBlockNum->numOfLastFiles > 0) { + break; + } + } + + // no blocks in current file, try next files + } + + taosArrayDestroy(pIndexList); + return loadDataFileTombDataForAll(pReader); +} + +static void resetTableListIndex(SReaderStatus* pStatus) { + STableUidList* pList = &pStatus->uidList; + + pList->currentIndex = 0; + uint64_t uid = pList->tableUidList[0]; + pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid)); +} + +static bool moveToNextTable(STableUidList* pOrderedCheckInfo, SReaderStatus* pStatus) { + pOrderedCheckInfo->currentIndex += 1; + if (pOrderedCheckInfo->currentIndex >= tSimpleHashGetSize(pStatus->pTableMap)) { + pStatus->pTableIter = NULL; + return false; + } + + uint64_t uid = pOrderedCheckInfo->tableUidList[pOrderedCheckInfo->currentIndex]; + pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid)); + return (pStatus->pTableIter != NULL); +} + +static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader; + STableUidList* pUidList = &pStatus->uidList; + int32_t code = TSDB_CODE_SUCCESS; + + if (tSimpleHashGetSize(pStatus->pTableMap) == 0) { + return TSDB_CODE_SUCCESS; + } + + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + + while (1) { + if (pReader->code != TSDB_CODE_SUCCESS) { + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + return pReader->code; + } + + // load the last data block of current table + STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; + if (pScanInfo == NULL) { + tsdbError("table Iter is null, invalid pScanInfo, try next table %s", pReader->idStr); + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + + continue; + } + + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pScanInfo->uid, sizeof(pScanInfo->uid))) { + // reset the index in last block when handing a new file + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + + continue; + } + + bool hasDataInLastFile = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + if (!hasDataInLastFile) { + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + + continue; + } + + int64_t st = taosGetTimestampUs(); + while (1) { + bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); + + // no data in last block and block, no need to proceed. + if (hasBlockLData == false) { + break; + } + + code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + if (code) { + return code; + } + + if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) { + break; + } + } + + double el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pScanInfo); + + if (pResBlock->info.rows > 0) { + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", + pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, + pResBlock->info.rows, el, pReader->idStr); + return TSDB_CODE_SUCCESS; + } + + // current table is exhausted, let's try next table + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + } +} + +static int32_t doBuildDataBlock(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + + SReaderStatus* pStatus = &pReader->status; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + STableBlockScanInfo* pScanInfo = NULL; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order); + return code; + } + + if (pReader->code != TSDB_CODE_SUCCESS) { + return pReader->code; + } + + pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); + if (pScanInfo == NULL) { + return terrno; + } + + initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); + + if (fileBlockShouldLoad(pReader, pBlockInfo, pScanInfo, keyInBuf, pLastBlockReader)) { + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // build composed data block + code = buildComposedDataBlock(pReader); + } else if (bufferDataInFileBlockGap(pReader->info.order, keyInBuf, pBlockInfo)) { + // data in memory that are earlier than current file block + // rows in buffer should be less than the file block in asc, greater than file block in desc + int64_t endKey = + (ASCENDING_TRAVERSE(pReader->info.order)) ? pBlockInfo->record.firstKey : pBlockInfo->record.lastKey; + code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); + } else { + bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader); + int64_t tsLast = bHasDataInLastBlock ? getCurrentKeyInLastBlock(pLastBlockReader) : INT64_MIN; + if (!bHasDataInLastBlock || ((ASCENDING_TRAVERSE(pReader->info.order) && pBlockInfo->record.lastKey < tsLast) || + (!ASCENDING_TRAVERSE(pReader->info.order) && pBlockInfo->record.firstKey > tsLast))) { + // whole block is required, return it directly + SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info; + pInfo->rows = pBlockInfo->record.numRow; + pInfo->id.uid = pScanInfo->uid; + pInfo->dataLoad = 0; + pInfo->window = (STimeWindow){.skey = pBlockInfo->record.firstKey, .ekey = pBlockInfo->record.lastKey}; + setComposedBlockFlag(pReader, false); + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order); + + // update the last key for the corresponding table + pScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? pInfo->window.ekey : pInfo->window.skey; + tsdbDebug("%p uid:%" PRIu64 + " clean file block retrieved from file, global index:%d, " + "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s", + pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->record.numRow, + pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr); + } else { + SBlockData* pBData = &pReader->status.fileBlockData; + tBlockDataReset(pBData); + + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + tsdbDebug("load data in last block firstly %s", pReader->idStr); + + int64_t st = taosGetTimestampUs(); + + while (1) { + bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); + + // no data in last block and block, no need to proceed. + if (hasBlockLData == false) { + break; + } + + code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + if (code) { + return code; + } + + if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) { + break; + } + } + + double el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pScanInfo); + + if (pResBlock->info.rows > 0) { + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", + pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, + pResBlock->info.rows, el, pReader->idStr); + } + } + + } + + return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code; +} + +static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { + int64_t st = taosGetTimestampUs(); + LRUHandle* handle = NULL; + int32_t code = tsdbCacheGetBlockIdx(pFileReader->pTsdb->biCache, pFileReader, &handle); + if (code != TSDB_CODE_SUCCESS || handle == NULL) { + goto _end; + } + + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + + SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle); + size_t num = taosArrayGetSize(aBlockIdx); + if (num == 0) { + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); + return TSDB_CODE_SUCCESS; + } + + SBlockIdx* pBlockIdx = NULL; + for (int32_t i = 0; i < num; ++i) { + pBlockIdx = (SBlockIdx*)taosArrayGet(aBlockIdx, i); + if (pBlockIdx->suid != pReader->info.suid) { + continue; + } + + STableBlockScanInfo** p = tSimpleHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(pBlockIdx->uid)); + if (p == NULL) { + continue; + } + + STableBlockScanInfo* pScanInfo = *p; + SDataBlk block = {0}; + // for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { + // tGetDataBlk(pScanInfo->mapData.pData + pScanInfo->mapData.aOffset[j], &block); + // pReader->rowsNum += block.nRow; + // } + } + +_end: + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); + return code; +} + +static int32_t doSumSttBlockRows(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + SSttBlockLoadInfo* pBlockLoadInfo = NULL; +#if 0 + for (int32_t i = 0; i < pReader->pFileReader->pSet->nSttF; ++i) { // open all last file + pBlockLoadInfo = &pLastBlockReader->pInfo[i]; + + code = tsdbReadSttBlk(pReader->pFileReader, i, pBlockLoadInfo->aSttBlk); + if (code) { + return code; + } + + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + if (size >= 1) { + SSttBlk* pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); + SSttBlk* pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1); + + // all identical + if (pStart->suid == pEnd->suid) { + if (pStart->suid != pReader->info.suid) { + // no qualified stt block existed + taosArrayClear(pBlockLoadInfo->aSttBlk); + continue; + } + for (int32_t j = 0; j < size; ++j) { + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, j); + pReader->rowsNum += p->nRow; + } + } else { + for (int32_t j = 0; j < size; ++j) { + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, j); + uint64_t s = p->suid; + if (s < pReader->info.suid) { + continue; + } + + if (s == pReader->info.suid) { + pReader->rowsNum += p->nRow; + } else if (s > pReader->info.suid) { + break; + } + } + } + } + } +#endif + + return code; +} + +static int32_t readRowsCountFromFiles(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + + while (1) { + bool hasNext = false; + code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext); + if (code) { + return code; + } + + if (!hasNext) { // no data files on disk + break; + } + + // code = doSumFileBlockRows(pReader, pReader->pFileReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doSumSttBlockRows(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + pReader->status.loadFromFile = false; + + return code; +} + +static int32_t readRowsCountFromMem(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t memNum = 0, imemNum = 0; + if (pReader->pReadSnap->pMem != NULL) { + tsdbMemTableCountRows(pReader->pReadSnap->pMem, pReader->status.pTableMap, &memNum); + } + + if (pReader->pReadSnap->pIMem != NULL) { + tsdbMemTableCountRows(pReader->pReadSnap->pIMem, pReader->status.pTableMap, &imemNum); + } + + pReader->rowsNum += memNum + imemNum; + + return code; +} + +static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + STableUidList* pUidList = &pStatus->uidList; + + while (1) { + if (pReader->code != TSDB_CODE_SUCCESS) { + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + return pReader->code; + } + + STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter; + if (pReader->pIgnoreTables && + taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) { + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + pBlockScanInfo = pStatus->pTableIter; + } + + initMemDataIterator(*pBlockScanInfo, pReader); + initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); + + int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; + int32_t code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (pReader->resBlockInfo.pResBlock->info.rows > 0) { + return TSDB_CODE_SUCCESS; + } + + // current table is exhausted, let's try next table + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + } +} + +// set the correct start position in case of the first/last file block, according to the query time window +static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { + int64_t lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? INT64_MIN : INT64_MAX; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SReaderStatus* pStatus = &pReader->status; + SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo; + + if (pBlockInfo) { + STableBlockScanInfo* pScanInfo = tSimpleHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + if (pScanInfo) { + lastKey = pScanInfo->lastKey; + } + + pDumpInfo->totalRows = pBlockInfo->record.numRow; + pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->record.numRow - 1; + } else { + pDumpInfo->totalRows = 0; + pDumpInfo->rowIndex = 0; + } + + pDumpInfo->allDumped = false; + pDumpInfo->lastKey = lastKey; +} + +static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { + SBlockNumber num = {0}; + SArray* pTableList = taosArrayInit(40, POINTER_BYTES); + + int32_t code = moveToNextFile(pReader, &num, pTableList); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pTableList); + return code; + } + + // all data files are consumed, try data in buffer + if (num.numOfBlocks + num.numOfLastFiles == 0) { + pReader->status.loadFromFile = false; + taosArrayDestroy(pTableList); + return code; + } + + // initialize the block iterator for a new fileset + if (num.numOfBlocks > 0) { + code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks, pTableList); + } else { // no block data, only last block exists + tBlockDataReset(&pReader->status.fileBlockData); + resetDataBlockIterator(pBlockIter, pReader->info.order); + resetTableListIndex(&pReader->status); + } + + // set the correct start position according to the query time window + initBlockDumpInfo(pReader, pBlockIter); + taosArrayDestroy(pTableList); + return code; +} + +static bool fileBlockPartiallyRead(SFileBlockDumpInfo* pDumpInfo, bool asc) { + return (!pDumpInfo->allDumped) && + ((pDumpInfo->rowIndex > 0 && asc) || (pDumpInfo->rowIndex < (pDumpInfo->totalRows - 1) && (!asc))); +} + +typedef enum { + TSDB_READ_RETURN = 0x1, + TSDB_READ_CONTINUE = 0x2, +} ERetrieveType; + +static ERetrieveType doReadDataFromLastFiles(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + SDataBlockIter* pBlockIter = &pReader->status.blockIter; + + while (1) { + terrno = 0; + + code = doLoadLastBlockSequentially(pReader); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return TSDB_READ_RETURN; + } + + if (pResBlock->info.rows > 0) { + return TSDB_READ_RETURN; + } + + // all data blocks are checked in this last block file, now let's try the next file + ASSERT(pReader->status.pTableIter == NULL); + code = initForFirstBlockInFile(pReader, pBlockIter); + + // error happens or all the data files are completely checked + if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { + terrno = code; + return TSDB_READ_RETURN; + } + + if (pBlockIter->numOfBlocks > 0) { // there are data blocks existed. + return TSDB_READ_CONTINUE; + } else { // all blocks in data file are checked, let's check the data in last files + resetTableListIndex(&pReader->status); + } + } +} + +static int32_t buildBlockFromFiles(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + + SDataBlockIter* pBlockIter = &pReader->status.blockIter; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + + if (pBlockIter->numOfBlocks == 0) { + // let's try to extract data from stt files. + ERetrieveType type = doReadDataFromLastFiles(pReader); + if (type == TSDB_READ_RETURN) { + return terrno; + } + + code = doBuildDataBlock(pReader); + if (code != TSDB_CODE_SUCCESS || pResBlock->info.rows > 0) { + return code; + } + } + + while (1) { + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + if (fileBlockPartiallyRead(pDumpInfo, asc)) { // file data block is partially loaded + code = buildComposedDataBlock(pReader); + } else { + // current block are exhausted, try the next file block + if (pDumpInfo->allDumped) { + // try next data block in current file + bool hasNext = blockIteratorNext(&pReader->status.blockIter, pReader->idStr); + if (hasNext) { // check for the next block in the block accessed order list + initBlockDumpInfo(pReader, pBlockIter); + } else { + // all data blocks in files are checked, let's check the data in last files. + // data blocks in current file are exhausted, let's try the next file now + SBlockData* pBlockData = &pReader->status.fileBlockData; + if (pBlockData->uid != 0) { + tBlockDataClear(pBlockData); + } + + tBlockDataReset(pBlockData); + resetDataBlockIterator(pBlockIter, pReader->info.order); + resetTableListIndex(&pReader->status); + + ERetrieveType type = doReadDataFromLastFiles(pReader); + if (type == TSDB_READ_RETURN) { + return terrno; + } + } + } + + code = doBuildDataBlock(pReader); + } + + if (code != TSDB_CODE_SUCCESS || pResBlock->info.rows > 0) { + return code; + } + } +} + +static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idStr, + int8_t* pLevel) { + if (VND_IS_RSMA(pVnode)) { + int8_t level = 0; + int8_t precision = pVnode->config.tsdbCfg.precision; + int64_t now = taosGetTimestamp(precision); + int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1L + : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L + : 1000000L); + + for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) { + SRetention* pRetention = retentions + level; + if (pRetention->keep <= 0) { + if (level > 0) { + --level; + } + break; + } + if ((now - pRetention->keep) <= (winSKey + offset)) { + break; + } + ++level; + } + + const char* str = (idStr != NULL) ? idStr : ""; + + if (level == TSDB_RETENTION_L0) { + *pLevel = TSDB_RETENTION_L0; + tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L0, str); + return VND_RSMA0(pVnode); + } else if (level == TSDB_RETENTION_L1) { + *pLevel = TSDB_RETENTION_L1; + tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L1, str); + return VND_RSMA1(pVnode); + } else { + *pLevel = TSDB_RETENTION_L2; + tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L2, str); + return VND_RSMA2(pVnode); + } + } + + return VND_TSDB(pVnode); +} + +SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) { + int64_t startVer = (pCond->startVersion == -1) ? 0 : pCond->startVersion; + + int64_t endVer = 0; + if (pCond->endVersion == -1) { + // user not specified end version, set current maximum version of vnode as the endVersion + endVer = pVnode->state.applied; + } else { + endVer = (pCond->endVersion > pVnode->state.applied) ? pVnode->state.applied : pCond->endVersion; + } + + return (SVersionRange){.minVer = startVer, .maxVer = endVer}; +} + +bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, + SVersionRange* pVerRange) { + if (pDelList == NULL || (taosArrayGetSize(pDelList) == 0)) { + return false; + } + + size_t num = taosArrayGetSize(pDelList); + bool asc = ASCENDING_TRAVERSE(order); + int32_t step = asc ? 1 : -1; + + if (asc) { + if (*index >= num - 1) { + TSDBKEY* last = taosArrayGetLast(pDelList); + ASSERT(key >= last->ts); + + if (key > last->ts) { + return false; + } else if (key == last->ts) { + TSDBKEY* prev = taosArrayGet(pDelList, num - 2); + return (prev->version >= ver && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer); + } + } else { + TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); + TSDBKEY* pNext = taosArrayGet(pDelList, (*index) + 1); + + if (key < pCurrent->ts) { + return false; + } + + if (pCurrent->ts <= key && pNext->ts >= key && pCurrent->version >= ver && + pVerRange->maxVer >= pCurrent->version) { + return true; + } + + while (pNext->ts <= key && (*index) < num - 1) { + (*index) += 1; + + if ((*index) < num - 1) { + pCurrent = taosArrayGet(pDelList, *index); + pNext = taosArrayGet(pDelList, (*index) + 1); + + // it is not a consecutive deletion range, ignore it + if (pCurrent->version == 0 && pNext->version > 0) { + continue; + } + + if (pCurrent->ts <= key && pNext->ts >= key && pCurrent->version >= ver && + pVerRange->maxVer >= pCurrent->version) { + return true; + } + } + } + + return false; + } + } else { + if (*index <= 0) { + TSDBKEY* pFirst = taosArrayGet(pDelList, 0); + + if (key < pFirst->ts) { + return false; + } else if (key == pFirst->ts) { + return pFirst->version >= ver; + } else { + ASSERT(0); + } + } else { + TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); + TSDBKEY* pPrev = taosArrayGet(pDelList, (*index) - 1); + + if (key > pCurrent->ts) { + return false; + } + + if (pPrev->ts <= key && pCurrent->ts >= key && pPrev->version >= ver) { + return true; + } + + while (pPrev->ts >= key && (*index) > 1) { + (*index) += step; + + if ((*index) >= 1) { + pCurrent = taosArrayGet(pDelList, *index); + pPrev = taosArrayGet(pDelList, (*index) - 1); + + // it is not a consecutive deletion range, ignore it + if (pCurrent->version > 0 && pPrev->version == 0) { + continue; + } + + if (pPrev->ts <= key && pCurrent->ts >= key && pPrev->version >= ver) { + return true; + } + } + } + + return false; + } + } + + return false; +} + +TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader) { + if (!pIter->hasVal) { + return NULL; + } + + TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter); + TSDBKEY key = TSDBROW_KEY(pRow); + + if (outOfTimeWindow(key.ts, &pReader->info.window)) { + pIter->hasVal = false; + return NULL; + } + + // it is a valid data version + if ((key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) && + (!hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, pReader->info.order, &pReader->info.verRange))) { + return pRow; + } + + while (1) { + pIter->hasVal = tsdbTbDataIterNext(pIter->iter); + if (!pIter->hasVal) { + return NULL; + } + + pRow = tsdbTbDataIterGet(pIter->iter); + + key = TSDBROW_KEY(pRow); + if (outOfTimeWindow(key.ts, &pReader->info.window)) { + pIter->hasVal = false; + return NULL; + } + + if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer && + (!hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, pReader->info.order, &pReader->info.verRange))) { + return pRow; + } + } +} + +int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, STsdbReader* pReader) { + SRowMerger* pMerger = &pReader->status.merger; + + while (1) { + pIter->hasVal = tsdbTbDataIterNext(pIter->iter); + if (!pIter->hasVal) { + break; + } + + // data exists but not valid + TSDBROW* pRow = getValidMemRow(pIter, pDelList, pReader); + if (pRow == NULL) { + break; + } + + // ts is not identical, quit + TSDBKEY k = TSDBROW_KEY(pRow); + if (k.ts != ts) { + break; + } + + if (pRow->type == TSDBROW_ROW_FMT) { + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } + + tsdbRowMergerAdd(pMerger, pRow, pTSchema); + } else { // column format + tsdbRowMergerAdd(pMerger, pRow, NULL); + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowIndex, int64_t key, SRowMerger* pMerger, + SVersionRange* pVerRange, int32_t step) { + while (rowIndex < pBlockData->nRow && rowIndex >= 0 && pBlockData->aTSKEY[rowIndex] == key) { + if (pBlockData->aVersion[rowIndex] > pVerRange->maxVer || pBlockData->aVersion[rowIndex] < pVerRange->minVer) { + rowIndex += step; + continue; + } + + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); + tsdbRowMergerAdd(pMerger, &fRow, NULL); + rowIndex += step; + } + + return rowIndex; +} + +typedef enum { + CHECK_FILEBLOCK_CONT = 0x1, + CHECK_FILEBLOCK_QUIT = 0x2, +} CHECK_FILEBLOCK_STATE; + +static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, + SFileDataBlockInfo* pFBlock, SRowMerger* pMerger, int64_t key, + CHECK_FILEBLOCK_STATE* state) { + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + SBlockData* pBlockData = &pReader->status.fileBlockData; + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + + *state = CHECK_FILEBLOCK_QUIT; + int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; + + bool loadNeighbor = true; + int32_t code = loadNeighborIfOverlap(pFBlock, pScanInfo, pReader, &loadNeighbor); + + if (loadNeighbor && (code == TSDB_CODE_SUCCESS)) { + pDumpInfo->rowIndex = + doMergeRowsInFileBlockImpl(pBlockData, pDumpInfo->rowIndex, key, pMerger, &pReader->info.verRange, step); + if ((pDumpInfo->rowIndex >= pDumpInfo->totalRows && asc) || (pDumpInfo->rowIndex < 0 && !asc)) { + *state = CHECK_FILEBLOCK_CONT; + } + } + + return code; +} + +int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + SRowMerger* pMerger = &pReader->status.merger; + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + int32_t step = asc ? 1 : -1; + + pDumpInfo->rowIndex += step; + if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) { + pDumpInfo->rowIndex = + doMergeRowsInFileBlockImpl(pBlockData, pDumpInfo->rowIndex, key, pMerger, &pReader->info.verRange, step); + } + + // all rows are consumed, let's try next file block + if ((pDumpInfo->rowIndex >= pBlockData->nRow && asc) || (pDumpInfo->rowIndex < 0 && !asc)) { + while (1) { + CHECK_FILEBLOCK_STATE st; + + SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + if (pFileBlockInfo == NULL) { + st = CHECK_FILEBLOCK_QUIT; + break; + } + + checkForNeighborFileBlock(pReader, pScanInfo, pFileBlockInfo, pMerger, key, &st); + if (st == CHECK_FILEBLOCK_QUIT) { + break; + } + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, + SRowMerger* pMerger, SVersionRange* pVerRange, const char* idStr) { + while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo, pVerRange)) { + int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); + if (next1 == ts) { + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tsdbRowMergerAdd(pMerger, pRow1, NULL); + } else { + tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, + pScanInfo->lastBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt, + idStr); + break; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, TSDBROW* pResRow, + STsdbReader* pReader, bool* freeTSRow) { + TSDBROW* pNextRow = NULL; + TSDBROW current = *pRow; + + { // if the timestamp of the next valid row has a different ts, return current row directly + pIter->hasVal = tsdbTbDataIterNext(pIter->iter); + + if (!pIter->hasVal) { + *pResRow = *pRow; + *freeTSRow = false; + return TSDB_CODE_SUCCESS; + } else { // has next point in mem/imem + pNextRow = getValidMemRow(pIter, pDelList, pReader); + if (pNextRow == NULL) { + *pResRow = current; + *freeTSRow = false; + return TSDB_CODE_SUCCESS; + } + + if (TSDBROW_TS(¤t) != TSDBROW_TS(pNextRow)) { + *pResRow = current; + *freeTSRow = false; + return TSDB_CODE_SUCCESS; + } + } + } + + terrno = 0; + int32_t code = 0; + + // start to merge duplicated rows + if (current.type == TSDBROW_ROW_FMT) { + // get the correct schema for data in memory + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } + + code = tsdbRowMergerAdd(&pReader->status.merger, ¤t, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); + if (pTSchema1 == NULL) { + return terrno; + } + + tsdbRowMergerAdd(&pReader->status.merger, pNextRow, pTSchema1); + } else { // let's merge rows in file block + code = tsdbRowMergerAdd(&pReader->status.merger, ¤t, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + tsdbRowMergerAdd(&pReader->status.merger, pNextRow, NULL); + } + + code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = tsdbRowMergerGetRow(&pReader->status.merger, &pResRow->pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + pResRow->type = TSDBROW_ROW_FMT; + tsdbRowMergerClear(&pReader->status.merger); + *freeTSRow = true; + + return TSDB_CODE_SUCCESS; +} + +int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, + SRow** pTSRow) { + SRowMerger* pMerger = &pReader->status.merger; + + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return terrno; + } + + STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + if (piSchema == NULL) { + return terrno; + } + + if (ASCENDING_TRAVERSE(pReader->info.order)) { // ascending order imem --> mem + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + } else { + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS || pMerger->pTSchema == NULL) { + return code; + } + + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + int32_t code = tsdbRowMergerGetRow(pMerger, pTSRow); + tsdbRowMergerClear(pMerger); + return code; +} + +static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow, + int64_t endKey, bool* freeTSRow) { + TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); + TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); + SArray* pDelList = pBlockScanInfo->delSkyline; + uint64_t uid = pBlockScanInfo->uid; + + // todo refactor + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + if (pBlockScanInfo->iter.hasVal) { + TSDBKEY k = TSDBROW_KEY(pRow); + if ((k.ts >= endKey && asc) || (k.ts <= endKey && !asc)) { + pRow = NULL; + } + } + + if (pBlockScanInfo->iiter.hasVal) { + TSDBKEY k = TSDBROW_KEY(piRow); + if ((k.ts >= endKey && asc) || (k.ts <= endKey && !asc)) { + piRow = NULL; + } + } + + if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal && pRow != NULL && piRow != NULL) { + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); + + int32_t code = TSDB_CODE_SUCCESS; + if (ik.ts != k.ts) { + if (((ik.ts < k.ts) && asc) || ((ik.ts > k.ts) && (!asc))) { // ik.ts < k.ts + code = doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pResRow, pReader, freeTSRow); + } else if (((k.ts < ik.ts) && asc) || ((k.ts > ik.ts) && (!asc))) { + code = doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, pResRow, pReader, freeTSRow); + } + } else { // ik.ts == k.ts + *freeTSRow = true; + pResRow->type = TSDBROW_ROW_FMT; + code = doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pResRow->pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + return code; + } + + if (pBlockScanInfo->iter.hasVal && pRow != NULL) { + return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pResRow, pReader, + freeTSRow); + } + + if (pBlockScanInfo->iiter.hasVal && piRow != NULL) { + return doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pResRow, pReader, freeTSRow); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) { + int32_t outputRowIndex = pBlock->info.rows; + int64_t uid = pScanInfo->uid; + int32_t code = TSDB_CODE_SUCCESS; + + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); + if (pSchema == NULL) { + return terrno; + } + + SColVal colVal = {0}; + int32_t i = 0, j = 0; + + if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts; + i += 1; + } + + while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) { + col_id_t colId = pSupInfo->colId[i]; + + if (colId == pSchema->columns[j].colId) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + + tRowGet(pTSRow, pSchema, j, &colVal); + code = doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); + if (code) { + return code; + } + i += 1; + j += 1; + } else if (colId < pSchema->columns[j].colId) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + + colDataSetNULL(pColInfoData, outputRowIndex); + i += 1; + } else if (colId > pSchema->columns[j].colId) { + j += 1; + } + } + + // set null value since current column does not exist in the "pSchema" + while (i < pSupInfo->numOfCols) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + colDataSetNULL(pColInfoData, outputRowIndex); + i += 1; + } + + pBlock->info.dataLoad = 1; + pBlock->info.rows += 1; + pScanInfo->lastKey = pTSRow->ts; + return TSDB_CODE_SUCCESS; +} + +int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, + int32_t rowIndex) { + int32_t i = 0, j = 0; + int32_t outputRowIndex = pResBlock->info.rows; + int32_t code = TSDB_CODE_SUCCESS; + + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + ((int64_t*)pReader->status.pPrimaryTsCol->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; + i += 1; + + SColVal cv = {0}; + int32_t numOfInputCols = pBlockData->nColData; + int32_t numOfOutputCols = pSupInfo->numOfCols; + + while (i < numOfOutputCols && j < numOfInputCols) { + SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); + if (pData->cid < pSupInfo->colId[i]) { + j += 1; + continue; + } + + SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pData->cid == pSupInfo->colId[i]) { + tColDataGetValue(pData, rowIndex, &cv); + code = doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); + if (code) { + return code; + } + j += 1; + } else if (pData->cid > pCol->info.colId) { + // the specified column does not exist in file block, fill with null data + colDataSetNULL(pCol, outputRowIndex); + } + + i += 1; + } + + while (i < numOfOutputCols) { + SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + colDataSetNULL(pCol, outputRowIndex); + i += 1; + } + + pResBlock->info.dataLoad = 1; + pResBlock->info.rows += 1; + return TSDB_CODE_SUCCESS; +} + +int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, + STsdbReader* pReader) { + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + int32_t code = TSDB_CODE_SUCCESS; + + do { + // SRow* pTSRow = NULL; + TSDBROW row = {.type = -1}; + bool freeTSRow = false; + tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); + if (row.type == -1) { + break; + } + + if (row.type == TSDBROW_ROW_FMT) { + code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo); + + if (freeTSRow) { + taosMemoryFree(row.pTSRow); + } + + if (code) { + return code; + } + } else { + code = doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow); + if (code) { + break; + } + } + + // no data in buffer, return immediately + if (!(pBlockScanInfo->iter.hasVal || pBlockScanInfo->iiter.hasVal)) { + break; + } + + if (pBlock->info.rows >= capacity) { + break; + } + } while (1); + + return code; +} + +// TODO refactor: with createDataBlockScanInfo +int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t num) { + int32_t size = tSimpleHashGetSize(pReader->status.pTableMap); + + STableBlockScanInfo** p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pReader->status.pTableMap, p, &iter)) != NULL) { + clearBlockScanInfo(*p); + } + + if (size < num) { + int32_t code = ensureBlockScanInfoBuf(&pReader->blockInfoBuf, num); + if (code) { + return code; + } + + char* p1 = taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num); + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pReader->status.uidList.tableUidList = (uint64_t*)p1; + } + + tSimpleHashClear(pReader->status.pTableMap); + STableUidList* pUidList = &pReader->status.uidList; + pUidList->currentIndex = 0; + + STableKeyInfo* pList = (STableKeyInfo*)pTableList; + for (int32_t i = 0; i < num; ++i) { + STableBlockScanInfo* pInfo = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i); + pInfo->uid = pList[i].uid; + pUidList->tableUidList[i] = pList[i].uid; + + // todo extract method + if (ASCENDING_TRAVERSE(pReader->info.order)) { + int64_t skey = pReader->info.window.skey; + pInfo->lastKey = (skey > INT64_MIN) ? (skey - 1) : skey; + pInfo->lastKeyInStt = skey; + } else { + int64_t ekey = pReader->info.window.ekey; + pInfo->lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey; + pInfo->lastKeyInStt = ekey; + } + + tSimpleHashPut(pReader->status.pTableMap, &pInfo->uid, sizeof(uint64_t), &pInfo, POINTER_BYTES); + } + + return TDB_CODE_SUCCESS; +} + +void* tsdbGetIdx2(SMeta* pMeta) { + if (pMeta == NULL) { + return NULL; + } + return metaGetIdx(pMeta); +} + +void* tsdbGetIvtIdx2(SMeta* pMeta) { + if (pMeta == NULL) { + return NULL; + } + return metaGetIvtIdx(pMeta); +} + +uint64_t tsdbGetReaderMaxVersion2(STsdbReader* pReader) { return pReader->info.verRange.maxVer; } + +static int32_t doOpenReaderImpl(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + + initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); + resetDataBlockIterator(&pStatus->blockIter, pReader->info.order); + + int32_t code = TSDB_CODE_SUCCESS; + if (pStatus->fileIter.numOfFiles == 0) { + pStatus->loadFromFile = false; + } else if (READ_MODE_COUNT_ONLY == pReader->info.readMode) { + // DO NOTHING + } else { + code = initForFirstBlockInFile(pReader, pBlockIter); + } + + if (!pStatus->loadFromFile) { + resetTableListIndex(pStatus); + } + + return code; +} + +static void freeSchemaFunc(void* param) { + void** p = (void**)param; + taosMemoryFreeClear(*p); +} + +static void clearSharedPtr(STsdbReader* p) { + p->status.pTableMap = NULL; + p->status.uidList.tableUidList = NULL; + p->info.pSchema = NULL; + p->pReadSnap = NULL; + p->pSchemaMap = NULL; +} + +static void setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { + pDst->status.pTableMap = pSrc->status.pTableMap; + pDst->status.uidList = pSrc->status.uidList; + pDst->info.pSchema = pSrc->info.pSchema; + pDst->pSchemaMap = pSrc->pSchemaMap; + pDst->pReadSnap = pSrc->pReadSnap; + pDst->pReadSnap->pfSetArray = pSrc->pReadSnap->pfSetArray; + + if (pDst->info.pSchema) { + tsdbRowMergerInit(&pDst->status.merger, pDst->info.pSchema); + } +} + +// ====================================== EXPOSED APIs ====================================== +int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, + SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly, + SHashObj** pIgnoreTables) { + STimeWindow window = pCond->twindows; + SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); + + int32_t capacity = pConf->tsdbCfg.maxRows; + if (pResBlock != NULL) { + blockDataEnsureCapacity(pResBlock, capacity); + } + + int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + // check for query time window + STsdbReader* pReader = *ppReader; + if (isEmptyQueryTimeWindow(&pReader->info.window) && pCond->type == TIMEWINDOW_RANGE_CONTAINED) { + tsdbDebug("%p query window not overlaps with the data set, no result returned, %s", pReader, pReader->idStr); + return TSDB_CODE_SUCCESS; + } + + if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) { + // update the SQueryTableDataCond to create inner reader + int32_t order = pCond->order; + if (order == TSDB_ORDER_ASC) { + pCond->twindows.ekey = window.skey - 1; + pCond->twindows.skey = INT64_MIN; + pCond->order = TSDB_ORDER_DESC; + } else { + pCond->twindows.skey = window.ekey + 1; + pCond->twindows.ekey = INT64_MAX; + pCond->order = TSDB_ORDER_ASC; + } + + // here we only need one more row, so the capacity is set to be ONE. + code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[0], 1, pResBlock, idstr); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + if (order == TSDB_ORDER_ASC) { + pCond->twindows.skey = window.ekey + 1; + pCond->twindows.ekey = INT64_MAX; + } else { + pCond->twindows.skey = INT64_MIN; + pCond->twindows.ekey = window.ekey - 1; + } + pCond->order = order; + + code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[1], 1, pResBlock, idstr); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + } + + // NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here. + // no valid error code set in metaGetTbTSchema, so let's set the error code here. + // we should proceed in case of tmq processing. + if (pCond->suid != 0) { + pReader->info.pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, -1, 1); + if (pReader->info.pSchema == NULL) { + tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->info.suid, pReader->idStr); + } + } else if (numOfTables > 0) { + STableKeyInfo* pKey = pTableList; + pReader->info.pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1); + if (pReader->info.pSchema == NULL) { + tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr); + } + } + + if (pReader->info.pSchema != NULL) { + tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema); + } + + pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash); + if (pReader->pSchemaMap == NULL) { + tsdbError("failed init schema hash for reader %s", pReader->idStr); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + tSimpleHashSetFreeFp(pReader->pSchemaMap, freeSchemaFunc); + if (pReader->info.pSchema != NULL) { + code = updateBlockSMAInfo(pReader->info.pSchema, &pReader->suppInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + } + + STsdbReader* p = (pReader->innerReader[0] != NULL) ? pReader->innerReader[0] : pReader; + pReader->status.pTableMap = + createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables); + if (pReader->status.pTableMap == NULL) { + *ppReader = NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + if (pReader->status.pLDataIterArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + pReader->flag = READER_STATUS_SUSPEND; + + if (countOnly) { + pReader->info.readMode = READ_MODE_COUNT_ONLY; + } + + pReader->pIgnoreTables = pIgnoreTables; + + tsdbDebug("%p total numOfTable:%d, window:%" PRId64 " - %" PRId64 ", verRange:%" PRId64 " - %" PRId64 + " in this query %s", + pReader, numOfTables, pReader->info.window.skey, pReader->info.window.ekey, pReader->info.verRange.minVer, + pReader->info.verRange.maxVer, pReader->idStr); + + return code; + +_err: + tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); + tsdbReaderClose2(*ppReader); + *ppReader = NULL; // reset the pointer value. + return code; +} + +void tsdbReaderClose2(STsdbReader* pReader) { + if (pReader == NULL) { + return; + } + + tsdbAcquireReader(pReader); + + { + if (pReader->innerReader[0] != NULL || pReader->innerReader[1] != NULL) { + STsdbReader* p = pReader->innerReader[0]; + clearSharedPtr(p); + + p = pReader->innerReader[1]; + clearSharedPtr(p); + + tsdbReaderClose2(pReader->innerReader[0]); + tsdbReaderClose2(pReader->innerReader[1]); + } + } + + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + TARRAY2_DESTROY(&pSupInfo->colAggArray, NULL); + for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) { + if (pSupInfo->buildBuf[i] != NULL) { + taosMemoryFreeClear(pSupInfo->buildBuf[i]); + } + } + + if (pReader->resBlockInfo.freeBlock) { + pReader->resBlockInfo.pResBlock = blockDataDestroy(pReader->resBlockInfo.pResBlock); + } + + taosMemoryFree(pSupInfo->colId); + tBlockDataDestroy(&pReader->status.fileBlockData); + cleanupDataBlockIterator(&pReader->status.blockIter); + + size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + if (pReader->status.pTableMap != NULL) { + destroyAllBlockScanInfo(pReader->status.pTableMap); + clearBlockScanInfoBuf(&pReader->blockInfoBuf); + pReader->status.pTableMap = NULL; + } + + if (pReader->pFileReader != NULL) { + tsdbDataFileReaderClose(&pReader->pFileReader); + } + + qTrace("tsdb/reader-close: %p, untake snapshot", pReader); + tsdbUntakeReadSnap2(pReader, pReader->pReadSnap, true); + pReader->pReadSnap = NULL; + + tsdbReleaseReader(pReader); + tsdbUninitReaderLock(pReader); + + SCostSummary* pCost = &pReader->cost; + SFilesetIter* pFilesetIter = &pReader->status.fileIter; + if (pFilesetIter->pLastBlockReader != NULL) { + SLastBlockReader* pLReader = pFilesetIter->pLastBlockReader; + tMergeTreeClose(&pLReader->mergeTree); + taosMemoryFree(pLReader); + } + + destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->lastBlockLoad, &pCost->lastBlockLoadTime); + taosMemoryFreeClear(pReader->status.uidList.tableUidList); + + tsdbDebug( + "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 + " SMA-time:%.2f ms, fileBlocks:%" PRId64 + ", fileBlocks-load-time:%.2f ms, " + "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64 + ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, createTime:%.2f ms,createSkylineIterTime:%.2f " + "ms, initLastBlockReader:%.2fms, %s", + pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks, + pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, pCost->lastBlockLoadTime, pCost->composedBlocks, + pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pCost->createScanInfoList, + pCost->createSkylineIterTime, pCost->initLastBlockReader, pReader->idStr); + + taosMemoryFree(pReader->idStr); + + tsdbRowMergerCleanup(&pReader->status.merger); + taosMemoryFree(pReader->info.pSchema); + + tSimpleHashCleanup(pReader->pSchemaMap); + taosMemoryFreeClear(pReader); +} + +int32_t tsdbReaderSuspend2(STsdbReader* pReader) { + int32_t code = 0; + + // save reader's base state & reset top state to be reconstructed from base state + SReaderStatus* pStatus = &pReader->status; + STableBlockScanInfo* pBlockScanInfo = NULL; + + if (pStatus->loadFromFile) { + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + if (pBlockInfo != NULL) { + pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); + if (pBlockScanInfo == NULL) { + goto _err; + } + } else { + pBlockScanInfo = *pStatus->pTableIter; + } + + tsdbDataFileReaderClose(&pReader->pFileReader); + int64_t loadBlocks = 0; + double elapse = 0; + pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &loadBlocks, &elapse); + pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + // resetDataBlockScanInfo excluding lastKey + STableBlockScanInfo** p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { + STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; + + pInfo->iterInit = false; + pInfo->iter.hasVal = false; + pInfo->iiter.hasVal = false; + + if (pInfo->iter.iter != NULL) { + pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); + } + + if (pInfo->iiter.iter != NULL) { + pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); + } + + pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); + pInfo->pfileDelData = taosArrayDestroy(pInfo->pfileDelData); + } + } else { + // resetDataBlockScanInfo excluding lastKey + STableBlockScanInfo** p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { + STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; + + pInfo->iterInit = false; + pInfo->iter.hasVal = false; + pInfo->iiter.hasVal = false; + + if (pInfo->iter.iter != NULL) { + pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); + } + + if (pInfo->iiter.iter != NULL) { + pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); + } + + pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); + } + + pBlockScanInfo = pStatus->pTableIter == NULL ? NULL : *pStatus->pTableIter; + if (pBlockScanInfo) { + // save lastKey to restore memory iterator + STimeWindow w = pReader->resBlockInfo.pResBlock->info.window; + pBlockScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? w.ekey : w.skey; + + // reset current current table's data block scan info, + pBlockScanInfo->iterInit = false; + + pBlockScanInfo->iter.hasVal = false; + pBlockScanInfo->iiter.hasVal = false; + if (pBlockScanInfo->iter.iter != NULL) { + pBlockScanInfo->iter.iter = tsdbTbDataIterDestroy(pBlockScanInfo->iter.iter); + } + + if (pBlockScanInfo->iiter.iter != NULL) { + pBlockScanInfo->iiter.iter = tsdbTbDataIterDestroy(pBlockScanInfo->iiter.iter); + } + + pBlockScanInfo->pBlockList = taosArrayDestroy(pBlockScanInfo->pBlockList); + // TODO: keep skyline for reuse + pBlockScanInfo->delSkyline = taosArrayDestroy(pBlockScanInfo->delSkyline); + } + } + + tsdbUntakeReadSnap2(pReader, pReader->pReadSnap, false); + pReader->pReadSnap = NULL; + pReader->flag = READER_STATUS_SUSPEND; + + tsdbDebug("reader: %p suspended uid %" PRIu64 " in this query %s", pReader, pBlockScanInfo ? pBlockScanInfo->uid : 0, + pReader->idStr); + return code; + +_err: + tsdbError("failed to suspend data reader, code:%s %s", tstrerror(code), pReader->idStr); + return code; +} + +static int32_t tsdbSetQueryReseek(void* pQHandle) { + int32_t code = 0; + STsdbReader* pReader = pQHandle; + + code = tsdbTryAcquireReader(pReader); + if (code == 0) { + if (pReader->flag == READER_STATUS_SUSPEND) { + tsdbReleaseReader(pReader); + return code; + } + + tsdbReaderSuspend2(pReader); + + tsdbReleaseReader(pReader); + + return code; + } else if (code == EBUSY) { + return TSDB_CODE_VND_QUERY_BUSY; + } else { + terrno = TAOS_SYSTEM_ERROR(code); + return TSDB_CODE_FAILED; + } +} + +int32_t tsdbReaderResume2(STsdbReader* pReader) { + int32_t code = 0; + + STableBlockScanInfo** pBlockScanInfo = pReader->status.pTableIter; + + // restore reader's state + // task snapshot + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + if (numOfTables > 0) { + qTrace("tsdb/reader: %p, take snapshot", pReader); + code = tsdbTakeReadSnap2(pReader, tsdbSetQueryReseek, &pReader->pReadSnap); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + STsdbReader* pPrevReader = pReader->innerReader[0]; + STsdbReader* pNextReader = pReader->innerReader[1]; + + // we need only one row + pPrevReader->resBlockInfo.capacity = 1; + setSharedPtr(pPrevReader, pReader); + + pNextReader->resBlockInfo.capacity = 1; + setSharedPtr(pNextReader, pReader); + + code = doOpenReaderImpl(pPrevReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } + + pReader->flag = READER_STATUS_NORMAL; + tsdbDebug("reader: %p resumed uid %" PRIu64 ", numOfTable:%" PRId32 ", in this query %s", pReader, + pBlockScanInfo ? (*pBlockScanInfo)->uid : 0, numOfTables, pReader->idStr); + return code; + +_err: + tsdbError("failed to resume data reader, code:%s %s", tstrerror(code), pReader->idStr); + return code; +} + +static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + + if (pReader->status.loadFromFile == false) { + return false; + } + + code = readRowsCountFromFiles(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + code = readRowsCountFromMem(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + pBlock->info.rows = pReader->rowsNum; + pBlock->info.id.uid = 0; + pBlock->info.dataLoad = 0; + + pReader->rowsNum = 0; + + return pBlock->info.rows > 0; +} + +static int32_t doTsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { + int32_t code = TSDB_CODE_SUCCESS; + + // cleanup the data that belongs to the previous data block + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + blockDataCleanup(pBlock); + + *hasNext = false; + + SReaderStatus* pStatus = &pReader->status; + if (tSimpleHashGetSize(pStatus->pTableMap) == 0) { + return code; + } + + if (READ_MODE_COUNT_ONLY == pReader->info.readMode) { + return tsdbReadRowsCountOnly(pReader); + } + + if (pStatus->loadFromFile) { + code = buildBlockFromFiles(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (pBlock->info.rows <= 0) { + resetTableListIndex(&pReader->status); + code = buildBlockFromBufferSequentially(pReader); + } + } else { // no data in files, let's try the buffer + code = buildBlockFromBufferSequentially(pReader); + } + + *hasNext = pBlock->info.rows > 0; + + return code; +} + +int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { + int32_t code = TSDB_CODE_SUCCESS; + + *hasNext = false; + + if (isEmptyQueryTimeWindow(&pReader->info.window) || pReader->step == EXTERNAL_ROWS_NEXT || + pReader->code != TSDB_CODE_SUCCESS) { + return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code; + } + + SReaderStatus* pStatus = &pReader->status; + + code = tsdbAcquireReader(pReader); + qTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); + + if (pReader->flag == READER_STATUS_SUSPEND) { + code = tsdbReaderResume2(pReader); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + } + + if (pReader->innerReader[0] != NULL && pReader->step == 0) { + code = doTsdbNextDataBlock2(pReader->innerReader[0], hasNext); + if (code) { + tsdbReleaseReader(pReader); + return code; + } + + pReader->step = EXTERNAL_ROWS_PREV; + if (*hasNext) { + pStatus = &pReader->innerReader[0]->status; + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + } + + return code; + } + } + + if (pReader->step == EXTERNAL_ROWS_PREV) { + // prepare for the main scan + code = doOpenReaderImpl(pReader); + int32_t step = 1; + resetAllDataBlockScanInfo(pReader->status.pTableMap, pReader->innerReader[0]->info.window.ekey, step); + + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + pReader->step = EXTERNAL_ROWS_MAIN; + } + + code = doTsdbNextDataBlock2(pReader, hasNext); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + + if (*hasNext) { + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + } + + return code; + } + + if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) { + // prepare for the next row scan + int32_t step = -1; + code = doOpenReaderImpl(pReader->innerReader[1]); + resetAllDataBlockScanInfo(pReader->innerReader[1]->status.pTableMap, pReader->info.window.ekey, step); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doTsdbNextDataBlock2(pReader->innerReader[1], hasNext); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + + pReader->step = EXTERNAL_ROWS_NEXT; + if (*hasNext) { + pStatus = &pReader->innerReader[1]->status; + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + } + + return code; + } + } + + qTrace("tsdb/read: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + + return code; +} + +static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { + // do fill all null column value SMA info + int32_t i = 0, j = 0; + int32_t size = (int32_t)TARRAY2_SIZE(&pSup->colAggArray); + TARRAY2_INSERT_PTR(&pSup->colAggArray, 0, pTsAgg); + size++; + + while (j < numOfCols && i < size) { + SColumnDataAgg* pAgg = &pSup->colAggArray.data[i]; + if (pAgg->colId == pSup->colId[j]) { + i += 1; + j += 1; + } else if (pAgg->colId < pSup->colId[j]) { + i += 1; + } else if (pSup->colId[j] < pAgg->colId) { + if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; + TARRAY2_INSERT_PTR(&pSup->colAggArray, i, &nullColAgg); + i += 1; + size++; + } + j += 1; + } + } + + while (j < numOfCols) { + if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; + TARRAY2_INSERT_PTR(&pSup->colAggArray, i, &nullColAgg); + i += 1; + } + j++; + } +} + +int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool* hasNullSMA) { + SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg; + + int32_t code = 0; + *allHave = false; + *pBlockSMA = NULL; + + if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { + return TSDB_CODE_SUCCESS; + } + + // there is no statistics data for composed block + if (pReader->status.composedDataBlock || (!pReader->suppInfo.smaValid)) { + return TSDB_CODE_SUCCESS; + } + + SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); + SBlockLoadSuppInfo* pSup = &pReader->suppInfo; + + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + if (pResBlock->info.id.uid != pFBlock->uid) { + return TSDB_CODE_SUCCESS; + } + + // int64_t st = taosGetTimestampUs(); + TARRAY2_CLEAR(&pSup->colAggArray, 0); + + code = tsdbDataFileReadBlockSma(pReader->pFileReader, &pFBlock->record, &pSup->colAggArray); + if (code != TSDB_CODE_SUCCESS) { + tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code), + pReader->idStr); + return code; + } + + if (pSup->colAggArray.size > 0) { + *allHave = true; + } else { + *pBlockSMA = NULL; + return TSDB_CODE_SUCCESS; + } + + // always load the first primary timestamp column data + SColumnDataAgg* pTsAgg = &pSup->tsColAgg; + + pTsAgg->numOfNull = 0; + pTsAgg->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + pTsAgg->min = pResBlock->info.window.skey; + pTsAgg->max = pResBlock->info.window.ekey; + + // update the number of NULL data rows + size_t numOfCols = pSup->numOfCols; + + if (pResBlock->pBlockAgg == NULL) { + size_t num = taosArrayGetSize(pResBlock->pDataBlock); + pResBlock->pBlockAgg = taosMemoryCalloc(num, POINTER_BYTES); + } + + // do fill all null column value SMA info + doFillNullColSMA(pSup, pFBlock->record.numRow, numOfCols, pTsAgg); + + size_t size = pSup->colAggArray.size; + + int32_t i = 0, j = 0; + while (j < numOfCols && i < size) { + SColumnDataAgg* pAgg = &pSup->colAggArray.data[i]; + if (pAgg->colId == pSup->colId[j]) { + pResBlock->pBlockAgg[pSup->slotId[j]] = pAgg; + i += 1; + j += 1; + } else if (pAgg->colId < pSup->colId[j]) { + i += 1; + } else if (pSup->colId[j] < pAgg->colId) { + pResBlock->pBlockAgg[pSup->slotId[j]] = NULL; + *allHave = false; + j += 1; + } + } + + *pBlockSMA = pResBlock->pBlockAgg; + pReader->cost.smaDataLoad += 1; + + // double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; + pReader->cost.smaLoadTime += 0; // elapsedTime; + + tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr); + return code; +} + +static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + int32_t code = TSDB_CODE_SUCCESS; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); + + if (pReader->code != TSDB_CODE_SUCCESS) { + return NULL; + } + + STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); + if (pBlockScanInfo == NULL) { + return NULL; + } + + code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); + if (code != TSDB_CODE_SUCCESS) { + tBlockDataDestroy(&pStatus->fileBlockData); + terrno = code; + return NULL; + } + + code = copyBlockDataToSDataBlock(pReader); + if (code != TSDB_CODE_SUCCESS) { + tBlockDataDestroy(&pStatus->fileBlockData); + terrno = code; + return NULL; + } + + return pReader->resBlockInfo.pResBlock; +} + +SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) { + STsdbReader* pTReader = pReader; + if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { + if (pReader->step == EXTERNAL_ROWS_PREV) { + pTReader = pReader->innerReader[0]; + } else if (pReader->step == EXTERNAL_ROWS_NEXT) { + pTReader = pReader->innerReader[1]; + } + } + + SReaderStatus* pStatus = &pTReader->status; + if (pStatus->composedDataBlock) { + return pTReader->resBlockInfo.pResBlock; + } + + SSDataBlock* ret = doRetrieveDataBlock(pTReader); + + qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + + return ret; +} + +int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { + int32_t code = TSDB_CODE_SUCCESS; + + qTrace("tsdb/reader-reset: %p, take read mutex", pReader); + tsdbAcquireReader(pReader); + + if (pReader->flag == READER_STATUS_SUSPEND) { + code = tsdbReaderResume2(pReader); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + } + + if (isEmptyQueryTimeWindow(&pReader->info.window) || pReader->pReadSnap == NULL) { + tsdbDebug("tsdb reader reset return %p, %s", pReader->pReadSnap, pReader->idStr); + tsdbReleaseReader(pReader); + return TSDB_CODE_SUCCESS; + } + + SReaderStatus* pStatus = &pReader->status; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + + pReader->info.order = pCond->order; + pReader->type = TIMEWINDOW_RANGE_CONTAINED; + pStatus->loadFromFile = true; + pStatus->pTableIter = NULL; + pReader->info.window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); + + // allocate buffer in order to load data blocks from file + memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg)); + + pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; + tsdbDataFileReaderClose(&pReader->pFileReader); + + int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap); + + initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); + resetDataBlockIterator(pBlockIter, pReader->info.order); + resetTableListIndex(&pReader->status); + + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + int32_t step = asc ? 1 : -1; + int64_t ts = asc ? pReader->info.window.skey - 1 : pReader->info.window.ekey + 1; + resetAllDataBlockScanInfo(pStatus->pTableMap, ts, step); + + // no data in files, let's try buffer in memory + if (pStatus->fileIter.numOfFiles == 0) { + pStatus->loadFromFile = false; + resetTableListIndex(pStatus); + } else { + code = initForFirstBlockInFile(pReader, pBlockIter); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%p reset reader failed, numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s", pReader, + numOfTables, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); + + tsdbReleaseReader(pReader); + return code; + } + } + + tsdbDebug("%p reset reader, suid:%" PRIu64 ", numOfTables:%d, skey:%" PRId64 ", query range:%" PRId64 " - %" PRId64 + " in query %s", + pReader, pReader->info.suid, numOfTables, pCond->twindows.skey, pReader->info.window.skey, + pReader->info.window.ekey, pReader->idStr); + + tsdbReleaseReader(pReader); + + return code; +} + +static int32_t getBucketIndex(int32_t startRow, int32_t bucketRange, int32_t numOfRows, int32_t numOfBucket) { + if (numOfRows < startRow) { + return 0; + } + int32_t bucketIndex = ((numOfRows - startRow) / bucketRange); + if (bucketIndex == numOfBucket) { + bucketIndex -= 1; + } + return bucketIndex; +} + +int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pTableBlockInfo) { + int32_t code = TSDB_CODE_SUCCESS; + pTableBlockInfo->totalSize = 0; + pTableBlockInfo->totalRows = 0; + pTableBlockInfo->numOfVgroups = 1; + + const int32_t numOfBuckets = 20.0; + const int32_t defaultRows = 4096; + + // find the start data block in file + tsdbAcquireReader(pReader); + if (pReader->flag == READER_STATUS_SUSPEND) { + code = tsdbReaderResume2(pReader); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + } + SReaderStatus* pStatus = &pReader->status; + + STsdbCfg* pc = &pReader->pTsdb->pVnode->config.tsdbCfg; + pTableBlockInfo->defMinRows = pc->minRows; + pTableBlockInfo->defMaxRows = pc->maxRows; + + int32_t bucketRange = ceil(((double)(pc->maxRows - pc->minRows)) / numOfBuckets); + + pTableBlockInfo->numOfFiles += 1; + + int32_t numOfTables = (int32_t)tSimpleHashGetSize(pStatus->pTableMap); + + SDataBlockIter* pBlockIter = &pStatus->blockIter; + pTableBlockInfo->numOfFiles += pStatus->fileIter.numOfFiles; + + if (pBlockIter->numOfBlocks > 0) { + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; + } + + pTableBlockInfo->numOfTables = numOfTables; + bool hasNext = (pBlockIter->numOfBlocks > 0); + + while (true) { + if (hasNext) { + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + int32_t numOfRows = pBlockInfo->record.numRow; + + pTableBlockInfo->totalRows += numOfRows; + + if (numOfRows > pTableBlockInfo->maxRows) { + pTableBlockInfo->maxRows = numOfRows; + } + + if (numOfRows < pTableBlockInfo->minRows) { + pTableBlockInfo->minRows = numOfRows; + } + + if (numOfRows < defaultRows) { + pTableBlockInfo->numOfSmallBlocks += 1; + } + + pTableBlockInfo->totalSize += pBlockInfo->record.blockSize; + + int32_t bucketIndex = getBucketIndex(pTableBlockInfo->defMinRows, bucketRange, numOfRows, numOfBuckets); + pTableBlockInfo->blockRowsHisto[bucketIndex]++; + + hasNext = blockIteratorNext(&pStatus->blockIter, pReader->idStr); + } else { + code = initForFirstBlockInFile(pReader, pBlockIter); + if ((code != TSDB_CODE_SUCCESS) || (pStatus->loadFromFile == false)) { + break; + } + + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; + hasNext = (pBlockIter->numOfBlocks > 0); + } + + // tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, + // pReader->pFileGroup->fid, pReader->idStr); + } + tsdbReleaseReader(pReader); + return code; +} + +int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t rows = 0; + + SReaderStatus* pStatus = &pReader->status; + tsdbAcquireReader(pReader); + if (pReader->flag == READER_STATUS_SUSPEND) { + code = tsdbReaderResume2(pReader); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + } + + int32_t iter = 0; + pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter); + + while (pStatus->pTableIter != NULL) { + STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; + + STbData* d = NULL; + if (pReader->pReadSnap->pMem != NULL) { + d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->info.suid, pBlockScanInfo->uid); + if (d != NULL) { + rows += tsdbGetNRowsInTbData(d); + } + } + + STbData* di = NULL; + if (pReader->pReadSnap->pIMem != NULL) { + di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->info.suid, pBlockScanInfo->uid); + if (di != NULL) { + rows += tsdbGetNRowsInTbData(di); + } + } + + // current table is exhausted, let's try the next table + pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, pStatus->pTableIter, &iter); + } + + tsdbReleaseReader(pReader); + + return rows; +} + +int32_t tsdbGetTableSchema2(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { + SMetaReader mr = {0}; + metaReaderDoInit(&mr, ((SVnode*)pVnode)->pMeta, 0); + int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid); + if (code != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; + metaReaderClear(&mr); + return terrno; + } + + *suid = 0; + + // only child table and ordinary table is allowed, super table is not allowed. + if (mr.me.type == TSDB_CHILD_TABLE) { + tDecoderClear(&mr.coder); + *suid = mr.me.ctbEntry.suid; + code = metaReaderGetTableEntryByUidCache(&mr, *suid); + if (code != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; + metaReaderClear(&mr); + return terrno; + } + } else if (mr.me.type == TSDB_NORMAL_TABLE) { // do nothing + } else { + terrno = TSDB_CODE_INVALID_PARA; + metaReaderClear(&mr); + return terrno; + } + + metaReaderClear(&mr); + + // get the newest table schema version + code = metaGetTbTSchemaEx(((SVnode*)pVnode)->pMeta, *suid, uid, -1, pSchema); + return code; +} + +int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STsdbReadSnap** ppSnap) { + int32_t code = 0; + STsdb* pTsdb = pReader->pTsdb; + SVersionRange* pRange = &pReader->info.verRange; + + // lock + taosThreadRwlockRdlock(&pTsdb->rwLock); + + // alloc + STsdbReadSnap* pSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(STsdbReadSnap)); + if (pSnap == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + // take snapshot + if (pTsdb->mem && (pRange->minVer <= pTsdb->mem->maxVer && pRange->maxVer >= pTsdb->mem->minVer)) { + pSnap->pMem = pTsdb->mem; + pSnap->pNode = taosMemoryMalloc(sizeof(*pSnap->pNode)); + if (pSnap->pNode == NULL) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pSnap->pNode->pQHandle = pReader; + pSnap->pNode->reseek = reseek; + + tsdbRefMemTable(pTsdb->mem, pSnap->pNode); + } + + if (pTsdb->imem && (pRange->minVer <= pTsdb->imem->maxVer && pRange->maxVer >= pTsdb->imem->minVer)) { + pSnap->pIMem = pTsdb->imem; + pSnap->pINode = taosMemoryMalloc(sizeof(*pSnap->pINode)); + if (pSnap->pINode == NULL) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pSnap->pINode->pQHandle = pReader; + pSnap->pINode->reseek = reseek; + + tsdbRefMemTable(pTsdb->imem, pSnap->pINode); + } + + // fs + code = tsdbFSCreateRefSnapshot(pTsdb->pFS, &pSnap->pfSetArray); + if (code) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; + } + + // unlock + taosThreadRwlockUnlock(&pTsdb->rwLock); + + tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); + +_exit: + if (code) { + *ppSnap = NULL; + if (pSnap) { + if (pSnap->pNode) taosMemoryFree(pSnap->pNode); + if (pSnap->pINode) taosMemoryFree(pSnap->pINode); + taosMemoryFree(pSnap); + } + } else { + *ppSnap = pSnap; + } + + return code; +} + +void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proactive) { + STsdb* pTsdb = pReader->pTsdb; + + if (pSnap) { + if (pSnap->pMem) { + tsdbUnrefMemTable(pSnap->pMem, pSnap->pNode, proactive); + } + + if (pSnap->pIMem) { + tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive); + } + + tsdbFSUnref(pTsdb, &pSnap->fs); + if (pSnap->pNode) taosMemoryFree(pSnap->pNode); + if (pSnap->pINode) taosMemoryFree(pSnap->pINode); + + tsdbFSDestroyRefSnapshot(&pSnap->pfSetArray); + + taosMemoryFree(pSnap); + } + tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode)); +} + +// if failed, do nothing +void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { + taosMemoryFreeClear(pReader->idStr); + pReader->idStr = taosStrdup(idstr); + pReader->status.fileIter.pLastBlockReader->mergeTree.idStr = pReader->idStr; +} + +void tsdbReaderSetCloseFlag2(STsdbReader* pReader) { pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..d560f0d5af7a00b775cd22a3f73380dd39918c59 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -0,0 +1,630 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbReadUtil.h" +#include "osDef.h" +#include "tsdb.h" +#include "tsdbDataFileRW.h" +#include "tsdbFS2.h" +#include "tsdbMerge.h" +#include "tsdbUtil2.h" +#include "tsimplehash.h" + +int32_t uidComparFunc(const void* p1, const void* p2) { + uint64_t pu1 = *(uint64_t*)p1; + uint64_t pu2 = *(uint64_t*)p2; + if (pu1 == pu2) { + return 0; + } else { + return (pu1 < pu2) ? -1 : 1; + } +} + +static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { + int32_t num = numOfTables / pBuf->numPerBucket; + int32_t remainder = numOfTables % pBuf->numPerBucket; + if (pBuf->pData == NULL) { + pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES); + } + + for (int32_t i = 0; i < num; ++i) { + char* p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosArrayPush(pBuf->pData, &p); + } + + if (remainder > 0) { + char* p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosArrayPush(pBuf->pData, &p); + } + + pBuf->numOfTables = numOfTables; + + return TSDB_CODE_SUCCESS; +} + +int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { + if (numOfTables <= pBuf->numOfTables) { + return TSDB_CODE_SUCCESS; + } + + if (pBuf->numOfTables > 0) { + STableBlockScanInfo** p = (STableBlockScanInfo**)taosArrayPop(pBuf->pData); + taosMemoryFree(*p); + pBuf->numOfTables /= pBuf->numPerBucket; + } + + int32_t num = (numOfTables - pBuf->numOfTables) / pBuf->numPerBucket; + int32_t remainder = (numOfTables - pBuf->numOfTables) % pBuf->numPerBucket; + if (pBuf->pData == NULL) { + pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES); + } + + for (int32_t i = 0; i < num; ++i) { + char* p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosArrayPush(pBuf->pData, &p); + } + + if (remainder > 0) { + char* p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosArrayPush(pBuf->pData, &p); + } + + pBuf->numOfTables = numOfTables; + + return TSDB_CODE_SUCCESS; +} + +void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) { + size_t num = taosArrayGetSize(pBuf->pData); + for (int32_t i = 0; i < num; ++i) { + char** p = taosArrayGet(pBuf->pData, i); + taosMemoryFree(*p); + } + + taosArrayDestroy(pBuf->pData); +} + +void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) { + int32_t bucketIndex = index / pBuf->numPerBucket; + char** pBucket = taosArrayGet(pBuf->pData, bucketIndex); + return (*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo); +} + +STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id) { + STableBlockScanInfo** p = tSimpleHashGet(pTableMap, &uid, sizeof(uid)); + if (p == NULL || *p == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + int32_t size = tSimpleHashGetSize(pTableMap); + tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id); + return NULL; + } + + return *p; +} + +// NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model +SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, + STableUidList* pUidList, int32_t numOfTables) { + // allocate buffer in order to load data blocks from file + // todo use simple hash instead, optimize the memory consumption + SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); + if (pTableMap == NULL) { + return NULL; + } + + int64_t st = taosGetTimestampUs(); + initBlockScanInfoBuf(pBuf, numOfTables); + + pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); + if (pUidList->tableUidList == NULL) { + tSimpleHashCleanup(pTableMap); + return NULL; + } + + pUidList->currentIndex = 0; + + for (int32_t j = 0; j < numOfTables; ++j) { + STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(pBuf, j); + + pScanInfo->uid = idList[j].uid; + pUidList->tableUidList[j] = idList[j].uid; + + if (ASCENDING_TRAVERSE(pTsdbReader->info.order)) { + int64_t skey = pTsdbReader->info.window.skey; + pScanInfo->lastKey = (skey > INT64_MIN) ? (skey - 1) : skey; + pScanInfo->lastKeyInStt = skey; + } else { + int64_t ekey = pTsdbReader->info.window.ekey; + pScanInfo->lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey; + pScanInfo->lastKeyInStt = ekey; + } + + tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); + tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReader, pScanInfo->uid, + pScanInfo->lastKey, pTsdbReader->idStr); + } + + taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc); + + pTsdbReader->cost.createScanInfoList = (taosGetTimestampUs() - st) / 1000.0; + tsdbDebug("%p create %d tables scan-info, size:%.2f Kb, elapsed time:%.2f ms, %s", pTsdbReader, numOfTables, + (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList, + pTsdbReader->idStr); + + return pTableMap; +} + +void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { + void* p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { + STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; + + pInfo->iterInit = false; + pInfo->iter.hasVal = false; + pInfo->iiter.hasVal = false; + + if (pInfo->iter.iter != NULL) { + pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); + } + + if (pInfo->iiter.iter != NULL) { + pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); + } + + pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); + pInfo->lastKey = ts; + pInfo->lastKeyInStt = ts + step; + } +} + +void clearBlockScanInfo(STableBlockScanInfo* p) { + p->iterInit = false; + p->iter.hasVal = false; + p->iiter.hasVal = false; + + if (p->iter.iter != NULL) { + p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter); + } + + if (p->iiter.iter != NULL) { + p->iiter.iter = tsdbTbDataIterDestroy(p->iiter.iter); + } + + p->delSkyline = taosArrayDestroy(p->delSkyline); + p->pBlockList = taosArrayDestroy(p->pBlockList); + p->pMemDelData = taosArrayDestroy(p->pMemDelData); + p->pfileDelData = taosArrayDestroy(p->pfileDelData); +} + +void destroyAllBlockScanInfo(SSHashObj* pTableMap) { + void* p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { + clearBlockScanInfo(*(STableBlockScanInfo**)p); + } + + tSimpleHashCleanup(pTableMap); +} + +static void doCleanupInfoForNextFileset(STableBlockScanInfo* pScanInfo) { + // reset the index in last block when handing a new file + taosArrayClear(pScanInfo->pBlockList); + taosArrayClear(pScanInfo->pfileDelData); // del data from each file set +} + +void cleanupInfoFoxNextFileset(SSHashObj* pTableMap) { + STableBlockScanInfo** p = NULL; + + int32_t iter = 0; + while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { + doCleanupInfoForNextFileset(*p); + } +} + +// brin records iterator +void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList) { + memset(&pIter->block, 0, sizeof(SBrinBlock)); + memset(&pIter->record, 0, sizeof(SBrinRecord)); + pIter->blockIndex = -1; + pIter->recordIndex = -1; + + pIter->pReader = pReader; + pIter->pBrinBlockList = pList; +} + +SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) { + if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= TARRAY2_SIZE(pIter->block.numRow)) { + pIter->blockIndex += 1; + if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) { + return NULL; + } + + pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex); + + tBrinBlockClear(&pIter->block); + tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); + pIter->recordIndex = -1; + } + + pIter->recordIndex += 1; + tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record); + return &pIter->record; +} + +void clearBrinBlockIter(SBrinRecordIter* pIter) { tBrinBlockDestroy(&pIter->block); } + +// initialize the file block access order +// sort the file blocks according to the offset of each data block in the files +static void cleanupBlockOrderSupporter(SBlockOrderSupporter* pSup) { + taosMemoryFreeClear(pSup->numOfBlocksPerTable); + taosMemoryFreeClear(pSup->indexPerTable); + + for (int32_t i = 0; i < pSup->numOfTables; ++i) { + SBlockOrderWrapper* pBlockInfo = pSup->pDataBlockInfo[i]; + taosMemoryFreeClear(pBlockInfo); + } + + taosMemoryFreeClear(pSup->pDataBlockInfo); +} + +static int32_t initBlockOrderSupporter(SBlockOrderSupporter* pSup, int32_t numOfTables) { + pSup->numOfBlocksPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); + pSup->indexPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); + pSup->pDataBlockInfo = taosMemoryCalloc(1, POINTER_BYTES * numOfTables); + + if (pSup->numOfBlocksPerTable == NULL || pSup->indexPerTable == NULL || pSup->pDataBlockInfo == NULL) { + cleanupBlockOrderSupporter(pSup); + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) { + int32_t leftIndex = *(int32_t*)pLeft; + int32_t rightIndex = *(int32_t*)pRight; + + SBlockOrderSupporter* pSupporter = (SBlockOrderSupporter*)param; + + int32_t leftTableBlockIndex = pSupporter->indexPerTable[leftIndex]; + int32_t rightTableBlockIndex = pSupporter->indexPerTable[rightIndex]; + + if (leftTableBlockIndex > pSupporter->numOfBlocksPerTable[leftIndex]) { + /* left block is empty */ + return 1; + } else if (rightTableBlockIndex > pSupporter->numOfBlocksPerTable[rightIndex]) { + /* right block is empty */ + return -1; + } + + SBlockOrderWrapper* pLeftBlock = &pSupporter->pDataBlockInfo[leftIndex][leftTableBlockIndex]; + SBlockOrderWrapper* pRightBlock = &pSupporter->pDataBlockInfo[rightIndex][rightTableBlockIndex]; + + return pLeftBlock->offset > pRightBlock->offset ? 1 : -1; +} + +int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) { + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + + SBlockOrderSupporter sup = {0}; + pBlockIter->numOfBlocks = numOfBlocks; + taosArrayClear(pBlockIter->blockList); + + pBlockIter->pTableMap = pReader->status.pTableMap; + + // access data blocks according to the offset of each block in asc/desc order. + int32_t numOfTables = taosArrayGetSize(pTableList); + + int64_t st = taosGetTimestampUs(); + int32_t code = initBlockOrderSupporter(&sup, numOfTables); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int32_t cnt = 0; + + for (int32_t i = 0; i < numOfTables; ++i) { + STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, i); + // ASSERT(pTableScanInfo->pBlockList != NULL && taosArrayGetSize(pTableScanInfo->pBlockList) > 0); + + size_t num = taosArrayGetSize(pTableScanInfo->pBlockList); + sup.numOfBlocksPerTable[sup.numOfTables] = num; + + char* buf = taosMemoryMalloc(sizeof(SBlockOrderWrapper) * num); + if (buf == NULL) { + cleanupBlockOrderSupporter(&sup); + return TSDB_CODE_OUT_OF_MEMORY; + } + + sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf; + + for (int32_t k = 0; k < num; ++k) { + SBrinRecord* pRecord = taosArrayGet(pTableScanInfo->pBlockList, k); + sup.pDataBlockInfo[sup.numOfTables][k] = + (SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pRecord->blockOffset, .pInfo = pTableScanInfo}; + cnt++; + } + + sup.numOfTables += 1; + } + + if (numOfBlocks != cnt && sup.numOfTables != numOfTables) { + cleanupBlockOrderSupporter(&sup); + return TSDB_CODE_INVALID_PARA; + } + + // since there is only one table qualified, blocks are not sorted + if (sup.numOfTables == 1) { + for (int32_t i = 0; i < numOfBlocks; ++i) { + SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[0][i].uid, .tbBlockIdx = i}; + blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[0][i].pInfo->pBlockList, i); + + taosArrayPush(pBlockIter->blockList, &blockInfo); + } + + int64_t et = taosGetTimestampUs(); + tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s", + pReader, numOfBlocks, (et - st) / 1000.0, pReader->idStr); + + pBlockIter->index = asc ? 0 : (numOfBlocks - 1); + cleanupBlockOrderSupporter(&sup); + return TSDB_CODE_SUCCESS; + } + + tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pReader, cnt, sup.numOfTables, + pReader->idStr); + + SMultiwayMergeTreeInfo* pTree = NULL; + + uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); + if (ret != TSDB_CODE_SUCCESS) { + cleanupBlockOrderSupporter(&sup); + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t numOfTotal = 0; + while (numOfTotal < cnt) { + int32_t pos = tMergeTreeGetChosenIndex(pTree); + int32_t index = sup.indexPerTable[pos]++; + + SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[pos][index].uid, .tbBlockIdx = index}; + blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); + + taosArrayPush(pBlockIter->blockList, &blockInfo); + + // set data block index overflow, in order to disable the offset comparator + if (sup.indexPerTable[pos] >= sup.numOfBlocksPerTable[pos]) { + sup.indexPerTable[pos] = sup.numOfBlocksPerTable[pos] + 1; + } + + numOfTotal += 1; + tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree)); + } + + int64_t et = taosGetTimestampUs(); + tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks, + (et - st) / 1000.0, pReader->idStr); + cleanupBlockOrderSupporter(&sup); + taosMemoryFree(pTree); + + pBlockIter->index = asc ? 0 : (numOfBlocks - 1); + return TSDB_CODE_SUCCESS; +} + +bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr) { + bool asc = ASCENDING_TRAVERSE(pBlockIter->order); + + int32_t step = asc ? 1 : -1; + if ((pBlockIter->index >= pBlockIter->numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) { + return false; + } + + pBlockIter->index += step; + return true; +} + +typedef enum { + BLK_CHECK_CONTINUE = 0x1, + BLK_CHECK_QUIT = 0x2, +} ETombBlkCheckEnum; + +static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_t numOfTables, int32_t* j, + STableBlockScanInfo** pScanInfo, ETombBlkCheckEnum* pRet) { + int32_t code = 0; + STombRecord record = {0}; + uint64_t uid = pReader->status.uidList.tableUidList[*j]; + + for (int32_t k = 0; k < TARRAY2_SIZE(pBlock->suid); ++k) { + code = tTombBlockGet(pBlock, k, &record); + if (code != TSDB_CODE_SUCCESS) { + *pRet = BLK_CHECK_QUIT; + return code; + } + + if (record.suid < pReader->info.suid) { + continue; + } + + if (record.suid > pReader->info.suid) { + *pRet = BLK_CHECK_QUIT; + return TSDB_CODE_SUCCESS; + } + + bool newTable = false; + if (uid < record.uid) { + while ((*j) < numOfTables && pReader->status.uidList.tableUidList[*j] < record.uid) { + (*j) += 1; + newTable = true; + } + + if ((*j) >= numOfTables) { + *pRet = BLK_CHECK_QUIT; + return TSDB_CODE_SUCCESS; + } + + uid = pReader->status.uidList.tableUidList[*j]; + } + + if (record.uid < uid) { + continue; + } + + ASSERT(record.suid == pReader->info.suid && uid == record.uid); + + if (newTable) { + (*pScanInfo) = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + if ((*pScanInfo)->pfileDelData == NULL) { + (*pScanInfo)->pfileDelData = taosArrayInit(4, sizeof(SDelData)); + } + } + + if (record.version <= pReader->info.verRange.maxVer) { + SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey}; + taosArrayPush((*pScanInfo)->pfileDelData, &delData); + } + } + + *pRet = BLK_CHECK_CONTINUE; + return TSDB_CODE_SUCCESS; +} + +// load tomb data API +static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STsdbReader* pReader, void* pFileReader, + bool isFile) { + int32_t code = 0; + STableUidList* pList = &pReader->status.uidList; + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + + int32_t i = 0, j = 0; + while (i < pTombBlkArray->size && j < numOfTables) { + STombBlk* pTombBlk = &pTombBlkArray->data[i]; + if (pTombBlk->maxTbid.suid < pReader->info.suid) { + i += 1; + continue; + } + + if (pTombBlk->minTbid.suid > pReader->info.suid) { + break; + } + + ASSERT(pTombBlk->minTbid.suid <= pReader->info.suid && pTombBlk->maxTbid.suid >= pReader->info.suid); + if (pTombBlk->maxTbid.suid == pReader->info.suid && pTombBlk->maxTbid.uid < pList->tableUidList[0]) { + i += 1; + continue; + } + + if (pTombBlk->minTbid.suid == pReader->info.suid && pTombBlk->minTbid.uid > pList->tableUidList[numOfTables - 1]) { + break; + } + + STombBlock block = {0}; + code = isFile ? tsdbDataFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block) + : tsdbSttFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + uint64_t uid = pReader->status.uidList.tableUidList[j]; + + STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + if (pScanInfo->pfileDelData == NULL) { + pScanInfo->pfileDelData = taosArrayInit(4, sizeof(SDelData)); + } + + ETombBlkCheckEnum ret = 0; + code = doCheckTombBlock(&block, pReader, numOfTables, &j, &pScanInfo, &ret); + + tTombBlockDestroy(&block); + if (code != TSDB_CODE_SUCCESS || ret == BLK_CHECK_QUIT) { + return code; + } + + i += 1; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t loadDataFileTombDataForAll(STsdbReader* pReader) { + if (pReader->status.pCurrentFileset == NULL || pReader->status.pCurrentFileset->farr[3] == NULL) { + return TSDB_CODE_SUCCESS; + } + + const TTombBlkArray* pBlkArray = NULL; + + int32_t code = tsdbDataFileReadTombBlk(pReader->pFileReader, &pBlkArray); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + return doLoadTombDataFromTombBlk(pBlkArray, pReader, pReader->pFileReader, true); +} + +int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo) { + const TTombBlkArray* pBlkArray = NULL; + int32_t code = tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + return doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false); +} + +void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) { + if (*ppMemDelData == NULL) { + *ppMemDelData = taosArrayInit(4, sizeof(SDelData)); + } + + SArray* pMemDelData = *ppMemDelData; + + SDelData* p = NULL; + if (pMemTbData != NULL) { + p = pMemTbData->pHead; + while (p) { + if (p->version <= ver) { + taosArrayPush(pMemDelData, p); + } + + p = p->pNext; + } + } + + if (piMemTbData != NULL) { + p = piMemTbData->pHead; + while (p) { + if (p->version <= ver) { + taosArrayPush(pMemDelData, p); + } + p = p->pNext; + } + } +} diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h new file mode 100644 index 0000000000000000000000000000000000000000..e7a1d6b038bb3c3f6fb629aed9507d2a87123160 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TSDBREADUTIL_H +#define TDENGINE_TSDBREADUTIL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tsdbDataFileRW.h" +#include "tsdbUtil2.h" + +#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) + +typedef enum { + READER_STATUS_SUSPEND = 0x1, + READER_STATUS_NORMAL = 0x2, +} EReaderStatus; + +typedef enum { + EXTERNAL_ROWS_PREV = 0x1, + EXTERNAL_ROWS_MAIN = 0x2, + EXTERNAL_ROWS_NEXT = 0x3, +} EContentData; + +typedef struct STsdbReaderInfo { + uint64_t suid; + STSchema* pSchema; + EReadMode readMode; + uint64_t rowsNum; + STimeWindow window; + SVersionRange verRange; + int16_t order; +} STsdbReaderInfo; + +typedef struct SBlockInfoBuf { + int32_t currentIndex; + SArray* pData; + int32_t numPerBucket; + int32_t numOfTables; +} SBlockInfoBuf; + +typedef struct { + STbDataIter* iter; + int32_t index; + bool hasVal; +} SIterInfo; + +typedef struct STableBlockScanInfo { + uint64_t uid; + TSKEY lastKey; + TSKEY lastKeyInStt; // last accessed key in stt + SArray* pBlockList; // block data index list, SArray + SArray* pMemDelData; // SArray + SArray* pfileDelData; // SArray from each file set + SIterInfo iter; // mem buffer skip list iterator + SIterInfo iiter; // imem buffer skip list iterator + SArray* delSkyline; // delete info for this table + int32_t fileDelIndex; // file block delete index + int32_t lastBlockDelIndex; // delete index for last block + bool iterInit; // whether to initialize the in-memory skip list iterator or not +} STableBlockScanInfo; + +typedef struct SResultBlockInfo { + SSDataBlock* pResBlock; + bool freeBlock; + int64_t capacity; +} SResultBlockInfo; + +typedef struct SCostSummary { + int64_t numOfBlocks; + double blockLoadTime; + double buildmemBlock; + int64_t headFileLoad; + double headFileLoadTime; + int64_t smaDataLoad; + double smaLoadTime; + int64_t lastBlockLoad; + double lastBlockLoadTime; + int64_t composedBlocks; + double buildComposedBlockTime; + double createScanInfoList; + double createSkylineIterTime; + double initLastBlockReader; +} SCostSummary; + +typedef struct STableUidList { + uint64_t* tableUidList; // access table uid list in uid ascending order list + int32_t currentIndex; // index in table uid list +} STableUidList; + +typedef struct { + int32_t numOfBlocks; + int32_t numOfLastFiles; +} SBlockNumber; + +typedef struct SBlockIndex { + int32_t ordinalIndex; + int64_t inFileOffset; + STimeWindow window; // todo replace it with overlap flag. +} SBlockIndex; + +typedef struct SBlockOrderWrapper { + int64_t uid; + int64_t offset; + STableBlockScanInfo* pInfo; +} SBlockOrderWrapper; + +typedef struct SBlockOrderSupporter { + SBlockOrderWrapper** pDataBlockInfo; + int32_t* indexPerTable; + int32_t* numOfBlocksPerTable; + int32_t numOfTables; +} SBlockOrderSupporter; + +typedef struct SBlockLoadSuppInfo { + TColumnDataAggArray colAggArray; + SColumnDataAgg tsColAgg; + int16_t* colId; + int16_t* slotId; + int32_t numOfCols; + char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. + bool smaValid; // the sma on all queried columns are activated +} SBlockLoadSuppInfo; + +typedef struct SLastBlockReader { + STimeWindow window; + SVersionRange verRange; + int32_t order; + uint64_t uid; + SMergeTree mergeTree; + SSttBlockLoadInfo* pInfo; + int64_t currentKey; +} SLastBlockReader; + +typedef struct SFilesetIter { + int32_t numOfFiles; // number of total files + int32_t index; // current accessed index in the list + TFileSetArray* pFilesetList; // data file set list + int32_t order; + SLastBlockReader* pLastBlockReader; // last file block reader +} SFilesetIter; + +typedef struct SFileDataBlockInfo { + // index position in STableBlockScanInfo in order to check whether neighbor block overlaps with it + uint64_t uid; + int32_t tbBlockIdx; + SBrinRecord record; +} SFileDataBlockInfo; + +typedef struct SDataBlockIter { + int32_t numOfBlocks; + int32_t index; + SArray* blockList; // SArray + int32_t order; + SDataBlk block; // current SDataBlk data + SSHashObj* pTableMap; +} SDataBlockIter; + +typedef struct SFileBlockDumpInfo { + int32_t totalRows; + int32_t rowIndex; + int64_t lastKey; + bool allDumped; +} SFileBlockDumpInfo; + +typedef struct SReaderStatus { + bool loadFromFile; // check file stage + bool composedDataBlock; // the returned data block is a composed block or not + SSHashObj* pTableMap; // SHash + STableBlockScanInfo** pTableIter; // table iterator used in building in-memory buffer data blocks. + STableUidList uidList; // check tables in uid order, to avoid the repeatly load of blocks in STT. + SFileBlockDumpInfo fBlockDumpInfo; + STFileSet* pCurrentFileset; // current opened file set + SBlockData fileBlockData; + SFilesetIter fileIter; + SDataBlockIter blockIter; + SArray* pLDataIterArray; + SRowMerger merger; + SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data +} SReaderStatus; + +struct STsdbReader { + STsdb* pTsdb; + STsdbReaderInfo info; + TdThreadMutex readerMutex; + EReaderStatus flag; + int32_t code; + uint64_t rowsNum; + SResultBlockInfo resBlockInfo; + SReaderStatus status; + char* idStr; // query info handle, for debug purpose + int32_t type; // query type: 1. retrieve all data blocks, 2. retrieve direct prev|next rows + SBlockLoadSuppInfo suppInfo; + STsdbReadSnap* pReadSnap; + SCostSummary cost; + SHashObj** pIgnoreTables; + SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema + SDataFileReader* pFileReader; // the file reader + SBlockInfoBuf blockInfoBuf; + EContentData step; + STsdbReader* innerReader[2]; +}; + +typedef struct SBrinRecordIter { + SArray* pBrinBlockList; + SBrinBlk* pCurrentBlk; + int32_t blockIndex; + int32_t recordIndex; + SDataFileReader* pReader; + SBrinBlock block; + SBrinRecord record; +} SBrinRecordIter; + +int32_t uidComparFunc(const void* p1, const void* p2); + +STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id); + +SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, + STableUidList* pUidList, int32_t numOfTables); +void clearBlockScanInfo(STableBlockScanInfo* p); +void destroyAllBlockScanInfo(SSHashObj* pTableMap); +void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step); +void cleanupInfoFoxNextFileset(SSHashObj* pTableMap); +int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables); +void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf); +void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index); + +// brin records iterator +void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList); +SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter); +void clearBrinBlockIter(SBrinRecordIter* pIter); + +// initialize block iterator API +int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList); +bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr); + +// load tomb data API (stt/mem only for one table each, tomb data from data files are load for all tables at one time) +void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver); +int32_t loadDataFileTombDataForAll(STsdbReader* pReader); +int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo); + +typedef struct { + SArray* pTombData; +} STableLoadInfo; + +struct SDataFileReader; + +typedef struct SCacheRowsReader { + STsdb* pTsdb; + STsdbReaderInfo info; + TdThreadMutex readerMutex; + SVnode* pVnode; + STSchema* pSchema; + STSchema* pCurrSchema; + uint64_t uid; + char** transferBuf; // todo remove it soon + int32_t numOfCols; + SArray* pCidList; + int32_t* pSlotIds; + int32_t type; + int32_t tableIndex; // currently returned result tables + STableKeyInfo* pTableList; // table id list + int32_t numOfTables; + uint64_t* uidList; + SSHashObj* pTableMap; + SArray* pLDataIterArray; + struct SDataFileReader* pFileReader; + STFileSet* pCurFileSet; + const TBrinBlkArray* pBlkArray; + STsdbReadSnap* pReadSnap; + char* idstr; + int64_t lastTs; +} SCacheRowsReader; + +int32_t tsdbCacheGetBatch(STsdb* pTsdb, tb_uid_t uid, SArray* pLastArray, SCacheRowsReader* pr, int8_t ltype); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TSDBREADUTIL_H diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 50fd9d7aa7035adbfe8d17fde078b301483fe015..89b7d019ae194f11c33151555f1a52c872f7fe5e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -16,7 +16,7 @@ #include "tsdb.h" // =============== PAGE-WISE FILE =============== -static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) { +int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) { int32_t code = 0; STsdbFD *pFD = NULL; @@ -68,7 +68,7 @@ _exit: return code; } -static void tsdbCloseFile(STsdbFD **ppFD) { +void tsdbCloseFile(STsdbFD **ppFD) { STsdbFD *pFD = *ppFD; if (pFD) { taosMemoryFree(pFD->pBuf); @@ -141,7 +141,7 @@ _exit: return code; } -static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size) { +int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size) { int32_t code = 0; int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); @@ -173,7 +173,7 @@ _exit: return code; } -static int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { +int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { int32_t code = 0; int64_t n = 0; int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); @@ -202,7 +202,7 @@ _exit: return code; } -static int32_t tsdbFsyncFile(STsdbFD *pFD) { +int32_t tsdbFsyncFile(STsdbFD *pFD) { int32_t code = 0; code = tsdbWriteFilePage(pFD); @@ -523,7 +523,7 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type) || ((pColData->flag & HAS_VALUE) == 0)) continue; + if ((!pColData->smaOn) || ((pColData->flag & HAS_VALUE) == 0)) continue; SColumnDataAgg sma = {.colId = pColData->cid}; tColDataCalcSMA[pColData->type](pColData, &sma.sum, &sma.max, &sma.min, &sma.numOfNull); @@ -749,7 +749,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { int64_t size; TdFilePtr pOutFD = NULL; TdFilePtr PInFD = NULL; - int32_t szPage = pTsdb->pVnode->config.szPage; + int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; char fNameFrom[TSDB_FILENAME_LEN]; char fNameTo[TSDB_FILENAME_LEN]; @@ -1489,6 +1489,10 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { } int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) { + return tsdbReadDelDatav1(pReader, pDelIdx, aDelData, INT64_MAX); +} + +int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) { int32_t code = 0; int64_t offset = pDelIdx->offset; int64_t size = pDelIdx->size; @@ -1510,11 +1514,15 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData SDelData delData; n += tGetDelData(pReader->aBuf[0] + n, &delData); + if (delData.version > maxVer) { + continue; + } if (taosArrayPush(aDelData, &delData) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } } + ASSERT(n == size); return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 7c7e1bd0f79254197e1c99dd5ca0544770656cea..a4d5715083a2b6d82f048b8950c2a74c8df11f6d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -14,101 +14,271 @@ */ #include "tsdb.h" +#include "tsdbFS2.h" -static bool tsdbShouldDoRetentionImpl(STsdb *pTsdb, int64_t now) { - for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; +typedef struct { + STsdb *tsdb; + int32_t szPage; + int64_t now; + int64_t cid; - if (expLevel == pSet->diskId.level) continue; + TFileSetArray *fsetArr; + TFileOpArray fopArr[1]; - if (expLevel < 0) { - return true; - } else { - if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { - return false; - } + struct { + int32_t fsetArrIdx; + STFileSet *fset; + } ctx[1]; +} SRTNer; - if (did.level == pSet->diskId.level) continue; +static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) { + STFileOp op = { + .optype = TSDB_FOP_REMOVE, + .fid = fobj->f->fid, + .of = fobj->f[0], + }; - return true; - } + return TARRAY2_APPEND(rtner->fopArr, op); +} + +static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile *to) { + int32_t code = 0; + int32_t lino = 0; + + char fname[TSDB_FILENAME_LEN]; + TdFilePtr fdFrom = NULL; + TdFilePtr fdTo = NULL; + + tsdbTFileName(rtner->tsdb, to, fname); + + fdFrom = taosOpenFile(from->fname, TD_FILE_READ); + if (fdFrom == NULL) code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + + fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (fdTo == NULL) code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + + int64_t n = taosFSendFile(fdTo, fdFrom, 0, tsdbLogicToFileSize(from->f->size, rtner->szPage)); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); } + taosCloseFile(&fdFrom); + taosCloseFile(&fdTo); - return false; +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + taosCloseFile(&fdFrom); + taosCloseFile(&fdTo); + } + return code; } -bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { - bool should; - taosThreadRwlockRdlock(&pTsdb->rwLock); - should = tsdbShouldDoRetentionImpl(pTsdb, now); - taosThreadRwlockUnlock(&pTsdb->rwLock); - return should; + +static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) { + int32_t code = 0; + int32_t lino = 0; + STFileOp op = {0}; + + // remove old + op = (STFileOp){ + .optype = TSDB_FOP_REMOVE, + .fid = fobj->f->fid, + .of = fobj->f[0], + }; + + code = TARRAY2_APPEND(rtner->fopArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + // create new + op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = fobj->f->fid, + .nf = + { + .type = fobj->f->type, + .did = did[0], + .fid = fobj->f->fid, + .cid = fobj->f->cid, + .size = fobj->f->size, + .stt[0] = + { + .level = fobj->f->stt[0].level, + }, + }, + }; + + code = TARRAY2_APPEND(rtner->fopArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + // do copy the file + code = tsdbDoCopyFile(rtner, fobj, &op.nf); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + } + return code; } -int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { +typedef struct { + STsdb *tsdb; + int32_t sync; + int64_t now; +} SRtnArg; + +static int32_t tsdbDoRetentionBegin(SRtnArg *arg, SRTNer *rtner) { int32_t code = 0; int32_t lino = 0; - STsdbFS fs = {0}; - code = tsdbFSCopy(pTsdb, &fs); + STsdb *tsdb = arg->tsdb; + + rtner->tsdb = tsdb; + rtner->szPage = tsdb->pVnode->config.tsdbPageSize; + rtner->now = arg->now; + rtner->cid = tsdbFSAllocEid(tsdb->pFS); + + code = tsdbFSCreateCopySnapshot(tsdb->pFS, &rtner->fsetArr); TSDB_CHECK_CODE(code, lino, _exit); - for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; - - if (expLevel < 0) { - taosMemoryFree(pSet->pHeadF); - taosMemoryFree(pSet->pDataF); - taosMemoryFree(pSet->pSmaF); - for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { - taosMemoryFree(pSet->aSttF[iStt]); +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + } else { + tsdbInfo("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__); + } + return code; +} + +static int32_t tsdbDoRetentionEnd(SRTNer *rtner) { + int32_t code = 0; + int32_t lino = 0; + + if (TARRAY2_SIZE(rtner->fopArr) == 0) goto _exit; + + code = tsdbFSEditBegin(rtner->tsdb->pFS, rtner->fopArr, TSDB_FEDIT_MERGE); + TSDB_CHECK_CODE(code, lino, _exit); + + taosThreadRwlockWrlock(&rtner->tsdb->rwLock); + + code = tsdbFSEditCommit(rtner->tsdb->pFS); + if (code) { + taosThreadRwlockUnlock(&rtner->tsdb->rwLock); + TSDB_CHECK_CODE(code, lino, _exit); + } + + taosThreadRwlockUnlock(&rtner->tsdb->rwLock); + + TARRAY2_DESTROY(rtner->fopArr, NULL); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + } else { + tsdbInfo("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__); + } + tsdbFSDestroyCopySnapshot(&rtner->fsetArr); + return code; +} + +static int32_t tsdbDoRetention2(void *arg) { + int32_t code = 0; + int32_t lino = 0; + SRTNer rtner[1] = {0}; + + code = tsdbDoRetentionBegin(arg, rtner); + TSDB_CHECK_CODE(code, lino, _exit); + + for (rtner->ctx->fsetArrIdx = 0; rtner->ctx->fsetArrIdx < TARRAY2_SIZE(rtner->fsetArr); rtner->ctx->fsetArrIdx++) { + rtner->ctx->fset = TARRAY2_GET(rtner->fsetArr, rtner->ctx->fsetArrIdx); + + STFileObj *fobj; + int32_t expLevel = tsdbFidLevel(rtner->ctx->fset->fid, &rtner->tsdb->keepCfg, rtner->now); + + if (expLevel < 0) { // remove the file set + for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = rtner->ctx->fset->farr[ftype], 1); ++ftype) { + if (fobj == NULL) continue; + + code = tsdbDoRemoveFileObject(rtner, fobj); + TSDB_CHECK_CODE(code, lino, _exit); + } + + SSttLvl *lvl; + TARRAY2_FOREACH(rtner->ctx->fset->lvlArr, lvl) { + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + code = tsdbDoRemoveFileObject(rtner, fobj); + TSDB_CHECK_CODE(code, lino, _exit); + } } - taosArrayRemove(fs.aDFileSet, iSet); - iSet--; + } else if (expLevel == 0) { + continue; } else { - if (expLevel == 0) continue; - if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { + SDiskID did; + + if (tfsAllocDisk(rtner->tsdb->pVnode->pTfs, expLevel, &did) < 0) { code = terrno; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } + tfsMkdirRecurAt(rtner->tsdb->pVnode->pTfs, rtner->tsdb->path, did); - if (did.level == pSet->diskId.level) continue; + // data + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX && (fobj = rtner->ctx->fset->farr[ftype], 1); ++ftype) { + if (fobj == NULL) continue; - // copy file to new disk (todo) - SDFileSet fSet = *pSet; - fSet.diskId = did; + if (fobj->f->did.level == did.level) continue; + code = tsdbDoMigrateFileObj(rtner, fobj, &did); + TSDB_CHECK_CODE(code, lino, _exit); + } - code = tsdbDFileSetCopy(pTsdb, pSet, &fSet); - TSDB_CHECK_CODE(code, lino, _exit); + // stt + SSttLvl *lvl; + TARRAY2_FOREACH(rtner->ctx->fset->lvlArr, lvl) { + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + if (fobj->f->did.level == did.level) continue; - code = tsdbFSUpsertFSet(&fs, &fSet); - TSDB_CHECK_CODE(code, lino, _exit); + code = tsdbDoMigrateFileObj(rtner, fobj, &did); + TSDB_CHECK_CODE(code, lino, _exit); + } + } } } - // do change fs - code = tsdbFSPrepareCommit(pTsdb, &fs); + code = tsdbDoRetentionEnd(rtner); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); } - tsdbFSDestroy(&fs); return code; } -static int32_t tsdbCommitRetentionImpl(STsdb *pTsdb) { return tsdbFSCommit(pTsdb); } +static void tsdbFreeRtnArg(void *arg) { + SRtnArg *rArg = (SRtnArg *)arg; + if (rArg->sync) { + tsem_post(&rArg->tsdb->pVnode->canCommit); + } + taosMemoryFree(arg); +} + +int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) { + SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); + if (arg == NULL) return TSDB_CODE_OUT_OF_MEMORY; + arg->tsdb = tsdb; + arg->sync = sync; + arg->now = now; -int32_t tsdbCommitRetention(STsdb *pTsdb) { - taosThreadRwlockWrlock(&pTsdb->rwLock); - tsdbCommitRetentionImpl(pTsdb); - taosThreadRwlockUnlock(&pTsdb->rwLock); - tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); - return 0; + if (sync) { + tsem_wait(&tsdb->pVnode->canCommit); + } + + int64_t taskid; + int32_t code = + tsdbFSScheduleBgTask(tsdb->pFS, TSDB_BG_TASK_RETENTION, tsdbDoRetention2, tsdbFreeRtnArg, arg, &taskid); + if (code) { + tsdbFreeRtnArg(arg); + } + return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index dfea125cc1026eefca2abdb7ed5bcbc9bd02fd1c..bdcf4a87c194c38487576ea796d5d844d0159ab4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -14,553 +14,519 @@ */ #include "tsdb.h" +#include "tsdbDataFileRW.h" +#include "tsdbFS2.h" +#include "tsdbFSetRW.h" +#include "tsdbIter.h" +#include "tsdbSttFileRW.h" extern int32_t tsdbUpdateTableSchema(SMeta* pMeta, int64_t suid, int64_t uid, SSkmInfo* pSkmInfo); -extern int32_t tsdbWriteDataBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SMapData* mDataBlk, int8_t cmprAlg); -extern int32_t tsdbWriteSttBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SArray* aSttBlk, int8_t cmprAlg); // STsdbSnapReader ======================================== struct STsdbSnapReader { - STsdb* pTsdb; - int64_t sver; - int64_t ever; - int8_t type; + STsdb* tsdb; + int64_t sver; + int64_t ever; + int8_t type; + uint8_t* aBuf[5]; + SSkmInfo skmTb[1]; + + TFileSetArray* fsetArr; - STsdbFS fs; - TABLEID tbid; - SSkmInfo skmTable; - - // timeseries data - int8_t dataDone; - int32_t fid; - - SDataFReader* pDataFReader; - STsdbDataIter2* iterList; - STsdbDataIter2* pIter; - SRBTree rbt; - SBlockData bData; - - // tombstone data - int8_t delDone; - SDelFReader* pDelFReader; - STsdbDataIter2* pTIter; - SArray* aDelData; + // context + struct { + int32_t fsetArrIdx; + STFileSet* fset; + bool isDataDone; + bool isTombDone; + } ctx[1]; + + // reader + SDataFileReader* dataReader; + TSttFileReaderArray sttReaderArr[1]; + + // iter + TTsdbIterArray dataIterArr[1]; + SIterMerger* dataIterMerger; + TTsdbIterArray tombIterArr[1]; + SIterMerger* tombIterMerger; + + // data + SBlockData blockData[1]; + STombBlock tombBlock[1]; }; -static int32_t tsdbSnapReadFileDataStart(STsdbSnapReader* pReader) { +static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) { int32_t code = 0; int32_t lino = 0; - SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &(SDFileSet){.fid = pReader->fid}, tDFileSetCmprFn, TD_GT); - if (pSet == NULL) { - pReader->fid = INT32_MAX; - goto _exit; + ASSERT(reader->dataReader == NULL); + ASSERT(TARRAY2_SIZE(reader->sttReaderArr) == 0); + + // data + SDataFileReaderConfig config = { + .tsdb = reader->tsdb, + .szPage = reader->tsdb->pVnode->config.tsdbPageSize, + .bufArr = reader->aBuf, + }; + bool hasDataFile = false; + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ftype++) { + if (reader->ctx->fset->farr[ftype] != NULL) { + hasDataFile = true; + config.files[ftype].exist = true; + config.files[ftype].file = reader->ctx->fset->farr[ftype]->f[0]; + } } - pReader->fid = pSet->fid; + if (hasDataFile) { + code = tsdbDataFileReaderOpen(NULL, &config, &reader->dataReader); + TSDB_CHECK_CODE(code, lino, _exit); + } - tRBTreeCreate(&pReader->rbt, tsdbDataIterCmprFn); + // stt + SSttLvl* lvl; + TARRAY2_FOREACH(reader->ctx->fset->lvlArr, lvl) { + STFileObj* fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + SSttFileReader* sttReader; + SSttFileReaderConfig config = { + .tsdb = reader->tsdb, + .szPage = reader->tsdb->pVnode->config.tsdbPageSize, + .file = fobj->f[0], + .bufArr = reader->aBuf, + }; + + code = tsdbSttFileReaderOpen(fobj->fname, &config, &sttReader); + TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet); - TSDB_CHECK_CODE(code, lino, _exit); + code = TARRAY2_APPEND(reader->sttReaderArr, sttReader); + TSDB_CHECK_CODE(code, lino, _exit); + } + } - code = tsdbOpenDataFileDataIter(pReader->pDataFReader, &pReader->pIter); - TSDB_CHECK_CODE(code, lino, _exit); +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapReadFileSetCloseReader(STsdbSnapReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + TARRAY2_CLEAR(reader->sttReaderArr, tsdbSttFileReaderClose); + tsdbDataFileReaderClose(&reader->dataReader); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapReadFileSetOpenIter(STsdbSnapReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(reader->dataIterMerger == NULL); + ASSERT(reader->tombIterMerger == NULL); + ASSERT(TARRAY2_SIZE(reader->dataIterArr) == 0); + ASSERT(TARRAY2_SIZE(reader->tombIterArr) == 0); + + STsdbIter* iter; + STsdbIterConfig config = { + .filterByVersion = true, + .verRange[0] = reader->sver, + .verRange[1] = reader->ever, + }; + + // data file + if (reader->dataReader) { + // data + config.type = TSDB_ITER_TYPE_DATA; + config.dataReader = reader->dataReader; + + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); - if (pReader->pIter) { - // iter to next with filter info (sver, ever) - code = tsdbDataIterNext2( - pReader->pIter, - &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag - .sver = pReader->sver, - .ever = pReader->ever}); + code = TARRAY2_APPEND(reader->dataIterArr, iter); TSDB_CHECK_CODE(code, lino, _exit); - if (pReader->pIter->rowInfo.suid || pReader->pIter->rowInfo.uid) { - // add to rbtree - tRBTreePut(&pReader->rbt, &pReader->pIter->rbtn); + // tomb + config.type = TSDB_ITER_TYPE_DATA_TOMB; + config.dataReader = reader->dataReader; - // add to iterList - pReader->pIter->next = pReader->iterList; - pReader->iterList = pReader->pIter; - } else { - tsdbCloseDataIter2(pReader->pIter); - } + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(reader->tombIterArr, iter); + TSDB_CHECK_CODE(code, lino, _exit); } - for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) { - code = tsdbOpenSttFileDataIter(pReader->pDataFReader, iStt, &pReader->pIter); + // stt file + SSttFileReader* sttReader; + TARRAY2_FOREACH(reader->sttReaderArr, sttReader) { + // data + config.type = TSDB_ITER_TYPE_STT; + config.sttReader = sttReader; + + code = tsdbIterOpen(&config, &iter); TSDB_CHECK_CODE(code, lino, _exit); - if (pReader->pIter) { - // iter to valid row - code = tsdbDataIterNext2( - pReader->pIter, - &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag - .sver = pReader->sver, - .ever = pReader->ever}); - TSDB_CHECK_CODE(code, lino, _exit); + code = TARRAY2_APPEND(reader->dataIterArr, iter); + TSDB_CHECK_CODE(code, lino, _exit); - if (pReader->pIter->rowInfo.suid || pReader->pIter->rowInfo.uid) { - // add to rbtree - tRBTreePut(&pReader->rbt, &pReader->pIter->rbtn); + // tomb + config.type = TSDB_ITER_TYPE_STT_TOMB; + config.sttReader = sttReader; - // add to iterList - pReader->pIter->next = pReader->iterList; - pReader->iterList = pReader->pIter; - } else { - tsdbCloseDataIter2(pReader->pIter); - } - } + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(reader->tombIterArr, iter); + TSDB_CHECK_CODE(code, lino, _exit); } - pReader->pIter = NULL; + // merger + code = tsdbIterMergerOpen(reader->dataIterArr, &reader->dataIterMerger, false); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbIterMergerOpen(reader->tombIterArr, &reader->tombIterMerger, true); + TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pReader->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbInfo("vgId:%d %s done, fid:%d", TD_VID(pReader->pTsdb->pVnode), __func__, pReader->fid); + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } return code; } -static void tsdbSnapReadFileDataEnd(STsdbSnapReader* pReader) { - while (pReader->iterList) { - STsdbDataIter2* pIter = pReader->iterList; - pReader->iterList = pIter->next; - tsdbCloseDataIter2(pIter); - } - - tsdbDataFReaderClose(&pReader->pDataFReader); +static int32_t tsdbSnapReadFileSetCloseIter(STsdbSnapReader* reader) { + tsdbIterMergerClose(&reader->dataIterMerger); + tsdbIterMergerClose(&reader->tombIterMerger); + TARRAY2_CLEAR(reader->dataIterArr, tsdbIterClose); + TARRAY2_CLEAR(reader->tombIterArr, tsdbIterClose); + return 0; } -static int32_t tsdbSnapReadNextRow(STsdbSnapReader* pReader, SRowInfo** ppRowInfo) { +static int32_t tsdbSnapReadFileSetBegin(STsdbSnapReader* reader) { int32_t code = 0; int32_t lino = 0; - if (pReader->pIter) { - code = tsdbDataIterNext2(pReader->pIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | - TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag - .sver = pReader->sver, - .ever = pReader->ever}); - TSDB_CHECK_CODE(code, lino, _exit); + ASSERT(reader->ctx->fset == NULL); - if (pReader->pIter->rowInfo.suid == 0 && pReader->pIter->rowInfo.uid == 0) { - pReader->pIter = NULL; - } else { - SRBTreeNode* pNode = tRBTreeMin(&pReader->rbt); - if (pNode) { - int32_t c = tsdbDataIterCmprFn(&pReader->pIter->rbtn, pNode); - if (c > 0) { - tRBTreePut(&pReader->rbt, &pReader->pIter->rbtn); - pReader->pIter = NULL; - } else if (c == 0) { - ASSERT(0); - } - } - } - } + if (reader->ctx->fsetArrIdx < TARRAY2_SIZE(reader->fsetArr)) { + reader->ctx->fset = TARRAY2_GET(reader->fsetArr, reader->ctx->fsetArrIdx++); + reader->ctx->isDataDone = false; + reader->ctx->isTombDone = false; - if (pReader->pIter == NULL) { - SRBTreeNode* pNode = tRBTreeMin(&pReader->rbt); - if (pNode) { - tRBTreeDrop(&pReader->rbt, pNode); - pReader->pIter = TSDB_RBTN_TO_DATA_ITER(pNode); - } - } + code = tsdbSnapReadFileSetOpenReader(reader); + TSDB_CHECK_CODE(code, lino, _exit); - if (ppRowInfo) { - if (pReader->pIter) { - *ppRowInfo = &pReader->pIter->rowInfo; - } else { - *ppRowInfo = NULL; - } + code = tsdbSnapReadFileSetOpenIter(reader); + TSDB_CHECK_CODE(code, lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pReader->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } return code; } -static int32_t tsdbSnapReadGetRow(STsdbSnapReader* pReader, SRowInfo** ppRowInfo) { - if (pReader->pIter) { - *ppRowInfo = &pReader->pIter->rowInfo; - return 0; - } - - return tsdbSnapReadNextRow(pReader, ppRowInfo); +static int32_t tsdbSnapReadFileSetEnd(STsdbSnapReader* reader) { + tsdbSnapReadFileSetCloseIter(reader); + tsdbSnapReadFileSetCloseReader(reader); + reader->ctx->fset = NULL; + return 0; } -static int32_t tsdbSnapCmprData(STsdbSnapReader* pReader, uint8_t** ppData) { +static int32_t tsdbSnapCmprData(STsdbSnapReader* reader, uint8_t** data) { int32_t code = 0; - - ASSERT(pReader->bData.nRow); + int32_t lino = 0; int32_t aBufN[5] = {0}; - code = tCmprBlockData(&pReader->bData, NO_COMPRESSION, NULL, NULL, pReader->aBuf, aBufN); - if (code) goto _exit; + code = tCmprBlockData(reader->blockData, NO_COMPRESSION, NULL, NULL, reader->aBuf, aBufN); + TSDB_CHECK_CODE(code, lino, _exit); int32_t size = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; - *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); - if (*ppData == NULL) { + *data = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); + if (*data == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } - SSnapDataHdr* pHdr = (SSnapDataHdr*)*ppData; - pHdr->type = pReader->type; + SSnapDataHdr* pHdr = (SSnapDataHdr*)*data; + pHdr->type = reader->type; pHdr->size = size; - memcpy(pHdr->data, pReader->aBuf[3], aBufN[3]); - memcpy(pHdr->data + aBufN[3], pReader->aBuf[2], aBufN[2]); + memcpy(pHdr->data, reader->aBuf[3], aBufN[3]); + memcpy(pHdr->data + aBufN[3], reader->aBuf[2], aBufN[2]); if (aBufN[1]) { - memcpy(pHdr->data + aBufN[3] + aBufN[2], pReader->aBuf[1], aBufN[1]); + memcpy(pHdr->data + aBufN[3] + aBufN[2], reader->aBuf[1], aBufN[1]); } if (aBufN[0]) { - memcpy(pHdr->data + aBufN[3] + aBufN[2] + aBufN[1], pReader->aBuf[0], aBufN[0]); + memcpy(pHdr->data + aBufN[3] + aBufN[2] + aBufN[1], reader->aBuf[0], aBufN[0]); } _exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), lino, code); + } return code; } -static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* pReader, uint8_t** ppData) { - int32_t code = 0; - int32_t lino = 0; - - STsdb* pTsdb = pReader->pTsdb; +static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** data) { + int32_t code = 0; + int32_t lino = 0; + SMetaInfo info; + + tBlockDataReset(reader->blockData); + + TABLEID tbid[1] = {0}; + for (SRowInfo* row; (row = tsdbIterMergerGetData(reader->dataIterMerger));) { + // skip dropped table + if (row->uid != tbid->uid) { + tbid->suid = row->suid; + tbid->uid = row->uid; + if (metaGetInfo(reader->tsdb->pVnode->pMeta, tbid->uid, &info, NULL) != 0) { + code = tsdbIterMergerSkipTableData(reader->dataIterMerger, tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; + } + } - tBlockDataReset(&pReader->bData); + if (reader->blockData->suid == 0 && reader->blockData->uid == 0) { + code = tsdbUpdateSkmTb(reader->tsdb, (TABLEID*)row, reader->skmTb); + TSDB_CHECK_CODE(code, lino, _exit); - for (;;) { - // start a new file read if need - if (pReader->pDataFReader == NULL) { - code = tsdbSnapReadFileDataStart(pReader); + TABLEID tbid1 = { + .suid = row->suid, + .uid = row->suid ? 0 : row->uid, + }; + code = tBlockDataInit(reader->blockData, &tbid1, reader->skmTb->pTSchema, NULL, 0); TSDB_CHECK_CODE(code, lino, _exit); } - if (pReader->pDataFReader == NULL) break; - - SRowInfo* pRowInfo; - code = tsdbSnapReadGetRow(pReader, &pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pRowInfo == NULL) { - tsdbSnapReadFileDataEnd(pReader); - continue; + if (!TABLE_SAME_SCHEMA(reader->blockData->suid, reader->blockData->uid, row->suid, row->uid)) { + break; } - code = tsdbUpdateTableSchema(pTsdb->pVnode->pMeta, pRowInfo->suid, pRowInfo->uid, &pReader->skmTable); + code = tBlockDataAppendRow(reader->blockData, &row->row, NULL, row->uid); TSDB_CHECK_CODE(code, lino, _exit); - code = tBlockDataInit(&pReader->bData, (TABLEID*)pRowInfo, pReader->skmTable.pTSchema, NULL, 0); + code = tsdbIterMergerNext(reader->dataIterMerger); TSDB_CHECK_CODE(code, lino, _exit); - do { - if (!TABLE_SAME_SCHEMA(pReader->bData.suid, pReader->bData.uid, pRowInfo->suid, pRowInfo->uid)) break; - - if (pReader->bData.uid && pReader->bData.uid != pRowInfo->uid) { - code = tRealloc((uint8_t**)&pReader->bData.aUid, sizeof(int64_t) * (pReader->bData.nRow + 1)); - TSDB_CHECK_CODE(code, lino, _exit); - - for (int32_t iRow = 0; iRow < pReader->bData.nRow; ++iRow) { - pReader->bData.aUid[iRow] = pReader->bData.uid; - } - pReader->bData.uid = 0; - } - - code = tBlockDataAppendRow(&pReader->bData, &pRowInfo->row, NULL, pRowInfo->uid); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbSnapReadNextRow(pReader, &pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pReader->bData.nRow >= 81920) break; - } while (pRowInfo); - - ASSERT(pReader->bData.nRow > 0); - - break; + if (reader->blockData->nRow >= 81920) { + break; + } } - if (pReader->bData.nRow > 0) { - ASSERT(pReader->bData.suid || pReader->bData.uid); - - code = tsdbSnapCmprData(pReader, ppData); + if (reader->blockData->nRow > 0) { + ASSERT(reader->blockData->suid || reader->blockData->uid); + code = tsdbSnapCmprData(reader, data); TSDB_CHECK_CODE(code, lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } return code; } -static int32_t tsdbSnapCmprTombData(STsdbSnapReader* pReader, uint8_t** ppData) { +static int32_t tsdbSnapCmprTombData(STsdbSnapReader* reader, uint8_t** data) { int32_t code = 0; int32_t lino = 0; - int64_t size = sizeof(TABLEID); - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); ++iDelData) { - size += tPutDelData(NULL, taosArrayGet(pReader->aDelData, iDelData)); + int64_t size = 0; + for (int32_t i = 0; i < ARRAY_SIZE(reader->tombBlock->dataArr); i++) { + size += TARRAY2_DATA_LEN(reader->tombBlock->dataArr + i); } - uint8_t* pData = (uint8_t*)taosMemoryMalloc(sizeof(SSnapDataHdr) + size); - if (pData == NULL) { + data[0] = taosMemoryMalloc(size + sizeof(SSnapDataHdr)); + if (data[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } - SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; - pHdr->type = SNAP_DATA_DEL; - pHdr->size = size; + SSnapDataHdr* hdr = (SSnapDataHdr*)(data[0]); + hdr->type = SNAP_DATA_DEL; + hdr->size = size; - TABLEID* pId = (TABLEID*)(pData + sizeof(SSnapDataHdr)); - *pId = pReader->tbid; - - size = sizeof(SSnapDataHdr) + sizeof(TABLEID); - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); ++iDelData) { - size += tPutDelData(pData + size, taosArrayGet(pReader->aDelData, iDelData)); + uint8_t* tdata = hdr->data; + for (int32_t i = 0; i < ARRAY_SIZE(reader->tombBlock->dataArr); i++) { + memcpy(tdata, TARRAY2_DATA(reader->tombBlock->dataArr + i), TARRAY2_DATA_LEN(reader->tombBlock->dataArr + i)); + tdata += TARRAY2_DATA_LEN(reader->tombBlock->dataArr + i); } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pReader->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } - *ppData = pData; return code; } -static void tsdbSnapReadGetTombData(STsdbSnapReader* pReader, SDelInfo** ppDelInfo) { - if (pReader->pTIter == NULL || (pReader->pTIter->delInfo.suid == 0 && pReader->pTIter->delInfo.uid == 0)) { - *ppDelInfo = NULL; - } else { - *ppDelInfo = &pReader->pTIter->delInfo; - } -} - -static int32_t tsdbSnapReadNextTombData(STsdbSnapReader* pReader, SDelInfo** ppDelInfo) { - int32_t code = 0; - int32_t lino = 0; +static int32_t tsdbSnapReadTombData(STsdbSnapReader* reader, uint8_t** data) { + int32_t code = 0; + int32_t lino = 0; + SMetaInfo info; - code = tsdbDataIterNext2( - pReader->pTIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, - .sver = pReader->sver, - .ever = pReader->ever}); - TSDB_CHECK_CODE(code, lino, _exit); + tTombBlockClear(reader->tombBlock); - if (ppDelInfo) { - tsdbSnapReadGetTombData(pReader, ppDelInfo); - } - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pReader->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tsdbSnapReadTombData(STsdbSnapReader* pReader, uint8_t** ppData) { - int32_t code = 0; - int32_t lino = 0; - - STsdb* pTsdb = pReader->pTsdb; - - // open tombstone data iter if need - if (pReader->pDelFReader == NULL) { - if (pReader->fs.pDelFile == NULL) goto _exit; - - // open - code = tsdbDelFReaderOpen(&pReader->pDelFReader, pReader->fs.pDelFile, pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbOpenTombFileDataIter(pReader->pDelFReader, &pReader->pTIter); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pReader->pTIter) { - code = tsdbSnapReadNextTombData(pReader, NULL); - TSDB_CHECK_CODE(code, lino, _exit); + TABLEID tbid[1] = {0}; + for (STombRecord* record; (record = tsdbIterMergerGetTombRecord(reader->tombIterMerger)) != NULL;) { + if (record->uid != tbid->uid) { + tbid->suid = record->suid; + tbid->uid = record->uid; + if (metaGetInfo(reader->tsdb->pVnode->pMeta, tbid->uid, &info, NULL) != 0) { + code = tsdbIterMergerSkipTableData(reader->tombIterMerger, tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; + } } - } - - // loop to get tombstone data - SDelInfo* pDelInfo; - tsdbSnapReadGetTombData(pReader, &pDelInfo); - if (pDelInfo == NULL) goto _exit; - - pReader->tbid = *(TABLEID*)pDelInfo; - - if (pReader->aDelData) { - taosArrayClear(pReader->aDelData); - } else if ((pReader->aDelData = taosArrayInit(16, sizeof(SDelData))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = tTombBlockPut(reader->tombBlock, record); TSDB_CHECK_CODE(code, lino, _exit); - } - while (pDelInfo && pDelInfo->suid == pReader->tbid.suid && pDelInfo->uid == pReader->tbid.uid) { - if (taosArrayPush(pReader->aDelData, &pDelInfo->delData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + if (TOMB_BLOCK_SIZE(reader->tombBlock) >= 81920) { + break; } - - code = tsdbSnapReadNextTombData(pReader, &pDelInfo); - TSDB_CHECK_CODE(code, lino, _exit); } - // encode tombstone data - if (taosArrayGetSize(pReader->aDelData) > 0) { - code = tsdbSnapCmprTombData(pReader, ppData); + if (TOMB_BLOCK_SIZE(reader->tombBlock) > 0) { + code = tsdbSnapCmprTombData(reader, data); TSDB_CHECK_CODE(code, lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbDebug("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } return code; } -int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader) { +int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** reader) { int32_t code = 0; int32_t lino = 0; - // alloc - STsdbSnapReader* pReader = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); - if (pReader == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - pReader->pTsdb = pTsdb; - pReader->sver = sver; - pReader->ever = ever; - pReader->type = type; - - taosThreadRwlockRdlock(&pTsdb->rwLock); - code = tsdbFSRef(pTsdb, &pReader->fs); - if (code) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - TSDB_CHECK_CODE(code, lino, _exit); - } - taosThreadRwlockUnlock(&pTsdb->rwLock); + reader[0] = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*reader[0])); + if (reader[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; - // init - pReader->fid = INT32_MIN; + reader[0]->tsdb = tsdb; + reader[0]->sver = sver; + reader[0]->ever = ever; + reader[0]->type = type; - code = tBlockDataCreate(&pReader->bData); + code = tsdbFSCreateRefSnapshot(tsdb->pFS, &reader[0]->fsetArr); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(pTsdb->pVnode), + tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code), sver, ever, type); - if (pReader) { - tBlockDataDestroy(&pReader->bData); - tsdbFSUnref(pTsdb, &pReader->fs); - taosMemoryFree(pReader); - pReader = NULL; - } + tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); + taosMemoryFree(reader[0]); + reader[0] = NULL; } else { - tsdbInfo("vgId:%d %s done, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(pTsdb->pVnode), __func__, sver, ever, + tsdbInfo("vgId:%d %s done, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, sver, ever, type); } - *ppReader = pReader; return code; } -int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { +int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) { + if (reader[0] == NULL) return 0; + int32_t code = 0; int32_t lino = 0; - STsdbSnapReader* pReader = *ppReader; - STsdb* pTsdb = pReader->pTsdb; + STsdb* tsdb = reader[0]->tsdb; - // tombstone - if (pReader->pTIter) { - tsdbCloseDataIter2(pReader->pTIter); - pReader->pTIter = NULL; - } - if (pReader->pDelFReader) { - tsdbDelFReaderClose(&pReader->pDelFReader); - } - taosArrayDestroy(pReader->aDelData); + tTombBlockDestroy(reader[0]->tombBlock); + tBlockDataDestroy(reader[0]->blockData); - // timeseries - while (pReader->iterList) { - STsdbDataIter2* pIter = pReader->iterList; - pReader->iterList = pIter->next; - tsdbCloseDataIter2(pIter); - } - if (pReader->pDataFReader) { - tsdbDataFReaderClose(&pReader->pDataFReader); - } - tBlockDataDestroy(&pReader->bData); + tsdbIterMergerClose(&reader[0]->dataIterMerger); + tsdbIterMergerClose(&reader[0]->tombIterMerger); + TARRAY2_DESTROY(reader[0]->dataIterArr, tsdbIterClose); + TARRAY2_DESTROY(reader[0]->tombIterArr, tsdbIterClose); + TARRAY2_DESTROY(reader[0]->sttReaderArr, tsdbSttFileReaderClose); + tsdbDataFileReaderClose(&reader[0]->dataReader); + + tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); + tDestroyTSchema(reader[0]->skmTb->pTSchema); - // other - tDestroyTSchema(pReader->skmTable.pTSchema); - tsdbFSUnref(pReader->pTsdb, &pReader->fs); - for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(pReader->aBuf[0]); iBuf++) { - tFree(pReader->aBuf[iBuf]); + for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->aBuf); ++i) { + tFree(reader[0]->aBuf[i]); } - taosMemoryFree(pReader); + + taosMemoryFree(reader[0]); + reader[0] = NULL; _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); } else { - tsdbDebug("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); } - *ppReader = NULL; return code; } -int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { +int32_t tsdbSnapRead(STsdbSnapReader* reader, uint8_t** data) { int32_t code = 0; int32_t lino = 0; - *ppData = NULL; + data[0] = NULL; - // read data file - if (!pReader->dataDone) { - code = tsdbSnapReadTimeSeriesData(pReader, ppData); - TSDB_CHECK_CODE(code, lino, _exit); - if (*ppData) { - goto _exit; - } else { - pReader->dataDone = 1; + for (;;) { + if (reader->ctx->fset == NULL) { + code = tsdbSnapReadFileSetBegin(reader); + TSDB_CHECK_CODE(code, lino, _exit); + + if (reader->ctx->fset == NULL) { + break; + } } - } - // read del file - if (!pReader->delDone) { - code = tsdbSnapReadTombData(pReader, ppData); - TSDB_CHECK_CODE(code, lino, _exit); - if (*ppData) { - goto _exit; - } else { - pReader->delDone = 1; + if (!reader->ctx->isDataDone) { + code = tsdbSnapReadTimeSeriesData(reader, data); + TSDB_CHECK_CODE(code, lino, _exit); + if (data[0]) { + goto _exit; + } else { + reader->ctx->isDataDone = true; + } + } + + if (!reader->ctx->isTombDone) { + code = tsdbSnapReadTombData(reader, data); + TSDB_CHECK_CODE(code, lino, _exit); + if (data[0]) { + goto _exit; + } else { + reader->ctx->isTombDone = true; + } } + + code = tsdbSnapReadFileSetEnd(reader); + TSDB_CHECK_CODE(code, lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pReader->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } else { - tsdbDebug("vgId:%d %s done", TD_VID(pReader->pTsdb->pVnode), __func__); + tsdbDebug("vgId:%d %s done", TD_VID(reader->tsdb->pVnode), __func__); } return code; } // STsdbSnapWriter ======================================== struct STsdbSnapWriter { - STsdb* pTsdb; + STsdb* tsdb; int64_t sver; int64_t ever; int32_t minutes; @@ -569,971 +535,595 @@ struct STsdbSnapWriter { int32_t maxRow; int8_t cmprAlg; int64_t commitID; + int32_t szPage; + int64_t compactVersion; + int64_t now; uint8_t* aBuf[5]; - STsdbFS fs; - TABLEID tbid; - - // time-series data - SBlockData inData; - - int32_t fid; - SSkmInfo skmTable; - - /* reader */ - SDataFReader* pDataFReader; - STsdbDataIter2* iterList; - STsdbDataIter2* pDIter; - STsdbDataIter2* pSIter; - SRBTree rbt; // SRBTree - - /* writer */ - SDataFWriter* pDataFWriter; - SArray* aBlockIdx; - SMapData mDataBlk; // SMapData - SArray* aSttBlk; // SArray - SBlockData bData; - SBlockData sData; - - // tombstone data - /* reader */ - SDelFReader* pDelFReader; - STsdbDataIter2* pTIter; - - /* writer */ - SDelFWriter* pDelFWriter; - SArray* aDelIdx; - SArray* aDelData; + TFileSetArray* fsetArr; + TFileOpArray fopArr[1]; + + struct { + bool fsetWriteBegin; + int32_t fid; + STFileSet* fset; + SDiskID did; + bool hasData; + bool hasTomb; + + // reader + SDataFileReader* dataReader; + TSttFileReaderArray sttReaderArr[1]; + + // iter/merger + TTsdbIterArray dataIterArr[1]; + SIterMerger* dataIterMerger; + TTsdbIterArray tombIterArr[1]; + SIterMerger* tombIterMerger; + + // writer + SFSetWriter* fsetWriter; + } ctx[1]; }; -// SNAP_DATA_TSDB -static int32_t tsdbSnapWriteTableDataStart(STsdbSnapWriter* pWriter, TABLEID* pId) { +// APIs +static int32_t tsdbSnapWriteTimeSeriesRow(STsdbSnapWriter* writer, SRowInfo* row) { int32_t code = 0; int32_t lino = 0; - if (pId) { - pWriter->tbid = *pId; - } else { - pWriter->tbid = (TABLEID){INT64_MAX, INT64_MAX}; - } - - if (pWriter->pDIter) { - STsdbDataIter2* pIter = pWriter->pDIter; - - // assert last table data end - ASSERT(pIter->dIter.iRow >= pIter->dIter.bData.nRow); - ASSERT(pIter->dIter.iDataBlk >= pIter->dIter.mDataBlk.nItem); - - for (;;) { - if (pIter->dIter.iBlockIdx >= taosArrayGetSize(pIter->dIter.aBlockIdx)) { - pWriter->pDIter = NULL; - break; - } - - SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->dIter.aBlockIdx, pIter->dIter.iBlockIdx); - - int32_t c = tTABLEIDCmprFn(pBlockIdx, &pWriter->tbid); - if (c < 0) { - code = tsdbReadDataBlk(pIter->dIter.pReader, pBlockIdx, &pIter->dIter.mDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - SBlockIdx* pNewBlockIdx = taosArrayReserve(pWriter->aBlockIdx, 1); - if (pNewBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - pNewBlockIdx->suid = pBlockIdx->suid; - pNewBlockIdx->uid = pBlockIdx->uid; - - code = tsdbWriteDataBlk(pWriter->pDataFWriter, &pIter->dIter.mDataBlk, pNewBlockIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - pIter->dIter.iBlockIdx++; - } else if (c == 0) { - code = tsdbReadDataBlk(pIter->dIter.pReader, pBlockIdx, &pIter->dIter.mDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - pIter->dIter.iDataBlk = 0; - pIter->dIter.iBlockIdx++; - - break; - } else { - pIter->dIter.iDataBlk = pIter->dIter.mDataBlk.nItem; - break; - } + while (writer->ctx->hasData) { + SRowInfo* row1 = tsdbIterMergerGetData(writer->ctx->dataIterMerger); + if (row1 == NULL) { + writer->ctx->hasData = false; + break; } - } - if (pId) { - code = tsdbUpdateTableSchema(pWriter->pTsdb->pVnode->pMeta, pId->suid, pId->uid, &pWriter->skmTable); - TSDB_CHECK_CODE(code, lino, _exit); - - tMapDataReset(&pWriter->mDataBlk); - - code = tBlockDataInit(&pWriter->bData, pId, pWriter->skmTable.pTSchema, NULL, 0); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (!TABLE_SAME_SCHEMA(pWriter->tbid.suid, pWriter->tbid.uid, pWriter->sData.suid, pWriter->sData.uid)) { - if ((pWriter->sData.nRow > 0)) { - code = tsdbWriteSttBlock(pWriter->pDataFWriter, &pWriter->sData, pWriter->aSttBlk, pWriter->cmprAlg); + int32_t c = tRowInfoCmprFn(row1, row); + if (c <= 0) { + code = tsdbFSetWriteRow(writer->ctx->fsetWriter, row1); TSDB_CHECK_CODE(code, lino, _exit); - } - if (pId) { - TABLEID id = {.suid = pWriter->tbid.suid, .uid = pWriter->tbid.suid ? 0 : pWriter->tbid.uid}; - code = tBlockDataInit(&pWriter->sData, &id, pWriter->skmTable.pTSchema, NULL, 0); + code = tsdbIterMergerNext(writer->ctx->dataIterMerger); TSDB_CHECK_CODE(code, lino, _exit); + } else { + break; } } -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbTrace("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), __func__, - pWriter->tbid.suid, pWriter->tbid.uid); + if (row->suid == INT64_MAX) { + ASSERT(writer->ctx->hasData == false); + goto _exit; } - return code; -} -static int32_t tsdbSnapWriteTableRowImpl(STsdbSnapWriter* pWriter, TSDBROW* pRow) { - int32_t code = 0; - int32_t lino = 0; - - code = tBlockDataAppendRow(&pWriter->bData, pRow, pWriter->skmTable.pTSchema, pWriter->tbid.uid); + code = tsdbFSetWriteRow(writer->ctx->fsetWriter, row); TSDB_CHECK_CODE(code, lino, _exit); - if (pWriter->bData.nRow >= pWriter->maxRow) { - code = tsdbWriteDataBlock(pWriter->pDataFWriter, &pWriter->bData, &pWriter->mDataBlk, pWriter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -static int32_t tsdbSnapWriteTableRow(STsdbSnapWriter* pWriter, TSDBROW* pRow) { +static int32_t tsdbSnapWriteFileSetOpenReader(STsdbSnapWriter* writer) { int32_t code = 0; int32_t lino = 0; - TSDBKEY inKey = pRow ? TSDBROW_KEY(pRow) : TSDBKEY_MAX; + ASSERT(writer->ctx->dataReader == NULL); + ASSERT(TARRAY2_SIZE(writer->ctx->sttReaderArr) == 0); - if (pWriter->pDIter == NULL || (pWriter->pDIter->dIter.iRow >= pWriter->pDIter->dIter.bData.nRow && - pWriter->pDIter->dIter.iDataBlk >= pWriter->pDIter->dIter.mDataBlk.nItem)) { - goto _write_row; - } else { - for (;;) { - while (pWriter->pDIter->dIter.iRow < pWriter->pDIter->dIter.bData.nRow) { - TSDBROW row = tsdbRowFromBlockData(&pWriter->pDIter->dIter.bData, pWriter->pDIter->dIter.iRow); - - int32_t c = tsdbKeyCmprFn(&inKey, &TSDBROW_KEY(&row)); - if (c < 0) { - goto _write_row; - } else if (c > 0) { - code = tsdbSnapWriteTableRowImpl(pWriter, &row); - TSDB_CHECK_CODE(code, lino, _exit); - - pWriter->pDIter->dIter.iRow++; - } else { - ASSERT(0); - } - } + if (writer->ctx->fset) { + // open data reader + SDataFileReaderConfig dataFileReaderConfig = { + .tsdb = writer->tsdb, + .bufArr = writer->aBuf, + .szPage = writer->szPage, + }; - for (;;) { - if (pWriter->pDIter->dIter.iDataBlk >= pWriter->pDIter->dIter.mDataBlk.nItem) goto _write_row; - - // FIXME: Here can be slow, use array instead - SDataBlk dataBlk; - tMapDataGetItemByIdx(&pWriter->pDIter->dIter.mDataBlk, pWriter->pDIter->dIter.iDataBlk, &dataBlk, tGetDataBlk); - - int32_t c = tDataBlkCmprFn(&dataBlk, &(SDataBlk){.minKey = inKey, .maxKey = inKey}); - if (c > 0) { - goto _write_row; - } else if (c < 0) { - if (pWriter->bData.nRow > 0) { - code = tsdbWriteDataBlock(pWriter->pDataFWriter, &pWriter->bData, &pWriter->mDataBlk, pWriter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - - tMapDataPutItem(&pWriter->mDataBlk, &dataBlk, tPutDataBlk); - pWriter->pDIter->dIter.iDataBlk++; - } else { - code = tsdbReadDataBlockEx(pWriter->pDataFReader, &dataBlk, &pWriter->pDIter->dIter.bData); - TSDB_CHECK_CODE(code, lino, _exit); - - pWriter->pDIter->dIter.iRow = 0; - pWriter->pDIter->dIter.iDataBlk++; - break; - } + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (writer->ctx->fset->farr[ftype] == NULL) { + continue; } + + dataFileReaderConfig.files[ftype].exist = true; + dataFileReaderConfig.files[ftype].file = writer->ctx->fset->farr[ftype]->f[0]; } - } -_write_row: - if (pRow) { - code = tsdbSnapWriteTableRowImpl(pWriter, pRow); + code = tsdbDataFileReaderOpen(NULL, &dataFileReaderConfig, &writer->ctx->dataReader); TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { - int32_t code = 0; - int32_t lino = 0; - // write a NULL row to end current table data write - code = tsdbSnapWriteTableRow(pWriter, NULL); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pWriter->bData.nRow > 0) { - if (pWriter->bData.nRow < pWriter->minRow) { - ASSERT(TABLE_SAME_SCHEMA(pWriter->sData.suid, pWriter->sData.uid, pWriter->tbid.suid, pWriter->tbid.uid)); - for (int32_t iRow = 0; iRow < pWriter->bData.nRow; iRow++) { - code = - tBlockDataAppendRow(&pWriter->sData, &tsdbRowFromBlockData(&pWriter->bData, iRow), NULL, pWriter->tbid.uid); + // open stt reader array + SSttLvl* lvl; + TARRAY2_FOREACH(writer->ctx->fset->lvlArr, lvl) { + STFileObj* fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + SSttFileReader* reader; + SSttFileReaderConfig sttFileReaderConfig = { + .tsdb = writer->tsdb, + .szPage = writer->szPage, + .bufArr = writer->aBuf, + .file = fobj->f[0], + }; + + code = tsdbSttFileReaderOpen(fobj->fname, &sttFileReaderConfig, &reader); TSDB_CHECK_CODE(code, lino, _exit); - if (pWriter->sData.nRow >= pWriter->maxRow) { - code = tsdbWriteSttBlock(pWriter->pDataFWriter, &pWriter->sData, pWriter->aSttBlk, pWriter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } + code = TARRAY2_APPEND(writer->ctx->sttReaderArr, reader); + TSDB_CHECK_CODE(code, lino, _exit); } - - tBlockDataClear(&pWriter->bData); - } else { - code = tsdbWriteDataBlock(pWriter->pDataFWriter, &pWriter->bData, &pWriter->mDataBlk, pWriter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - if (pWriter->mDataBlk.nItem) { - SBlockIdx* pBlockIdx = taosArrayReserve(pWriter->aBlockIdx, 1); - if (pBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); } - - pBlockIdx->suid = pWriter->tbid.suid; - pBlockIdx->uid = pWriter->tbid.uid; - - code = tsdbWriteDataBlk(pWriter->pDataFWriter, &pWriter->mDataBlk, pBlockIdx); - TSDB_CHECK_CODE(code, lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -static int32_t tsdbSnapWriteFileDataStart(STsdbSnapWriter* pWriter, int32_t fid) { +static int32_t tsdbSnapWriteFileSetCloseReader(STsdbSnapWriter* writer) { + TARRAY2_CLEAR(writer->ctx->sttReaderArr, tsdbSttFileReaderClose); + tsdbDataFileReaderClose(&writer->ctx->dataReader); + return 0; +} + +static int32_t tsdbSnapWriteFileSetOpenIter(STsdbSnapWriter* writer) { int32_t code = 0; int32_t lino = 0; - ASSERT(pWriter->pDataFWriter == NULL && pWriter->fid < fid); - - STsdb* pTsdb = pWriter->pTsdb; + // data ieter + if (writer->ctx->dataReader) { + STsdbIter* iter; + STsdbIterConfig config = {0}; - pWriter->fid = fid; - pWriter->tbid = (TABLEID){0}; - SDFileSet* pSet = taosArraySearch(pWriter->fs.aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ); + // data + config.type = TSDB_ITER_TYPE_DATA; + config.dataReader = writer->ctx->dataReader; - // open reader - pWriter->pDataFReader = NULL; - pWriter->iterList = NULL; - pWriter->pDIter = NULL; - pWriter->pSIter = NULL; - tRBTreeCreate(&pWriter->rbt, tsdbDataIterCmprFn); - if (pSet) { - code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet); + code = tsdbIterOpen(&config, &iter); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbOpenDataFileDataIter(pWriter->pDataFReader, &pWriter->pDIter); + code = TARRAY2_APPEND(writer->ctx->dataIterArr, iter); TSDB_CHECK_CODE(code, lino, _exit); - if (pWriter->pDIter) { - pWriter->pDIter->next = pWriter->iterList; - pWriter->iterList = pWriter->pDIter; - } - - for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { - code = tsdbOpenSttFileDataIter(pWriter->pDataFReader, iStt, &pWriter->pSIter); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pWriter->pSIter) { - code = tsdbDataIterNext2(pWriter->pSIter, NULL); - TSDB_CHECK_CODE(code, lino, _exit); - // add to tree - tRBTreePut(&pWriter->rbt, &pWriter->pSIter->rbtn); + // tome + config.type = TSDB_ITER_TYPE_DATA_TOMB; + config.dataReader = writer->ctx->dataReader; - // add to list - pWriter->pSIter->next = pWriter->iterList; - pWriter->iterList = pWriter->pSIter; - } - } - - pWriter->pSIter = NULL; - } - - // open writer - SDiskID diskId; - if (pSet) { - diskId = pSet->diskId; - } else { - tfsAllocDisk(pTsdb->pVnode->pTfs, 0 /*TODO*/, &diskId); - tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, diskId); - } - SDFileSet wSet = {.diskId = diskId, - .fid = fid, - .pHeadF = &(SHeadFile){.commitID = pWriter->commitID}, - .pDataF = (pSet) ? pSet->pDataF : &(SDataFile){.commitID = pWriter->commitID}, - .pSmaF = (pSet) ? pSet->pSmaF : &(SSmaFile){.commitID = pWriter->commitID}, - .nSttF = 1, - .aSttF = {&(SSttFile){.commitID = pWriter->commitID}}}; - code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet); - TSDB_CHECK_CODE(code, lino, _exit); + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); - if (pWriter->aBlockIdx) { - taosArrayClear(pWriter->aBlockIdx); - } else if ((pWriter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = TARRAY2_APPEND(writer->ctx->tombIterArr, iter); TSDB_CHECK_CODE(code, lino, _exit); } - tMapDataReset(&pWriter->mDataBlk); + // stt iter + SSttFileReader* sttFileReader; + TARRAY2_FOREACH(writer->ctx->sttReaderArr, sttFileReader) { + STsdbIter* iter; + STsdbIterConfig config = {0}; - if (pWriter->aSttBlk) { - taosArrayClear(pWriter->aSttBlk); - } else if ((pWriter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + // data + config.type = TSDB_ITER_TYPE_STT; + config.sttReader = sttFileReader; + + code = tsdbIterOpen(&config, &iter); TSDB_CHECK_CODE(code, lino, _exit); - } - tBlockDataReset(&pWriter->bData); - tBlockDataReset(&pWriter->sData); + code = TARRAY2_APPEND(writer->ctx->dataIterArr, iter); + TSDB_CHECK_CODE(code, lino, _exit); -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s, fid:%d", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code), - fid); - } else { - tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(pTsdb->pVnode), __func__, fid); - } - return code; -} + // tomb + config.type = TSDB_ITER_TYPE_STT_TOMB; + config.sttReader = sttFileReader; -static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, SRowInfo* pRowInfo) { - int32_t code = 0; - int32_t lino = 0; - - // switch to new table if need - if (pRowInfo == NULL || pRowInfo->uid != pWriter->tbid.uid) { - if (pWriter->tbid.uid) { - code = tsdbSnapWriteTableDataEnd(pWriter); - TSDB_CHECK_CODE(code, lino, _exit); - } + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbSnapWriteTableDataStart(pWriter, (TABLEID*)pRowInfo); + code = TARRAY2_APPEND(writer->ctx->tombIterArr, iter); TSDB_CHECK_CODE(code, lino, _exit); } - if (pRowInfo == NULL) goto _exit; + // open merger + code = tsdbIterMergerOpen(writer->ctx->dataIterArr, &writer->ctx->dataIterMerger, false); + TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbSnapWriteTableRow(pWriter, &pRowInfo->row); + code = tsdbIterMergerOpen(writer->ctx->tombIterArr, &writer->ctx->tombIterMerger, true); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -static int32_t tsdbSnapWriteNextRow(STsdbSnapWriter* pWriter, SRowInfo** ppRowInfo) { - int32_t code = 0; - int32_t lino = 0; - - if (pWriter->pSIter) { - code = tsdbDataIterNext2(pWriter->pSIter, NULL); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pWriter->pSIter->rowInfo.suid == 0 && pWriter->pSIter->rowInfo.uid == 0) { - pWriter->pSIter = NULL; - } else { - SRBTreeNode* pNode = tRBTreeMin(&pWriter->rbt); - if (pNode) { - int32_t c = tsdbDataIterCmprFn(&pWriter->pSIter->rbtn, pNode); - if (c > 0) { - tRBTreePut(&pWriter->rbt, &pWriter->pSIter->rbtn); - pWriter->pSIter = NULL; - } else if (c == 0) { - ASSERT(0); - } - } - } - } - - if (pWriter->pSIter == NULL) { - SRBTreeNode* pNode = tRBTreeMin(&pWriter->rbt); - if (pNode) { - tRBTreeDrop(&pWriter->rbt, pNode); - pWriter->pSIter = TSDB_RBTN_TO_DATA_ITER(pNode); - } - } - - if (ppRowInfo) { - if (pWriter->pSIter) { - *ppRowInfo = &pWriter->pSIter->rowInfo; - } else { - *ppRowInfo = NULL; - } - } - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; +static int32_t tsdbSnapWriteFileSetCloseIter(STsdbSnapWriter* writer) { + tsdbIterMergerClose(&writer->ctx->dataIterMerger); + tsdbIterMergerClose(&writer->ctx->tombIterMerger); + TARRAY2_CLEAR(writer->ctx->dataIterArr, tsdbIterClose); + TARRAY2_CLEAR(writer->ctx->tombIterArr, tsdbIterClose); + return 0; } -static int32_t tsdbSnapWriteGetRow(STsdbSnapWriter* pWriter, SRowInfo** ppRowInfo) { +static int32_t tsdbSnapWriteFileSetOpenWriter(STsdbSnapWriter* writer) { int32_t code = 0; int32_t lino = 0; - if (pWriter->pSIter) { - *ppRowInfo = &pWriter->pSIter->rowInfo; - goto _exit; - } - - code = tsdbSnapWriteNextRow(pWriter, ppRowInfo); + SFSetWriterConfig config = { + .tsdb = writer->tsdb, + .toSttOnly = false, + .compactVersion = writer->compactVersion, + .minRow = writer->minRow, + .maxRow = writer->maxRow, + .szPage = writer->szPage, + .cmprAlg = writer->cmprAlg, + .fid = writer->ctx->fid, + .cid = writer->commitID, + .did = writer->ctx->did, + .level = 0, + }; + + code = tsdbFSetWriterOpen(&config, &writer->ctx->fsetWriter); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -static int32_t tsdbSnapWriteFileDataEnd(STsdbSnapWriter* pWriter) { +static int32_t tsdbSnapWriteFileSetCloseWriter(STsdbSnapWriter* writer) { + return tsdbFSetWriterClose(&writer->ctx->fsetWriter, 0, writer->fopArr); +} + +static int32_t tsdbSnapWriteFileSetBegin(STsdbSnapWriter* writer, int32_t fid) { int32_t code = 0; int32_t lino = 0; - ASSERT(pWriter->pDataFWriter); + ASSERT(writer->ctx->fsetWriteBegin == false); - // consume remain data and end with a NULL table row - SRowInfo* pRowInfo; - code = tsdbSnapWriteGetRow(pWriter, &pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); - for (;;) { - code = tsdbSnapWriteTableData(pWriter, pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); + STFileSet* fset = &(STFileSet){.fid = fid}; - if (pRowInfo == NULL) break; + writer->ctx->fid = fid; + STFileSet** fsetPtr = TARRAY2_SEARCH(writer->fsetArr, &fset, tsdbTFileSetCmprFn, TD_EQ); + writer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr; - code = tsdbSnapWriteNextRow(pWriter, &pRowInfo); + int32_t level = tsdbFidLevel(fid, &writer->tsdb->keepCfg, taosGetTimestampSec()); + if (tfsAllocDisk(writer->tsdb->pVnode->pTfs, level, &writer->ctx->did)) { + code = TSDB_CODE_NO_AVAIL_DISK; TSDB_CHECK_CODE(code, lino, _exit); } + tfsMkdirRecurAt(writer->tsdb->pVnode->pTfs, writer->tsdb->path, writer->ctx->did); - // do file-level updates - code = tsdbWriteSttBlk(pWriter->pDataFWriter, pWriter->aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdx); - TSDB_CHECK_CODE(code, lino, _exit); + writer->ctx->hasData = true; + writer->ctx->hasTomb = true; - code = tsdbUpdateDFileSetHeader(pWriter->pDataFWriter); + code = tsdbSnapWriteFileSetOpenReader(writer); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->pDataFWriter->wSet); + code = tsdbSnapWriteFileSetOpenIter(writer); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1); + code = tsdbSnapWriteFileSetOpenWriter(writer); TSDB_CHECK_CODE(code, lino, _exit); - if (pWriter->pDataFReader) { - code = tsdbDataFReaderClose(&pWriter->pDataFReader); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // clear sources - while (pWriter->iterList) { - STsdbDataIter2* pIter = pWriter->iterList; - pWriter->iterList = pIter->next; - tsdbCloseDataIter2(pIter); - } + writer->ctx->fsetWriteBegin = true; _exit: if (code) { - tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code)); - } else { - tsdbDebug("vgId:%d %s is done", TD_VID(pWriter->pTsdb->pVnode), __func__); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -static int32_t tsdbSnapWriteTimeSeriesData(STsdbSnapWriter* pWriter, SSnapDataHdr* pHdr) { +static int32_t tsdbSnapWriteTombRecord(STsdbSnapWriter* writer, const STombRecord* record) { int32_t code = 0; int32_t lino = 0; - code = tDecmprBlockData(pHdr->data, pHdr->size, &pWriter->inData, pWriter->aBuf); - TSDB_CHECK_CODE(code, lino, _exit); - - ASSERT(pWriter->inData.nRow > 0); + while (writer->ctx->hasTomb) { + STombRecord* record1 = tsdbIterMergerGetTombRecord(writer->ctx->tombIterMerger); + if (record1 == NULL) { + writer->ctx->hasTomb = false; + break; + } - // switch to new data file if need - int32_t fid = tsdbKeyFid(pWriter->inData.aTSKEY[0], pWriter->minutes, pWriter->precision); - if (pWriter->fid != fid) { - if (pWriter->pDataFWriter) { - code = tsdbSnapWriteFileDataEnd(pWriter); + int32_t c = tTombRecordCompare(record1, record); + if (c <= 0) { + code = tsdbFSetWriteTombRecord(writer->ctx->fsetWriter, record1); TSDB_CHECK_CODE(code, lino, _exit); + } else { + break; } + } - code = tsdbSnapWriteFileDataStart(pWriter, fid); - TSDB_CHECK_CODE(code, lino, _exit); + if (record->suid == INT64_MAX) { + ASSERT(writer->ctx->hasTomb == false); + goto _exit; } - // loop write each row - SRowInfo* pRowInfo; - code = tsdbSnapWriteGetRow(pWriter, &pRowInfo); + code = tsdbFSetWriteTombRecord(writer->ctx->fsetWriter, record); TSDB_CHECK_CODE(code, lino, _exit); - for (int32_t iRow = 0; iRow < pWriter->inData.nRow; ++iRow) { - SRowInfo rInfo = {.suid = pWriter->inData.suid, - .uid = pWriter->inData.uid ? pWriter->inData.uid : pWriter->inData.aUid[iRow], - .row = tsdbRowFromBlockData(&pWriter->inData, iRow)}; - - for (;;) { - if (pRowInfo == NULL) { - code = tsdbSnapWriteTableData(pWriter, &rInfo); - TSDB_CHECK_CODE(code, lino, _exit); - break; - } else { - int32_t c = tRowInfoCmprFn(&rInfo, pRowInfo); - if (c < 0) { - code = tsdbSnapWriteTableData(pWriter, &rInfo); - TSDB_CHECK_CODE(code, lino, _exit); - break; - } else if (c > 0) { - code = tsdbSnapWriteTableData(pWriter, pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbSnapWriteNextRow(pWriter, &pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - ASSERT(0); - } - } - } - } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbDebug("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64 " nRow:%d", TD_VID(pWriter->pTsdb->pVnode), __func__, - pWriter->inData.suid, pWriter->inData.uid, pWriter->inData.nRow); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -// SNAP_DATA_DEL -static int32_t tsdbSnapWriteDelTableDataStart(STsdbSnapWriter* pWriter, TABLEID* pId) { +static int32_t tsdbSnapWriteFileSetEnd(STsdbSnapWriter* writer) { + if (!writer->ctx->fsetWriteBegin) return 0; + int32_t code = 0; int32_t lino = 0; - if (pId) { - pWriter->tbid = *pId; - } else { - pWriter->tbid = (TABLEID){.suid = INT64_MAX, .uid = INT64_MAX}; - } - - taosArrayClear(pWriter->aDelData); + SRowInfo row = { + .suid = INT64_MAX, + .uid = INT64_MAX, + }; - if (pWriter->pTIter) { - while (pWriter->pTIter->tIter.iDelIdx < taosArrayGetSize(pWriter->pTIter->tIter.aDelIdx)) { - SDelIdx* pDelIdx = taosArrayGet(pWriter->pTIter->tIter.aDelIdx, pWriter->pTIter->tIter.iDelIdx); + code = tsdbSnapWriteTimeSeriesRow(writer, &row); + TSDB_CHECK_CODE(code, lino, _exit); - int32_t c = tTABLEIDCmprFn(pDelIdx, &pWriter->tbid); - if (c < 0) { - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->pTIter->tIter.aDelData); - TSDB_CHECK_CODE(code, lino, _exit); + STombRecord record = { + .suid = INT64_MAX, + .uid = INT64_MAX, + }; - SDelIdx* pDelIdxNew = taosArrayReserve(pWriter->aDelIdx, 1); - if (pDelIdxNew == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } + code = tsdbSnapWriteTombRecord(writer, &record); + TSDB_CHECK_CODE(code, lino, _exit); - pDelIdxNew->suid = pDelIdx->suid; - pDelIdxNew->uid = pDelIdx->uid; + // close write + code = tsdbSnapWriteFileSetCloseWriter(writer); + TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->pTIter->tIter.aDelData, pDelIdxNew); - TSDB_CHECK_CODE(code, lino, _exit); + code = tsdbSnapWriteFileSetCloseIter(writer); + TSDB_CHECK_CODE(code, lino, _exit); - pWriter->pTIter->tIter.iDelIdx++; - } else if (c == 0) { - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); - TSDB_CHECK_CODE(code, lino, _exit); + code = tsdbSnapWriteFileSetCloseReader(writer); + TSDB_CHECK_CODE(code, lino, _exit); - pWriter->pTIter->tIter.iDelIdx++; - break; - } else { - break; - } - } - } + writer->ctx->fsetWriteBegin = false; _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbTrace("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), __func__, - pWriter->tbid.suid, pWriter->tbid.uid); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -static int32_t tsdbSnapWriteDelTableDataEnd(STsdbSnapWriter* pWriter) { +static int32_t tsdbSnapWriteTimeSeriesData(STsdbSnapWriter* writer, SSnapDataHdr* hdr) { int32_t code = 0; int32_t lino = 0; - if (taosArrayGetSize(pWriter->aDelData) > 0) { - SDelIdx* pDelIdx = taosArrayReserve(pWriter->aDelIdx, 1); - if (pDelIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } + SBlockData blockData[1] = {0}; - pDelIdx->suid = pWriter->tbid.suid; - pDelIdx->uid = pWriter->tbid.uid; + code = tDecmprBlockData(hdr->data, hdr->size - sizeof(*hdr), blockData, writer->aBuf); + TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, pDelIdx); + int32_t fid = tsdbKeyFid(blockData->aTSKEY[0], writer->minutes, writer->precision); + if (!writer->ctx->fsetWriteBegin || fid != writer->ctx->fid) { + code = tsdbSnapWriteFileSetEnd(writer); TSDB_CHECK_CODE(code, lino, _exit); - } -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbTrace("vgId:%d %s done", TD_VID(pWriter->pTsdb->pVnode), __func__); - } - return code; -} - -static int32_t tsdbSnapWriteDelTableData(STsdbSnapWriter* pWriter, TABLEID* pId, uint8_t* pData, int64_t size) { - int32_t code = 0; - int32_t lino = 0; - - if (pId == NULL || pId->uid != pWriter->tbid.uid) { - if (pWriter->tbid.uid) { - code = tsdbSnapWriteDelTableDataEnd(pWriter); - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tsdbSnapWriteDelTableDataStart(pWriter, pId); + code = tsdbSnapWriteFileSetBegin(writer, fid); TSDB_CHECK_CODE(code, lino, _exit); } - if (pId == NULL) goto _exit; - - int64_t n = 0; - while (n < size) { - SDelData delData; - n += tGetDelData(pData + n, &delData); + for (int32_t i = 0; i < blockData->nRow; ++i) { + SRowInfo rowInfo = { + .suid = blockData->suid, + .uid = blockData->uid ? blockData->uid : blockData->aUid[i], + .row = tsdbRowFromBlockData(blockData, i), + }; - if (taosArrayPush(pWriter->aDelData, &delData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } + code = tsdbSnapWriteTimeSeriesRow(writer, &rowInfo); + TSDB_CHECK_CODE(code, lino, _exit); } - ASSERT(n == size); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64 " nRow:%d", TD_VID(writer->tsdb->pVnode), __func__, + blockData->suid, blockData->uid, blockData->nRow); } + tBlockDataDestroy(blockData); return code; } -static int32_t tsdbSnapWriteDelDataStart(STsdbSnapWriter* pWriter) { +static int32_t tsdbSnapWriteDecmprTombBlock(SSnapDataHdr* hdr, STombBlock* tombBlock) { int32_t code = 0; int32_t lino = 0; - STsdb* pTsdb = pWriter->pTsdb; - SDelFile* pDelFile = pWriter->fs.pDelFile; - - pWriter->tbid = (TABLEID){0}; + int64_t size = hdr->size; + ASSERT(size % TOMB_RECORD_ELEM_NUM == 0); + size = size / TOMB_RECORD_ELEM_NUM; + ASSERT(size % sizeof(int64_t) == 0); - // reader - if (pDelFile) { - code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbOpenTombFileDataIter(pWriter->pDelFReader, &pWriter->pTIter); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // writer - code = tsdbDelFWriterOpen(&pWriter->pDelFWriter, &(SDelFile){.commitID = pWriter->commitID}, pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); - - if ((pWriter->aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - if ((pWriter->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + int64_t* data = (int64_t*)hdr->data; + for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { + code = TARRAY2_APPEND_BATCH(&tombBlock->dataArr[i], hdr->data + i * size, size / sizeof(int64_t)); TSDB_CHECK_CODE(code, lino, _exit); } _exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbDebug("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); - } return code; } -static int32_t tsdbSnapWriteDelDataEnd(STsdbSnapWriter* pWriter) { +static int32_t tsdbSnapWriteTombData(STsdbSnapWriter* writer, SSnapDataHdr* hdr) { int32_t code = 0; int32_t lino = 0; - STsdb* pTsdb = pWriter->pTsdb; - - // end remaining table with NULL data - code = tsdbSnapWriteDelTableData(pWriter, NULL, NULL, 0); - TSDB_CHECK_CODE(code, lino, _exit); - - // update file-level info - code = tsdbWriteDelIdx(pWriter->pDelFWriter, pWriter->aDelIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbUpdateDelFileHdr(pWriter->pDelFWriter); - TSDB_CHECK_CODE(code, lino, _exit); + STombRecord record; + STombBlock tombBlock[1] = {0}; - code = tsdbFSUpsertDelFile(&pWriter->fs, &pWriter->pDelFWriter->fDel); + code = tsdbSnapWriteDecmprTombBlock(hdr, tombBlock); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbDelFWriterClose(&pWriter->pDelFWriter, 1); - TSDB_CHECK_CODE(code, lino, _exit); + tTombBlockGet(tombBlock, 0, &record); + int32_t fid = tsdbKeyFid(record.skey, writer->minutes, writer->precision); + if (!writer->ctx->fsetWriteBegin || fid != writer->ctx->fid) { + code = tsdbSnapWriteFileSetEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); - if (pWriter->pDelFReader) { - code = tsdbDelFReaderClose(&pWriter->pDelFReader); + code = tsdbSnapWriteFileSetBegin(writer, fid); TSDB_CHECK_CODE(code, lino, _exit); } - if (pWriter->pTIter) { - tsdbCloseDataIter2(pWriter->pTIter); - pWriter->pTIter = NULL; - } + if (writer->ctx->hasData) { + SRowInfo row = { + .suid = INT64_MAX, + .uid = INT64_MAX, + }; -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + code = tsdbSnapWriteTimeSeriesRow(writer, &row); + TSDB_CHECK_CODE(code, lino, _exit); } - return code; -} -static int32_t tsdbSnapWriteDelData(STsdbSnapWriter* pWriter, SSnapDataHdr* pHdr) { - int32_t code = 0; - int32_t lino = 0; + ASSERT(writer->ctx->hasData == false); - STsdb* pTsdb = pWriter->pTsdb; + for (int32_t i = 0; i < TOMB_BLOCK_SIZE(tombBlock); ++i) { + tTombBlockGet(tombBlock, i, &record); - // start to write del data if need - if (pWriter->pDelFWriter == NULL) { - code = tsdbSnapWriteDelDataStart(pWriter); + code = tsdbSnapWriteTombRecord(writer, &record); TSDB_CHECK_CODE(code, lino, _exit); } - // do write del data - code = tsdbSnapWriteDelTableData(pWriter, (TABLEID*)pHdr->data, pHdr->data + sizeof(TABLEID), - pHdr->size - sizeof(TABLEID)); - TSDB_CHECK_CODE(code, lino, _exit); + tTombBlockDestroy(tombBlock); _exit: if (code) { - tsdbError("vgId:%d %s failed since %s", TD_VID(pTsdb->pVnode), __func__, tstrerror(code)); - } else { - tsdbTrace("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -// APIs -int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) { +int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** writer) { int32_t code = 0; int32_t lino = 0; - // alloc - STsdbSnapWriter* pWriter = (STsdbSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); - if (pWriter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - pWriter->pTsdb = pTsdb; - pWriter->sver = sver; - pWriter->ever = ever; - pWriter->minutes = pTsdb->keepCfg.days; - pWriter->precision = pTsdb->keepCfg.precision; - pWriter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; - pWriter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; - pWriter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; - pWriter->commitID = pTsdb->pVnode->state.commitID; - - code = tsdbFSCopy(pTsdb, &pWriter->fs); - TSDB_CHECK_CODE(code, lino, _exit); - - // SNAP_DATA_TSDB - code = tBlockDataCreate(&pWriter->inData); - TSDB_CHECK_CODE(code, lino, _exit); - - pWriter->fid = INT32_MIN; - - code = tBlockDataCreate(&pWriter->bData); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tBlockDataCreate(&pWriter->sData); + writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); + if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->tsdb = pTsdb; + writer[0]->sver = sver; + writer[0]->ever = ever; + writer[0]->minutes = pTsdb->keepCfg.days; + writer[0]->precision = pTsdb->keepCfg.precision; + writer[0]->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; + writer[0]->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; + writer[0]->commitID = tsdbFSAllocEid(pTsdb->pFS); + writer[0]->szPage = pTsdb->pVnode->config.tsdbPageSize; + writer[0]->compactVersion = INT64_MAX; + writer[0]->now = taosGetTimestampMs(); + + code = tsdbFSCreateCopySnapshot(pTsdb->pFS, &writer[0]->fsetArr); TSDB_CHECK_CODE(code, lino, _exit); - // SNAP_DATA_DEL + tsdbFSDisableBgTask(pTsdb->pFS); _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - if (pWriter) { - tBlockDataDestroy(&pWriter->sData); - tBlockDataDestroy(&pWriter->bData); - tBlockDataDestroy(&pWriter->inData); - tsdbFSDestroy(&pWriter->fs); - taosMemoryFree(pWriter); - pWriter = NULL; - } } else { tsdbInfo("vgId:%d %s done, sver:%" PRId64 " ever:%" PRId64, TD_VID(pTsdb->pVnode), __func__, sver, ever); } - *ppWriter = pWriter; return code; } -int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter) { +int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* writer) { int32_t code = 0; int32_t lino = 0; - if (pWriter->pDataFWriter) { - code = tsdbSnapWriteFileDataEnd(pWriter); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (pWriter->pDelFWriter) { - code = tsdbSnapWriteDelDataEnd(pWriter); - TSDB_CHECK_CODE(code, lino, _exit); - } + code = tsdbSnapWriteFileSetEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbFSPrepareCommit(pWriter->pTsdb, &pWriter->fs); + code = tsdbFSEditBegin(writer->tsdb->pFS, writer->fopArr, TSDB_FEDIT_COMMIT); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } else { - tsdbDebug("vgId:%d %s done", TD_VID(pWriter->pTsdb->pVnode), __func__); + tsdbDebug("vgId:%d %s done", TD_VID(writer->tsdb->pVnode), __func__); } return code; } -int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { +int32_t tsdbSnapWriterClose(STsdbSnapWriter** writer, int8_t rollback) { + if (writer[0] == NULL) return 0; + int32_t code = 0; int32_t lino = 0; - STsdbSnapWriter* pWriter = *ppWriter; - STsdb* pTsdb = pWriter->pTsdb; + STsdb* tsdb = writer[0]->tsdb; if (rollback) { - tsdbRollbackCommit(pWriter->pTsdb); + code = tsdbFSEditAbort(writer[0]->tsdb->pFS); + TSDB_CHECK_CODE(code, lino, _exit); } else { - // lock - taosThreadRwlockWrlock(&pTsdb->rwLock); + taosThreadRwlockWrlock(&writer[0]->tsdb->rwLock); - code = tsdbFSCommit(pWriter->pTsdb); + code = tsdbFSEditCommit(writer[0]->tsdb->pFS); if (code) { - taosThreadRwlockUnlock(&pTsdb->rwLock); + taosThreadRwlockUnlock(&writer[0]->tsdb->rwLock); TSDB_CHECK_CODE(code, lino, _exit); } - // unlock - taosThreadRwlockUnlock(&pTsdb->rwLock); + taosThreadRwlockUnlock(&writer[0]->tsdb->rwLock); } + tsdbFSEnableBgTask(tsdb->pFS); + + tsdbIterMergerClose(&writer[0]->ctx->tombIterMerger); + tsdbIterMergerClose(&writer[0]->ctx->dataIterMerger); + TARRAY2_DESTROY(writer[0]->ctx->tombIterArr, tsdbIterClose); + TARRAY2_DESTROY(writer[0]->ctx->dataIterArr, tsdbIterClose); + TARRAY2_DESTROY(writer[0]->ctx->sttReaderArr, tsdbSttFileReaderClose); + tsdbDataFileReaderClose(&writer[0]->ctx->dataReader); - // SNAP_DATA_DEL - taosArrayDestroy(pWriter->aDelData); - taosArrayDestroy(pWriter->aDelIdx); - - // SNAP_DATA_TSDB - tBlockDataDestroy(&pWriter->sData); - tBlockDataDestroy(&pWriter->bData); - taosArrayDestroy(pWriter->aSttBlk); - tMapDataClear(&pWriter->mDataBlk); - taosArrayDestroy(pWriter->aBlockIdx); - tDestroyTSchema(pWriter->skmTable.pTSchema); - tBlockDataDestroy(&pWriter->inData); - - for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { - tFree(pWriter->aBuf[iBuf]); + TARRAY2_DESTROY(writer[0]->fopArr, NULL); + tsdbFSDestroyCopySnapshot(&writer[0]->fsetArr); + + for (int32_t i = 0; i < ARRAY_SIZE(writer[0]->aBuf); ++i) { + tFree(writer[0]->aBuf[i]); } - tsdbFSDestroy(&pWriter->fs); - taosMemoryFree(pWriter); - *ppWriter = NULL; + + taosMemoryFree(writer[0]); + writer[0] = NULL; _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); } else { - tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); } return code; } -int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, SSnapDataHdr* pHdr) { +int32_t tsdbSnapWrite(STsdbSnapWriter* writer, SSnapDataHdr* hdr) { int32_t code = 0; int32_t lino = 0; - if (pHdr->type == SNAP_DATA_TSDB) { - code = tsdbSnapWriteTimeSeriesData(pWriter, pHdr); + if (hdr->type == SNAP_DATA_TSDB) { + code = tsdbSnapWriteTimeSeriesData(writer, hdr); TSDB_CHECK_CODE(code, lino, _exit); - goto _exit; - } else if (pWriter->pDataFWriter) { - code = tsdbSnapWriteFileDataEnd(pWriter); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (pHdr->type == SNAP_DATA_DEL) { - code = tsdbSnapWriteDelData(pWriter, pHdr); + } else if (hdr->type == SNAP_DATA_DEL) { + code = tsdbSnapWriteTombData(writer, hdr); TSDB_CHECK_CODE(code, lino, _exit); - goto _exit; + } else { + ASSERT(0); } _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s, type:%d index:%" PRId64 " size:%" PRId64, - TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code), pHdr->type, pHdr->index, pHdr->size); + TD_VID(writer->tsdb->pVnode), __func__, lino, tstrerror(code), hdr->type, hdr->index, hdr->size); } else { - tsdbDebug("vgId:%d %s done, type:%d index:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), __func__, - pHdr->type, pHdr->index, pHdr->size); + tsdbDebug("vgId:%d %s done, type:%d index:%" PRId64 " size:%" PRId64, TD_VID(writer->tsdb->pVnode), __func__, + hdr->type, hdr->index, hdr->size); } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c new file mode 100644 index 0000000000000000000000000000000000000000..27fae9dc6ee6892dac60a7b7df3badddb6cfa9d6 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -0,0 +1,987 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbSttFileRW.h" + +// SSttFReader ============================================================ +struct SSttFileReader { + SSttFileReaderConfig config[1]; + STsdbFD *fd; + SSttFooter footer[1]; + struct { + bool sttBlkLoaded; + bool statisBlkLoaded; + bool tombBlkLoaded; + } ctx[1]; + TSttBlkArray sttBlkArray[1]; + TStatisBlkArray statisBlkArray[1]; + TTombBlkArray tombBlkArray[1]; + uint8_t *bufArr[5]; +}; + +// SSttFileReader +int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *config, SSttFileReader **reader) { + int32_t code = 0; + int32_t lino = 0; + + reader[0] = taosMemoryCalloc(1, sizeof(*reader[0])); + if (reader[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + reader[0]->config[0] = config[0]; + if (reader[0]->config->bufArr == NULL) { + reader[0]->config->bufArr = reader[0]->bufArr; + } + + // open file + if (fname) { + code = tsdbOpenFile(fname, config->szPage, TD_FILE_READ, &reader[0]->fd); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + char fname1[TSDB_FILENAME_LEN]; + tsdbTFileName(config->tsdb, config->file, fname1); + code = tsdbOpenFile(fname1, config->szPage, TD_FILE_READ, &reader[0]->fd); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // // open each segment reader + int64_t offset = config->file->size - sizeof(SSttFooter); + ASSERT(offset >= TSDB_FHDR_SIZE); + + code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter)); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + tsdbSttFileReaderClose(reader); + } + return code; +} + +int32_t tsdbSttFileReaderClose(SSttFileReader **reader) { + if (reader[0]) { + for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->bufArr); ++i) { + tFree(reader[0]->bufArr[i]); + } + tsdbCloseFile(&reader[0]->fd); + TARRAY2_DESTROY(reader[0]->tombBlkArray, NULL); + TARRAY2_DESTROY(reader[0]->statisBlkArray, NULL); + TARRAY2_DESTROY(reader[0]->sttBlkArray, NULL); + taosMemoryFree(reader[0]); + reader[0] = NULL; + } + return 0; +} + +// SSttFSegReader +int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray **statisBlkArray) { + if (!reader->ctx->statisBlkLoaded) { + if (reader->footer->statisBlkPtr->size > 0) { + ASSERT(reader->footer->statisBlkPtr->size % sizeof(SStatisBlk) == 0); + + int32_t size = reader->footer->statisBlkPtr->size / sizeof(SStatisBlk); + void *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size); + if (!data) return TSDB_CODE_OUT_OF_MEMORY; + + int32_t code = + tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size); + if (code) { + taosMemoryFree(data); + return code; + } + + TARRAY2_INIT_EX(reader->statisBlkArray, size, size, data); + } else { + TARRAY2_INIT(reader->statisBlkArray); + } + + reader->ctx->statisBlkLoaded = true; + } + + statisBlkArray[0] = reader->statisBlkArray; + return 0; +} + +int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tombBlkArray) { + if (!reader->ctx->tombBlkLoaded) { + if (reader->footer->tombBlkPtr->size > 0) { + ASSERT(reader->footer->tombBlkPtr->size % sizeof(STombBlk) == 0); + + int32_t size = reader->footer->tombBlkPtr->size / sizeof(STombBlk); + void *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size); + if (!data) return TSDB_CODE_OUT_OF_MEMORY; + + int32_t code = + tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size); + if (code) { + taosMemoryFree(data); + return code; + } + + TARRAY2_INIT_EX(reader->tombBlkArray, size, size, data); + } else { + TARRAY2_INIT(reader->tombBlkArray); + } + + reader->ctx->tombBlkLoaded = true; + } + + tombBlkArray[0] = reader->tombBlkArray; + return 0; +} + +int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBlkArray) { + if (!reader->ctx->sttBlkLoaded) { + if (reader->footer->sttBlkPtr->size > 0) { + ASSERT(reader->footer->sttBlkPtr->size % sizeof(SSttBlk) == 0); + + int32_t size = reader->footer->sttBlkPtr->size / sizeof(SSttBlk); + void *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size); + if (!data) return TSDB_CODE_OUT_OF_MEMORY; + + int32_t code = tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size); + if (code) { + taosMemoryFree(data); + return code; + } + + TARRAY2_INIT_EX(reader->sttBlkArray, size, size, data); + } else { + TARRAY2_INIT(reader->sttBlkArray); + } + + reader->ctx->sttBlkLoaded = true; + } + + sttBlkArray[0] = reader->sttBlkArray; + return 0; +} + +int32_t tsdbSttFileReadBlockData(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData) { + int32_t code = 0; + int32_t lino = 0; + + code = tRealloc(&reader->config->bufArr[0], sttBlk->bInfo.szBlock); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szBlock); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tDecmprBlockData(reader->config->bufArr[0], sttBlk->bInfo.szBlock, bData, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData, + STSchema *pTSchema, int16_t cids[], int32_t ncid) { + int32_t code = 0; + int32_t lino = 0; + + TABLEID tbid = {.suid = sttBlk->suid}; + if (tbid.suid == 0) { + tbid.uid = sttBlk->minUid; + } else { + tbid.uid = 0; + } + + code = tBlockDataInit(bData, &tbid, pTSchema, cids, ncid); + TSDB_CHECK_CODE(code, lino, _exit); + + // uid + version + tskey + code = tRealloc(&reader->config->bufArr[0], sttBlk->bInfo.szKey); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szKey); + TSDB_CHECK_CODE(code, lino, _exit); + + // hdr + SDiskDataHdr hdr[1]; + int32_t size = 0; + + size += tGetDiskDataHdr(reader->config->bufArr[0] + size, hdr); + + ASSERT(hdr->delimiter == TSDB_FILE_DLMT); + + bData->nRow = hdr->nRow; + bData->uid = hdr->uid; + + // uid + if (hdr->uid == 0) { + ASSERT(hdr->szUid); + code = tsdbDecmprData(reader->config->bufArr[0] + size, hdr->szUid, TSDB_DATA_TYPE_BIGINT, hdr->cmprAlg, + (uint8_t **)&bData->aUid, sizeof(int64_t) * hdr->nRow, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + ASSERT(hdr->szUid == 0); + } + size += hdr->szUid; + + // version + code = tsdbDecmprData(reader->config->bufArr[0] + size, hdr->szVer, TSDB_DATA_TYPE_BIGINT, hdr->cmprAlg, + (uint8_t **)&bData->aVersion, sizeof(int64_t) * hdr->nRow, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + size += hdr->szVer; + + // ts + code = tsdbDecmprData(reader->config->bufArr[0] + size, hdr->szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr->cmprAlg, + (uint8_t **)&bData->aTSKEY, sizeof(TSKEY) * hdr->nRow, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + size += hdr->szKey; + + ASSERT(size == sttBlk->bInfo.szKey); + + // other columns + if (bData->nColData > 0) { + if (hdr->szBlkCol > 0) { + code = tRealloc(&reader->config->bufArr[0], hdr->szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, reader->config->bufArr[0], + hdr->szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + } + + SBlockCol bc[1] = {{.cid = 0}}; + SBlockCol *blockCol = bc; + + size = 0; + for (int32_t i = 0; i < bData->nColData; i++) { + SColData *colData = tBlockDataGetColDataByIdx(bData, i); + + while (blockCol && blockCol->cid < colData->cid) { + if (size < hdr->szBlkCol) { + size += tGetBlockCol(reader->config->bufArr[0] + size, blockCol); + } else { + ASSERT(size == hdr->szBlkCol); + blockCol = NULL; + } + } + + if (blockCol == NULL || blockCol->cid > colData->cid) { + for (int32_t iRow = 0; iRow < hdr->nRow; iRow++) { + code = tColDataAppendValue(colData, &COL_VAL_NONE(colData->cid, colData->type)); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + ASSERT(blockCol->type == colData->type); + ASSERT(blockCol->flag && blockCol->flag != HAS_NONE); + + if (blockCol->flag == HAS_NULL) { + for (int32_t iRow = 0; iRow < hdr->nRow; iRow++) { + code = tColDataAppendValue(colData, &COL_VAL_NULL(blockCol->cid, blockCol->type)); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + int32_t size1 = blockCol->szBitmap + blockCol->szOffset + blockCol->szValue; + + code = tRealloc(&reader->config->bufArr[1], size1); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr->szBlkCol + blockCol->offset, + reader->config->bufArr[1], size1); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDecmprColData(reader->config->bufArr[1], blockCol, hdr->cmprAlg, hdr->nRow, colData, + &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk, STombBlock *tombBlock) { + int32_t code = 0; + int32_t lino = 0; + + code = tRealloc(&reader->config->bufArr[0], tombBlk->dp->size); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd, tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size); + if (code) TSDB_CHECK_CODE(code, lino, _exit); + + int64_t size = 0; + tTombBlockClear(tombBlock); + for (int32_t i = 0; i < ARRAY_SIZE(tombBlock->dataArr); ++i) { + code = tsdbDecmprData(reader->config->bufArr[0] + size, tombBlk->size[i], TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, + &reader->config->bufArr[1], sizeof(int64_t) * tombBlk->numRec, &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND_BATCH(&tombBlock->dataArr[i], reader->config->bufArr[1], tombBlk->numRec); + TSDB_CHECK_CODE(code, lino, _exit); + + size += tombBlk->size[i]; + } + + ASSERT(size == tombBlk->dp->size); +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *statisBlk, STbStatisBlock *statisBlock) { + int32_t code = 0; + int32_t lino = 0; + + code = tRealloc(&reader->config->bufArr[0], statisBlk->dp->size); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd, statisBlk->dp->offset, reader->config->bufArr[0], statisBlk->dp->size); + TSDB_CHECK_CODE(code, lino, _exit); + + int64_t size = 0; + tStatisBlockClear(statisBlock); + for (int32_t i = 0; i < ARRAY_SIZE(statisBlock->dataArr); ++i) { + code = + tsdbDecmprData(reader->config->bufArr[0] + size, statisBlk->size[i], TSDB_DATA_TYPE_BIGINT, statisBlk->cmprAlg, + &reader->config->bufArr[1], sizeof(int64_t) * statisBlk->numRec, &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND_BATCH(statisBlock->dataArr + i, reader->config->bufArr[1], statisBlk->numRec); + TSDB_CHECK_CODE(code, lino, _exit); + + size += statisBlk->size[i]; + } + + ASSERT(size == statisBlk->dp->size); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +// SSttFWriter ============================================================ +struct SSttFileWriter { + SSttFileWriterConfig config[1]; + struct { + bool opened; + TABLEID tbid[1]; + } ctx[1]; + // file + STsdbFD *fd; + STFile file[1]; + // data + SSttFooter footer[1]; + TTombBlkArray tombBlkArray[1]; + TSttBlkArray sttBlkArray[1]; + TStatisBlkArray statisBlkArray[1]; + STombBlock tombBlock[1]; + STbStatisBlock staticBlock[1]; + SBlockData blockData[1]; + // helper data + SSkmInfo skmTb[1]; + SSkmInfo skmRow[1]; + uint8_t *bufArr[5]; +}; + +int32_t tsdbFileDoWriteBlockData(STsdbFD *fd, SBlockData *blockData, int8_t cmprAlg, int64_t *fileSize, + TSttBlkArray *sttBlkArray, uint8_t **bufArr) { + if (blockData->nRow == 0) return 0; + + int32_t code = 0; + + SSttBlk sttBlk[1] = {{ + .suid = blockData->suid, + .minUid = blockData->uid ? blockData->uid : blockData->aUid[0], + .maxUid = blockData->uid ? blockData->uid : blockData->aUid[blockData->nRow - 1], + .minKey = blockData->aTSKEY[0], + .maxKey = blockData->aTSKEY[0], + .minVer = blockData->aVersion[0], + .maxVer = blockData->aVersion[0], + .nRow = blockData->nRow, + }}; + + for (int32_t iRow = 1; iRow < blockData->nRow; iRow++) { + if (sttBlk->minKey > blockData->aTSKEY[iRow]) sttBlk->minKey = blockData->aTSKEY[iRow]; + if (sttBlk->maxKey < blockData->aTSKEY[iRow]) sttBlk->maxKey = blockData->aTSKEY[iRow]; + if (sttBlk->minVer > blockData->aVersion[iRow]) sttBlk->minVer = blockData->aVersion[iRow]; + if (sttBlk->maxVer < blockData->aVersion[iRow]) sttBlk->maxVer = blockData->aVersion[iRow]; + } + + int32_t sizeArr[5] = {0}; + code = tCmprBlockData(blockData, cmprAlg, NULL, NULL, bufArr, sizeArr); + if (code) return code; + + sttBlk->bInfo.offset = *fileSize; + sttBlk->bInfo.szKey = sizeArr[2] + sizeArr[3]; + sttBlk->bInfo.szBlock = sizeArr[0] + sizeArr[1] + sttBlk->bInfo.szKey; + + for (int32_t i = 3; i >= 0; i--) { + if (sizeArr[i]) { + code = tsdbWriteFile(fd, *fileSize, bufArr[i], sizeArr[i]); + if (code) return code; + *fileSize += sizeArr[i]; + } + } + + code = TARRAY2_APPEND_PTR(sttBlkArray, sttBlk); + if (code) return code; + + tBlockDataClear(blockData); + + return 0; +} + +static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) { + if (writer->blockData->nRow == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileDoWriteBlockData(writer->fd, writer->blockData, writer->config->cmprAlg, &writer->file->size, + writer->sttBlkArray, writer->config->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) { + if (STATIS_BLOCK_SIZE(writer->staticBlock) == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + SStatisBlk statisBlk[1] = {{ + .dp[0] = + { + .offset = writer->file->size, + .size = 0, + }, + .minTbid = + { + .suid = TARRAY2_FIRST(writer->staticBlock->suid), + .uid = TARRAY2_FIRST(writer->staticBlock->uid), + }, + .maxTbid = + { + .suid = TARRAY2_LAST(writer->staticBlock->suid), + .uid = TARRAY2_LAST(writer->staticBlock->uid), + }, + .numRec = STATIS_BLOCK_SIZE(writer->staticBlock), + .cmprAlg = writer->config->cmprAlg, + }}; + + for (int32_t i = 0; i < STATIS_RECORD_NUM_ELEM; i++) { + code = tsdbCmprData((uint8_t *)TARRAY2_DATA(writer->staticBlock->dataArr + i), + TARRAY2_DATA_LEN(&writer->staticBlock->dataArr[i]), TSDB_DATA_TYPE_BIGINT, statisBlk->cmprAlg, + &writer->config->bufArr[0], 0, &statisBlk->size[i], &writer->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbWriteFile(writer->fd, writer->file->size, writer->config->bufArr[0], statisBlk->size[i]); + TSDB_CHECK_CODE(code, lino, _exit); + + statisBlk->dp->size += statisBlk->size[i]; + writer->file->size += statisBlk->size[i]; + } + + code = TARRAY2_APPEND_PTR(writer->statisBlkArray, statisBlk); + TSDB_CHECK_CODE(code, lino, _exit); + + tStatisBlockClear(writer->staticBlock); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, + TTombBlkArray *tombBlkArray, uint8_t **bufArr) { + int32_t code; + + if (TOMB_BLOCK_SIZE(tombBlock) == 0) return 0; + + STombBlk tombBlk[1] = {{ + .dp[0] = + { + .offset = *fileSize, + .size = 0, + }, + .minTbid = + { + .suid = TARRAY2_FIRST(tombBlock->suid), + .uid = TARRAY2_FIRST(tombBlock->uid), + }, + .maxTbid = + { + .suid = TARRAY2_LAST(tombBlock->suid), + .uid = TARRAY2_LAST(tombBlock->uid), + }, + .minVer = TARRAY2_FIRST(tombBlock->version), + .maxVer = TARRAY2_FIRST(tombBlock->version), + .numRec = TOMB_BLOCK_SIZE(tombBlock), + .cmprAlg = cmprAlg, + }}; + + for (int32_t i = 1; i < TOMB_BLOCK_SIZE(tombBlock); i++) { + if (tombBlk->minVer > TARRAY2_GET(tombBlock->version, i)) { + tombBlk->minVer = TARRAY2_GET(tombBlock->version, i); + } + if (tombBlk->maxVer < TARRAY2_GET(tombBlock->version, i)) { + tombBlk->maxVer = TARRAY2_GET(tombBlock->version, i); + } + } + + for (int32_t i = 0; i < ARRAY_SIZE(tombBlock->dataArr); i++) { + code = tsdbCmprData((uint8_t *)TARRAY2_DATA(&tombBlock->dataArr[i]), TARRAY2_DATA_LEN(&tombBlock->dataArr[i]), + TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, &bufArr[0], 0, &tombBlk->size[i], &bufArr[1]); + if (code) return code; + + code = tsdbWriteFile(fd, *fileSize, bufArr[0], tombBlk->size[i]); + if (code) return code; + + tombBlk->dp->size += tombBlk->size[i]; + *fileSize += tombBlk->size[i]; + } + + code = TARRAY2_APPEND_PTR(tombBlkArray, tombBlk); + if (code) return code; + + tTombBlockClear(tombBlock); + return 0; +} + +static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) { + if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size, + writer->tombBlkArray, writer->config->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize) { + ptr->size = TARRAY2_DATA_LEN(sttBlkArray); + if (ptr->size > 0) { + ptr->offset = *fileSize; + + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size); + if (code) { + return code; + } + + *fileSize += ptr->size; + } + return 0; +} + +static int32_t tsdbSttFileDoWriteSttBlk(SSttFileWriter *writer) { + int32_t code = 0; + int32_t lino; + + code = tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSttFileDoWriteStatisBlk(SSttFileWriter *writer) { + int32_t code = 0; + int32_t lino; + + writer->footer->statisBlkPtr->size = TARRAY2_DATA_LEN(writer->statisBlkArray); + if (writer->footer->statisBlkPtr->size) { + writer->footer->statisBlkPtr->offset = writer->file->size; + code = tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray), + writer->footer->statisBlkPtr->size); + TSDB_CHECK_CODE(code, lino, _exit); + writer->file->size += writer->footer->statisBlkPtr->size; + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize) { + ptr->size = TARRAY2_DATA_LEN(tombBlkArray); + if (ptr->size > 0) { + ptr->offset = *fileSize; + + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size); + if (code) { + return code; + } + + *fileSize += ptr->size; + } + return 0; +} + +static int32_t tsdbSttFileDoWriteTombBlk(SSttFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, &writer->file->size); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize) { + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer)); + if (code) return code; + *fileSize += sizeof(*footer); + return 0; +} + +static int32_t tsdbSttFileDoWriteFooter(SSttFileWriter *writer) { + return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size); +} + +static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + // set + if (!writer->config->skmTb) writer->config->skmTb = writer->skmTb; + if (!writer->config->skmRow) writer->config->skmRow = writer->skmRow; + if (!writer->config->bufArr) writer->config->bufArr = writer->bufArr; + + writer->file[0] = (STFile){ + .type = TSDB_FTYPE_STT, + .did = writer->config->did, + .fid = writer->config->fid, + .cid = writer->config->cid, + .size = 0, + .stt[0] = + { + .level = writer->config->level, + }, + }; + + // open file + int32_t flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + char fname[TSDB_FILENAME_LEN]; + + tsdbTFileName(writer->config->tsdb, writer->file, fname); + code = tsdbOpenFile(fname, writer->config->szPage, flag, &writer->fd); + TSDB_CHECK_CODE(code, lino, _exit); + + uint8_t hdr[TSDB_FHDR_SIZE] = {0}; + code = tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr)); + TSDB_CHECK_CODE(code, lino, _exit); + writer->file->size += sizeof(hdr); + + writer->ctx->opened = true; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static void tsdbSttFWriterDoClose(SSttFileWriter *writer) { + ASSERT(writer->fd == NULL); + + for (int32_t i = 0; i < ARRAY_SIZE(writer->bufArr); ++i) { + tFree(writer->bufArr[i]); + } + tDestroyTSchema(writer->skmRow->pTSchema); + tDestroyTSchema(writer->skmTb->pTSchema); + tTombBlockDestroy(writer->tombBlock); + tStatisBlockDestroy(writer->staticBlock); + tBlockDataDestroy(writer->blockData); + TARRAY2_DESTROY(writer->tombBlkArray, NULL); + TARRAY2_DESTROY(writer->statisBlkArray, NULL); + TARRAY2_DESTROY(writer->sttBlkArray, NULL); +} + +static int32_t tsdbSttFileDoUpdateHeader(SSttFileWriter *writer) { + // TODO + return 0; +} + +static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *opArray) { + int32_t lino; + int32_t code; + + code = tsdbSttFileDoWriteBlockData(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSttFileDoWriteStatisBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSttFileDoWriteTombBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSttFileDoWriteSttBlk(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSttFileDoWriteStatisBlk(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSttFileDoWriteTombBlk(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSttFileDoWriteFooter(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSttFileDoUpdateHeader(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFsyncFile(writer->fd); + TSDB_CHECK_CODE(code, lino, _exit); + + tsdbCloseFile(&writer->fd); + + ASSERT(writer->file->size > 0); + STFileOp op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = writer->config->fid, + .nf = writer->file[0], + }; + + code = TARRAY2_APPEND(opArray, op); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSttFWriterCloseAbort(SSttFileWriter *writer) { + char fname[TSDB_FILENAME_LEN]; + tsdbTFileName(writer->config->tsdb, writer->file, fname); + tsdbCloseFile(&writer->fd); + taosRemoveFile(fname); + return 0; +} + +int32_t tsdbSttFileWriterOpen(const SSttFileWriterConfig *config, SSttFileWriter **writer) { + writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); + if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->config[0] = config[0]; + writer[0]->ctx->opened = false; + return 0; +} + +int32_t tsdbSttFileWriterClose(SSttFileWriter **writer, int8_t abort, TFileOpArray *opArray) { + int32_t code = 0; + int32_t lino = 0; + + if (writer[0]->ctx->opened) { + if (abort) { + code = tsdbSttFWriterCloseAbort(writer[0]); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbSttFWriterCloseCommit(writer[0], opArray); + TSDB_CHECK_CODE(code, lino, _exit); + } + tsdbSttFWriterDoClose(writer[0]); + } + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) { + int32_t code = 0; + int32_t lino = 0; + + if (!writer->ctx->opened) { + code = tsdbSttFWriterDoOpen(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (!TABLE_SAME_SCHEMA(row->suid, row->uid, writer->ctx->tbid->suid, writer->ctx->tbid->uid)) { + code = tsdbSttFileDoWriteBlockData(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbUpdateSkmTb(writer->config->tsdb, (TABLEID *)row, writer->config->skmTb); + TSDB_CHECK_CODE(code, lino, _exit); + + TABLEID id = {.suid = row->suid, .uid = row->suid ? 0 : row->uid}; + code = tBlockDataInit(writer->blockData, &id, writer->config->skmTb->pTSchema, NULL, 0); + TSDB_CHECK_CODE(code, lino, _exit); + } + + TSDBKEY key[1]; + if (row->row.type == TSDBROW_ROW_FMT) { + key->ts = row->row.pTSRow->ts; + key->version = row->row.version; + } else { + key->ts = row->row.pBlockData->aTSKEY[row->row.iRow]; + key->version = row->row.pBlockData->aVersion[row->row.iRow]; + } + + if (writer->ctx->tbid->uid != row->uid) { + writer->ctx->tbid->suid = row->suid; + writer->ctx->tbid->uid = row->uid; + + if (STATIS_BLOCK_SIZE(writer->staticBlock) >= writer->config->maxRow) { + code = tsdbSttFileDoWriteStatisBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + STbStatisRecord record = { + .suid = row->suid, + .uid = row->uid, + .firstKey = key->ts, + .lastKey = key->ts, + .count = 1, + }; + code = tStatisBlockPut(writer->staticBlock, &record); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + ASSERT(key->ts >= TARRAY2_LAST(writer->staticBlock->lastKey)); + + if (key->ts > TARRAY2_LAST(writer->staticBlock->lastKey)) { + TARRAY2_LAST(writer->staticBlock->count)++; + TARRAY2_LAST(writer->staticBlock->lastKey) = key->ts; + } + } + + if (row->row.type == TSDBROW_ROW_FMT) { + code = tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, // + TSDBROW_SVERSION(&row->row), writer->config->skmRow); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // row to col conversion + if (key->version <= writer->config->compactVersion // + && writer->blockData->nRow > 0 // + && writer->blockData->aTSKEY[writer->blockData->nRow - 1] == key->ts // + && (writer->blockData->uid // + ? writer->blockData->uid // + : writer->blockData->aUid[writer->blockData->nRow - 1]) == row->uid // + ) { + code = tBlockDataUpdateRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + if (writer->blockData->nRow >= writer->config->maxRow) { + code = tsdbSttFileDoWriteBlockData(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tBlockDataAppendRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema, row->uid); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *bdata) { + int32_t code = 0; + int32_t lino = 0; + + SRowInfo row[1]; + row->suid = bdata->suid; + for (int32_t i = 0; i < bdata->nRow; i++) { + row->uid = bdata->uid ? bdata->uid : bdata->aUid[i]; + row->row = tsdbRowFromBlockData(bdata, i); + + code = tsdbSttFileWriteRow(writer, row); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *record) { + int32_t code; + int32_t lino; + + if (!writer->ctx->opened) { + code = tsdbSttFWriterDoOpen(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + if (writer->blockData->nRow > 0) { + code = tsdbSttFileDoWriteBlockData(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (STATIS_BLOCK_SIZE(writer->staticBlock) > 0) { + code = tsdbSttFileDoWriteStatisBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + code = tTombBlockPut(writer->tombBlock, record); + TSDB_CHECK_CODE(code, lino, _exit); + + if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) { + code = tsdbSttFileDoWriteTombBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } else { + tsdbTrace("vgId:%d write tomb record to stt file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64 + ", version:%" PRId64, + TD_VID(writer->config->tsdb->pVnode), writer->fd->path, writer->config->cid, record->suid, record->uid, + record->version); + } + return code; +} + +bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer) { return writer->ctx->opened; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h new file mode 100644 index 0000000000000000000000000000000000000000..242b55795c99be8c416c9238df2cd0356cb89004 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFS2.h" +#include "tsdbUtil2.h" + +#ifndef _TSDB_STT_FILE_RW_H +#define _TSDB_STT_FILE_RW_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef TARRAY2(SSttBlk) TSttBlkArray; +typedef TARRAY2(SStatisBlk) TStatisBlkArray; + +typedef struct { + SFDataPtr sttBlkPtr[1]; + SFDataPtr statisBlkPtr[1]; + SFDataPtr tombBlkPtr[1]; + SFDataPtr rsrvd[2]; +} SSttFooter; + +// SSttFileReader ========================================== +typedef struct SSttFileReader SSttFileReader; +typedef struct SSttFileReaderConfig SSttFileReaderConfig; +typedef TARRAY2(SSttFileReader *) TSttFileReaderArray; + +// SSttFileReader +int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *config, SSttFileReader **reader); +int32_t tsdbSttFileReaderClose(SSttFileReader **reader); + +// SSttSegReader +int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBlkArray); +int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray **statisBlkArray); +int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **delBlkArray); + +int32_t tsdbSttFileReadBlockData(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData); +int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData, + STSchema *pTSchema, int16_t cids[], int32_t ncid); +int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *statisBlk, STbStatisBlock *sData); +int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *delBlk, STombBlock *dData); + +struct SSttFileReaderConfig { + STsdb *tsdb; + int32_t szPage; + STFile file[1]; + uint8_t **bufArr; +}; + +// SSttFileWriter ========================================== +typedef struct SSttFileWriter SSttFileWriter; +typedef struct SSttFileWriterConfig SSttFileWriterConfig; + +int32_t tsdbSttFileWriterOpen(const SSttFileWriterConfig *config, SSttFileWriter **writer); +int32_t tsdbSttFileWriterClose(SSttFileWriter **writer, int8_t abort, TFileOpArray *opArray); +int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row); +int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *pBlockData); +int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *record); +bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer); + +struct SSttFileWriterConfig { + STsdb *tsdb; + int32_t maxRow; + int32_t szPage; + int8_t cmprAlg; + int64_t compactVersion; + SDiskID did; + int32_t fid; + int64_t cid; + int32_t level; + SSkmInfo *skmTb; + SSkmInfo *skmRow; + uint8_t **bufArr; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_STT_FILE_RW_H*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c new file mode 100644 index 0000000000000000000000000000000000000000..59ba51c371c9038c396d67a1799c219628032ab1 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c @@ -0,0 +1,640 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbUpgrade.h" + +// old +extern void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t); +extern int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData); + +// new +extern int32_t save_fs(const TFileSetArray *arr, const char *fname); +extern int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype); +extern int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize, + TBrinBlkArray *brinBlkArray, uint8_t **bufArr); +extern int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize); +extern int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer); +extern int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl); +extern int32_t tsdbSttLvlClear(SSttLvl **lvl); +extern int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize); +extern int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize); +extern int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, + TTombBlkArray *tombBlkArray, uint8_t **bufArr); +extern int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize); +extern int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize); + +static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *reader, STFileSet *fset) { + int32_t code = 0; + int32_t lino = 0; + + // init + struct { + // config + int32_t maxRow; + int8_t cmprAlg; + int32_t szPage; + uint8_t *bufArr[8]; + // reader + SArray *aBlockIdx; + SMapData mDataBlk[1]; + SBlockData blockData[1]; + // writer + STsdbFD *fd; + SBrinBlock brinBlock[1]; + TBrinBlkArray brinBlkArray[1]; + SHeadFooter footer[1]; + } ctx[1] = {{ + .maxRow = tsdb->pVnode->config.tsdbCfg.maxRows, + .cmprAlg = tsdb->pVnode->config.tsdbCfg.compression, + .szPage = tsdb->pVnode->config.tsdbPageSize, + }}; + + // read SBlockIdx array + if ((ctx->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbReadBlockIdx(reader, ctx->aBlockIdx); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosArrayGetSize(ctx->aBlockIdx) > 0) { + // init/open file fd + STFile file = { + .type = TSDB_FTYPE_HEAD, + .did = pDFileSet->diskId, + .fid = fset->fid, + .cid = pDFileSet->pHeadF->commitID, + .size = pDFileSet->pHeadF->size, + }; + + code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_HEAD]); + TSDB_CHECK_CODE(code, lino, _exit); + + // open fd + char fname[TSDB_FILENAME_LEN]; + tsdbTFileName(tsdb, &file, fname); + + code = tsdbOpenFile(fname, ctx->szPage, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd); + TSDB_CHECK_CODE(code, lino, _exit); + + // convert + for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(ctx->aBlockIdx); ++iBlockIdx) { + SBlockIdx *pBlockIdx = taosArrayGet(ctx->aBlockIdx, iBlockIdx); + + code = tsdbReadDataBlk(reader, pBlockIdx, ctx->mDataBlk); + TSDB_CHECK_CODE(code, lino, _exit); + + for (int32_t iDataBlk = 0; iDataBlk < ctx->mDataBlk->nItem; ++iDataBlk) { + SDataBlk dataBlk[1]; + tMapDataGetItemByIdx(ctx->mDataBlk, iDataBlk, dataBlk, tGetDataBlk); + + SBrinRecord record = { + .suid = pBlockIdx->suid, + .uid = pBlockIdx->uid, + .firstKey = dataBlk->minKey.ts, + .firstKeyVer = dataBlk->minKey.version, + .lastKey = dataBlk->maxKey.ts, + .lastKeyVer = dataBlk->maxKey.version, + .minVer = dataBlk->minVer, + .maxVer = dataBlk->maxVer, + .blockOffset = dataBlk->aSubBlock->offset, + .smaOffset = dataBlk->smaInfo.offset, + .blockSize = dataBlk->aSubBlock->szBlock, + .blockKeySize = dataBlk->aSubBlock->szKey, + .smaSize = dataBlk->smaInfo.size, + .numRow = dataBlk->nRow, + .count = dataBlk->nRow, + }; + + if (dataBlk->hasDup) { + code = tsdbReadDataBlockEx(reader, dataBlk, ctx->blockData); + TSDB_CHECK_CODE(code, lino, _exit); + + record.count = 1; + for (int32_t i = 1; i < ctx->blockData->nRow; ++i) { + if (ctx->blockData->aTSKEY[i] != ctx->blockData->aTSKEY[i - 1]) { + record.count++; + } + } + } + + code = tBrinBlockPut(ctx->brinBlock, &record); + TSDB_CHECK_CODE(code, lino, _exit); + + if (BRIN_BLOCK_SIZE(ctx->brinBlock) >= ctx->maxRow) { + code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, + ctx->brinBlkArray, ctx->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } + + if (BRIN_BLOCK_SIZE(ctx->brinBlock) > 0) { + code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, + ctx->brinBlkArray, ctx->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbFileWriteBrinBlk(ctx->fd, ctx->brinBlkArray, ctx->footer->brinBlkPtr, + &fset->farr[TSDB_FTYPE_HEAD]->f->size); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFsyncFile(ctx->fd); + TSDB_CHECK_CODE(code, lino, _exit); + + tsdbCloseFile(&ctx->fd); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + TARRAY2_DESTROY(ctx->brinBlkArray, NULL); + tBrinBlockDestroy(ctx->brinBlock); + tBlockDataDestroy(ctx->blockData); + tMapDataClear(ctx->mDataBlk); + taosArrayDestroy(ctx->aBlockIdx); + for (int32_t i = 0; i < ARRAY_SIZE(ctx->bufArr); ++i) { + tFree(ctx->bufArr[i]); + } + return code; +} + +static int32_t tsdbUpgradeData(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *reader, STFileSet *fset) { + int32_t code = 0; + int32_t lino = 0; + + if (fset->farr[TSDB_FTYPE_HEAD] == NULL) { + return 0; + } + + STFile file = { + .type = TSDB_FTYPE_DATA, + .did = pDFileSet->diskId, + .fid = fset->fid, + .cid = pDFileSet->pDataF->commitID, + .size = pDFileSet->pDataF->size, + }; + + code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_DATA]); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbUpgradeSma(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *reader, STFileSet *fset) { + int32_t code = 0; + int32_t lino = 0; + + if (fset->farr[TSDB_FTYPE_HEAD] == NULL) { + return 0; + } + + STFile file = { + .type = TSDB_FTYPE_SMA, + .did = pDFileSet->diskId, + .fid = fset->fid, + .cid = pDFileSet->pSmaF->commitID, + .size = pDFileSet->pSmaF->size, + }; + + code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_SMA]); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *reader, STFileSet *fset, + int32_t iStt, SSttLvl *lvl) { + int32_t code = 0; + int32_t lino = 0; + + SArray *aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); + if (aSttBlk == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbReadSttBlk(reader, iStt, aSttBlk); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosArrayGetSize(aSttBlk) > 0) { + SSttFile *pSttF = pDFileSet->aSttF[iStt]; + STFileObj *fobj; + struct { + int32_t szPage; + // writer + STsdbFD *fd; + TSttBlkArray sttBlkArray[1]; + SSttFooter footer[1]; + } ctx[1] = {{ + .szPage = tsdb->pVnode->config.tsdbPageSize, + }}; + + STFile file = { + .type = TSDB_FTYPE_STT, + .did = pDFileSet->diskId, + .fid = fset->fid, + .cid = pSttF->commitID, + .size = pSttF->size, + }; + code = tsdbTFileObjInit(tsdb, &file, &fobj); + TSDB_CHECK_CODE(code, lino, _exit1); + + code = tsdbOpenFile(fobj->fname, ctx->szPage, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd); + TSDB_CHECK_CODE(code, lino, _exit1); + + for (int32_t iSttBlk = 0; iSttBlk < taosArrayGetSize(aSttBlk); iSttBlk++) { + code = TARRAY2_APPEND_PTR(ctx->sttBlkArray, (SSttBlk *)taosArrayGet(aSttBlk, iSttBlk)); + TSDB_CHECK_CODE(code, lino, _exit1); + } + + code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size); + TSDB_CHECK_CODE(code, lino, _exit1); + + code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size); + TSDB_CHECK_CODE(code, lino, _exit1); + + code = tsdbFsyncFile(ctx->fd); + TSDB_CHECK_CODE(code, lino, _exit1); + + tsdbCloseFile(&ctx->fd); + + code = TARRAY2_APPEND(lvl->fobjArr, fobj); + TSDB_CHECK_CODE(code, lino, _exit1); + + _exit1: + TARRAY2_DESTROY(ctx->sttBlkArray, NULL); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + taosArrayDestroy(aSttBlk); + return code; +} + +static int32_t tsdbUpgradeStt(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *reader, STFileSet *fset) { + int32_t code = 0; + int32_t lino = 0; + + if (pDFileSet->nSttF == 0) { + return 0; + } + + SSttLvl *lvl; + code = tsdbSttLvlInit(0, &lvl); + TSDB_CHECK_CODE(code, lino, _exit); + + for (int32_t iStt = 0; iStt < pDFileSet->nSttF; ++iStt) { + code = tsdbUpgradeSttFile(tsdb, pDFileSet, reader, fset, iStt, lvl); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (TARRAY2_SIZE(lvl->fobjArr) > 0) { + code = TARRAY2_APPEND(fset->lvlArr, lvl); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + tsdbSttLvlClear(&lvl); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbUpgradeFileSet(STsdb *tsdb, SDFileSet *pDFileSet, TFileSetArray *fileSetArray) { + int32_t code = 0; + int32_t lino = 0; + + SDataFReader *reader; + STFileSet *fset; + + code = tsdbTFileSetInit(pDFileSet->fid, &fset); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFReaderOpen(&reader, tsdb, pDFileSet); + TSDB_CHECK_CODE(code, lino, _exit); + + // .head + code = tsdbUpgradeHead(tsdb, pDFileSet, reader, fset); + TSDB_CHECK_CODE(code, lino, _exit); + + // .data + code = tsdbUpgradeData(tsdb, pDFileSet, reader, fset); + TSDB_CHECK_CODE(code, lino, _exit); + + // .sma + code = tsdbUpgradeSma(tsdb, pDFileSet, reader, fset); + TSDB_CHECK_CODE(code, lino, _exit); + + // .stt + if (pDFileSet->nSttF > 0) { + code = tsdbUpgradeStt(tsdb, pDFileSet, reader, fset); + TSDB_CHECK_CODE(code, lino, _exit); + } + + tsdbDataFReaderClose(&reader); + + code = TARRAY2_APPEND(fileSetArray, fset); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **fd, STFileObj **fobj, bool *toStt) { + int32_t code = 0; + int32_t lino = 0; + + if (TARRAY2_SIZE(fset->lvlArr) == 0) { // to .tomb file + *toStt = false; + + STFile file = { + .type = TSDB_FTYPE_TOMB, + .did = fset->farr[TSDB_FTYPE_HEAD]->f->did, + .fid = fset->fid, + .cid = 0, + .size = 0, + }; + + code = tsdbTFileObjInit(tsdb, &file, fobj); + TSDB_CHECK_CODE(code, lino, _exit); + + fset->farr[TSDB_FTYPE_TOMB] = *fobj; + } else { // to .stt file + *toStt = true; + SSttLvl *lvl = TARRAY2_GET(fset->lvlArr, 0); + + STFile file = { + .type = TSDB_FTYPE_STT, + .did = TARRAY2_GET(lvl->fobjArr, 0)->f->did, + .fid = fset->fid, + .cid = 0, + .size = 0, + }; + + code = tsdbTFileObjInit(tsdb, &file, fobj); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(lvl->fobjArr, fobj[0]); + TSDB_CHECK_CODE(code, lino, _exit); + } + + char fname[TSDB_FILENAME_LEN] = {0}; + code = tsdbOpenFile(fobj[0]->fname, tsdb->pVnode->config.tsdbPageSize, + TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd); + TSDB_CHECK_CODE(code, lino, _exit); + + uint8_t hdr[TSDB_FHDR_SIZE] = {0}; + code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE); + TSDB_CHECK_CODE(code, lino, _exit); + fobj[0]->f->size += TSDB_FHDR_SIZE; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *aDelIdx, STFileSet *fset) { + int32_t code = 0; + int32_t lino = 0; + + struct { + // context + bool toStt; + int8_t cmprAlg; + int32_t maxRow; + int64_t minKey; + int64_t maxKey; + uint8_t *bufArr[8]; + // reader + SArray *aDelData; + // writer + STsdbFD *fd; + STFileObj *fobj; + STombBlock tombBlock[1]; + TTombBlkArray tombBlkArray[1]; + STombFooter tombFooter[1]; + SSttFooter sttFooter[1]; + } ctx[1] = {{ + .maxRow = tsdb->pVnode->config.tsdbCfg.maxRows, + .cmprAlg = tsdb->pVnode->config.tsdbCfg.compression, + }}; + + tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &ctx->minKey, &ctx->maxKey); + + if ((ctx->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { + SDelIdx *pDelIdx = (SDelIdx *)taosArrayGet(aDelIdx, iDelIdx); + + code = tsdbReadDelData(reader, pDelIdx, ctx->aDelData); + TSDB_CHECK_CODE(code, lino, _exit); + + for (int32_t iDelData = 0; iDelData < taosArrayGetSize(ctx->aDelData); iDelData++) { + SDelData *pDelData = (SDelData *)taosArrayGet(ctx->aDelData, iDelData); + + STombRecord record = { + .suid = pDelIdx->suid, + .uid = pDelIdx->uid, + .version = pDelData->version, + .skey = pDelData->sKey, + .ekey = pDelData->eKey, + }; + + code = tTombBlockPut(ctx->tombBlock, &record); + TSDB_CHECK_CODE(code, lino, _exit); + + if (TOMB_BLOCK_SIZE(ctx->tombBlock) > ctx->maxRow) { + if (ctx->fd == NULL) { + code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt); + TSDB_CHECK_CODE(code, lino, _exit); + } + code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, + ctx->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } + + if (TOMB_BLOCK_SIZE(ctx->tombBlock) > 0) { + if (ctx->fd == NULL) { + code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt); + TSDB_CHECK_CODE(code, lino, _exit); + } + code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, + ctx->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (ctx->fd != NULL) { + if (ctx->toStt) { + code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->sttFooter->tombBlkPtr, &ctx->fobj->f->size); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFileWriteTombFooter(ctx->fd, ctx->tombFooter, &ctx->fobj->f->size); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbFsyncFile(ctx->fd); + TSDB_CHECK_CODE(code, lino, _exit); + + tsdbCloseFile(&ctx->fd); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + for (int32_t i = 0; i < ARRAY_SIZE(ctx->bufArr); i++) { + tFree(ctx->bufArr[i]); + } + TARRAY2_DESTROY(ctx->tombBlkArray, NULL); + tTombBlockDestroy(ctx->tombBlock); + taosArrayDestroy(ctx->aDelData); + return code; +} + +static int32_t tsdbUpgradeTombFile(STsdb *tsdb, SDelFile *pDelFile, TFileSetArray *fileSetArray) { + int32_t code = 0; + int32_t lino = 0; + + SDelFReader *reader = NULL; + SArray *aDelIdx = NULL; + + if ((aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDelFReaderOpen(&reader, pDelFile, tsdb); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadDelIdx(reader, aDelIdx); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosArrayGetSize(aDelIdx) > 0) { + STFileSet *fset; + TARRAY2_FOREACH(fileSetArray, fset) { + code = tsdbDumpTombDataToFSet(tsdb, reader, aDelIdx, fset); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + tsdbDelFReaderClose(&reader); + taosArrayDestroy(aDelIdx); + return code; +} + +static int32_t tsdbDoUpgradeFileSystem(STsdb *tsdb, TFileSetArray *fileSetArray) { + int32_t code = 0; + int32_t lino = 0; + + // upgrade each file set + for (int32_t i = 0; i < taosArrayGetSize(tsdb->fs.aDFileSet); i++) { + code = tsdbUpgradeFileSet(tsdb, taosArrayGet(tsdb->fs.aDFileSet, i), fileSetArray); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // upgrade tomb file + if (tsdb->fs.pDelFile != NULL) { + code = tsdbUpgradeTombFile(tsdb, tsdb->fs.pDelFile, fileSetArray); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbUpgradeFileSystem(STsdb *tsdb, int8_t rollback) { + int32_t code = 0; + int32_t lino = 0; + + TFileSetArray fileSetArray[1] = {0}; + + // open old file system + code = tsdbFSOpen(tsdb, rollback); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDoUpgradeFileSystem(tsdb, fileSetArray); + TSDB_CHECK_CODE(code, lino, _exit); + + // close file system + code = tsdbFSClose(tsdb); + TSDB_CHECK_CODE(code, lino, _exit); + + // save new file system + char fname[TSDB_FILENAME_LEN]; + current_fname(tsdb, fname, TSDB_FCURRENT); + code = save_fs(fileSetArray, fname); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + TARRAY2_DESTROY(fileSetArray, tsdbTFileSetClear); + return code; +} + +int32_t tsdbCheckAndUpgradeFileSystem(STsdb *tsdb, int8_t rollback) { + char fname[TSDB_FILENAME_LEN]; + + tsdbGetCurrentFName(tsdb, fname, NULL); + if (!taosCheckExistFile(fname)) return 0; + + int32_t code = tsdbUpgradeFileSystem(tsdb, rollback); + if (code) return code; + + taosRemoveFile(fname); + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbUpgrade.h b/source/dnode/vnode/src/tsdb/tsdbUpgrade.h new file mode 100644 index 0000000000000000000000000000000000000000..f9aac94e00026bd833f27df5227d7d351664ff63 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbUpgrade.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" +#include "tsdbDataFileRW.h" +#include "tsdbDef.h" +#include "tsdbFS2.h" +#include "tsdbUtil2.h" + +#ifndef _TSDB_UPGRADE_H_ +#define _TSDB_UPGRADE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t tsdbCheckAndUpgradeFileSystem(STsdb *tsdb, int8_t rollback); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_UPGRADE_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 556ec335266e1019470555a0655ce6d6977a7253..9340b24d747e05cc3840a627ae9724c41e3f203e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -528,25 +528,27 @@ void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minK *maxKey = *minKey + tsTickPerMin[precision] * minutes - 1; } -int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now) { +int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t nowSec) { int32_t aFid[3]; TSKEY key; if (pKeepCfg->precision == TSDB_TIME_PRECISION_MILLI) { - now = now * 1000; + nowSec = nowSec * 1000; } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_MICRO) { - now = now * 1000000l; + nowSec = nowSec * 1000000l; } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) { - now = now * 1000000000l; + nowSec = nowSec * 1000000000l; } else { ASSERT(0); } - key = now - pKeepCfg->keep0 * tsTickPerMin[pKeepCfg->precision]; + nowSec = nowSec - tsKeepTimeOffset * tsTickPerHour[pKeepCfg->precision]; + + key = nowSec - pKeepCfg->keep0 * tsTickPerMin[pKeepCfg->precision]; aFid[0] = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision); - key = now - pKeepCfg->keep1 * tsTickPerMin[pKeepCfg->precision]; + key = nowSec - pKeepCfg->keep1 * tsTickPerMin[pKeepCfg->precision]; aFid[1] = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision); - key = now - pKeepCfg->keep2 * tsTickPerMin[pKeepCfg->precision]; + key = nowSec - pKeepCfg->keep2 * tsTickPerMin[pKeepCfg->precision]; aFid[2] = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision); if (fid >= aFid[0]) { @@ -640,7 +642,7 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); - SColVal * pColVal = &(SColVal){0}; + SColVal *pColVal = &(SColVal){0}; STColumn *pTColumn; int32_t iCol, jCol = 1; @@ -764,7 +766,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) } } -int32_t tsdbRowMergerInit(SRowMerger* pMerger, STSchema *pSchema) { +int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pSchema) { pMerger->pTSchema = pSchema; pMerger->pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal)); if (pMerger->pArray == NULL) { @@ -774,7 +776,7 @@ int32_t tsdbRowMergerInit(SRowMerger* pMerger, STSchema *pSchema) { } } -void tsdbRowMergerClear(SRowMerger* pMerger) { +void tsdbRowMergerClear(SRowMerger *pMerger) { for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); if (IS_VAR_DATA_TYPE(pTColVal->type)) { @@ -785,7 +787,7 @@ void tsdbRowMergerClear(SRowMerger* pMerger) { taosArrayClear(pMerger->pArray); } -void tsdbRowMergerCleanup(SRowMerger* pMerger) { +void tsdbRowMergerCleanup(SRowMerger *pMerger) { int32_t numOfCols = taosArrayGetSize(pMerger->pArray); for (int32_t iCol = 1; iCol < numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); @@ -1041,8 +1043,6 @@ int32_t tsdbBuildDeleteSkyline2(SArray *aDelData, int32_t sidx, int32_t eidx, SA // SBlockData ====================================================== int32_t tBlockDataCreate(SBlockData *pBlockData) { - int32_t code = 0; - pBlockData->suid = 0; pBlockData->uid = 0; pBlockData->nRow = 0; @@ -1051,7 +1051,7 @@ int32_t tBlockDataCreate(SBlockData *pBlockData) { pBlockData->aTSKEY = NULL; pBlockData->nColData = 0; pBlockData->aColData = NULL; - return code; + return 0; } void tBlockDataDestroy(SBlockData *pBlockData) { @@ -1107,8 +1107,8 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int32_t iColumn = 1; STColumn *pTColumn = &pTSchema->columns[iColumn]; for (int32_t iCid = 0; iCid < nCid; iCid++) { - - // aCid array (from taos client catalog) contains columns that does not exist in the pTSchema. the pTSchema is newer + // aCid array (from taos client catalog) contains columns that does not exist in the pTSchema. the pTSchema is + // newer if (pTColumn == NULL) { continue; } @@ -1239,7 +1239,7 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS _exit: return code; } -static int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; // version diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil2.c b/source/dnode/vnode/src/tsdb/tsdbUtil2.c new file mode 100644 index 0000000000000000000000000000000000000000..e938caa1184f34c366dec638b4a5dd9db9eaafb6 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbUtil2.c @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbUtil2.h" + +// SDelBlock ---------- +int32_t tTombBlockInit(STombBlock *tombBlock) { + for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { + TARRAY2_INIT(&tombBlock->dataArr[i]); + } + return 0; +} + +int32_t tTombBlockDestroy(STombBlock *tombBlock) { + for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { + TARRAY2_DESTROY(&tombBlock->dataArr[i], NULL); + } + return 0; +} + +int32_t tTombBlockClear(STombBlock *tombBlock) { + for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { + TARRAY2_CLEAR(&tombBlock->dataArr[i], NULL); + } + return 0; +} + +int32_t tTombBlockPut(STombBlock *tombBlock, const STombRecord *record) { + int32_t code; + for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { + code = TARRAY2_APPEND(&tombBlock->dataArr[i], record->dataArr[i]); + if (code) return code; + } + return 0; +} + +int32_t tTombBlockGet(STombBlock *tombBlock, int32_t idx, STombRecord *record) { + if (idx >= TOMB_BLOCK_SIZE(tombBlock)) return TSDB_CODE_OUT_OF_RANGE; + for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { + record->dataArr[i] = TARRAY2_GET(&tombBlock->dataArr[i], idx); + } + return 0; +} + +int32_t tTombRecordCompare(const STombRecord *r1, const STombRecord *r2) { + if (r1->suid < r2->suid) return -1; + if (r1->suid > r2->suid) return 1; + if (r1->uid < r2->uid) return -1; + if (r1->uid > r2->uid) return 1; + if (r1->version < r2->version) return -1; + if (r1->version > r2->version) return 1; + return 0; +} + +// STbStatisBlock ---------- +int32_t tStatisBlockInit(STbStatisBlock *statisBlock) { + for (int32_t i = 0; i < STATIS_RECORD_NUM_ELEM; ++i) { + TARRAY2_INIT(&statisBlock->dataArr[i]); + } + return 0; +} + +int32_t tStatisBlockDestroy(STbStatisBlock *statisBlock) { + for (int32_t i = 0; i < STATIS_RECORD_NUM_ELEM; ++i) { + TARRAY2_DESTROY(&statisBlock->dataArr[i], NULL); + } + return 0; +} + +int32_t tStatisBlockClear(STbStatisBlock *statisBlock) { + for (int32_t i = 0; i < STATIS_RECORD_NUM_ELEM; ++i) { + TARRAY2_CLEAR(&statisBlock->dataArr[i], NULL); + } + return 0; +} + +int32_t tStatisBlockPut(STbStatisBlock *statisBlock, const STbStatisRecord *record) { + int32_t code; + for (int32_t i = 0; i < STATIS_RECORD_NUM_ELEM; ++i) { + code = TARRAY2_APPEND(&statisBlock->dataArr[i], record->dataArr[i]); + if (code) return code; + } + return 0; +} + +int32_t tStatisBlockGet(STbStatisBlock *statisBlock, int32_t idx, STbStatisRecord *record) { + if (idx >= STATIS_BLOCK_SIZE(statisBlock)) return TSDB_CODE_OUT_OF_RANGE; + for (int32_t i = 0; i < STATIS_RECORD_NUM_ELEM; ++i) { + record->dataArr[i] = TARRAY2_GET(&statisBlock->dataArr[i], idx); + } + return 0; +} + +// SBrinRecord ---------- +int32_t tBrinBlockInit(SBrinBlock *brinBlock) { + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); ++i) { + TARRAY2_INIT(&brinBlock->dataArr1[i]); + } + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr2); ++i) { + TARRAY2_INIT(&brinBlock->dataArr2[i]); + } + return 0; +} + +int32_t tBrinBlockDestroy(SBrinBlock *brinBlock) { + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); ++i) { + TARRAY2_DESTROY(&brinBlock->dataArr1[i], NULL); + } + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr2); ++i) { + TARRAY2_DESTROY(&brinBlock->dataArr2[i], NULL); + } + return 0; +} + +int32_t tBrinBlockClear(SBrinBlock *brinBlock) { + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); ++i) { + TARRAY2_CLEAR(&brinBlock->dataArr1[i], NULL); + } + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr2); ++i) { + TARRAY2_CLEAR(&brinBlock->dataArr2[i], NULL); + } + return 0; +} + +int32_t tBrinBlockPut(SBrinBlock *brinBlock, const SBrinRecord *record) { + int32_t code; + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); ++i) { + code = TARRAY2_APPEND(&brinBlock->dataArr1[i], record->dataArr1[i]); + if (code) return code; + } + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr2); ++i) { + code = TARRAY2_APPEND(&brinBlock->dataArr2[i], record->dataArr2[i]); + if (code) return code; + } + return 0; +} + +int32_t tBrinBlockGet(SBrinBlock *brinBlock, int32_t idx, SBrinRecord *record) { + if (idx >= BRIN_BLOCK_SIZE(brinBlock)) return TSDB_CODE_OUT_OF_RANGE; + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); ++i) { + record->dataArr1[i] = TARRAY2_GET(&brinBlock->dataArr1[i], idx); + } + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr2); ++i) { + record->dataArr2[i] = TARRAY2_GET(&brinBlock->dataArr2[i], idx); + } + return 0; +} + +// other apis ---------- +int32_t tsdbUpdateSkmTb(STsdb *pTsdb, const TABLEID *tbid, SSkmInfo *pSkmTb) { + if (tbid->suid) { + if (pSkmTb->suid == tbid->suid) { + pSkmTb->uid = tbid->uid; + return 0; + } + } else if (pSkmTb->uid == tbid->uid) { + return 0; + } + + pSkmTb->suid = tbid->suid; + pSkmTb->uid = tbid->uid; + tDestroyTSchema(pSkmTb->pTSchema); + return metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, tbid->suid, tbid->uid, -1, &pSkmTb->pTSchema); +} + +int32_t tsdbUpdateSkmRow(STsdb *pTsdb, const TABLEID *tbid, int32_t sver, SSkmInfo *pSkmRow) { + if (pSkmRow->pTSchema && pSkmRow->suid == tbid->suid) { + if (pSkmRow->suid) { + if (sver == pSkmRow->pTSchema->version) return 0; + } else if (pSkmRow->uid == tbid->uid && pSkmRow->pTSchema->version == sver) { + return 0; + } + } + + pSkmRow->suid = tbid->suid; + pSkmRow->uid = tbid->uid; + tDestroyTSchema(pSkmRow->pTSchema); + return metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, tbid->suid, tbid->uid, sver, &pSkmRow->pTSchema); +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil2.h b/source/dnode/vnode/src/tsdb/tsdbUtil2.h new file mode 100644 index 0000000000000000000000000000000000000000..fa0636834155df5d8f8ddfbb3988d6a5a7d86bbf --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbUtil2.h @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TSDB_UTIL_H +#define _TSDB_UTIL_H + +#include "tsdbDef.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// STombRecord ---------- +#define TOMB_RECORD_ELEM_NUM 5 +typedef union { + int64_t dataArr[TOMB_RECORD_ELEM_NUM]; + struct { + int64_t suid; + int64_t uid; + int64_t version; + int64_t skey; + int64_t ekey; + }; +} STombRecord; + +typedef union { + TARRAY2(int64_t) dataArr[TOMB_RECORD_ELEM_NUM]; + struct { + TARRAY2(int64_t) suid[1]; + TARRAY2(int64_t) uid[1]; + TARRAY2(int64_t) version[1]; + TARRAY2(int64_t) skey[1]; + TARRAY2(int64_t) ekey[1]; + }; +} STombBlock; + +typedef struct { + SFDataPtr dp[1]; + TABLEID minTbid; + TABLEID maxTbid; + int64_t minVer; + int64_t maxVer; + int32_t numRec; + int32_t size[TOMB_RECORD_ELEM_NUM]; + int8_t cmprAlg; + int8_t rsvd[7]; +} STombBlk; + +typedef TARRAY2(STombBlk) TTombBlkArray; + +#define TOMB_BLOCK_SIZE(db) TARRAY2_SIZE((db)->suid) + +int32_t tTombBlockInit(STombBlock *tombBlock); +int32_t tTombBlockDestroy(STombBlock *tombBlock); +int32_t tTombBlockClear(STombBlock *tombBlock); +int32_t tTombBlockPut(STombBlock *tombBlock, const STombRecord *record); +int32_t tTombBlockGet(STombBlock *tombBlock, int32_t idx, STombRecord *record); +int32_t tTombRecordCompare(const STombRecord *record1, const STombRecord *record2); + +// STbStatisRecord ---------- +#define STATIS_RECORD_NUM_ELEM 5 +typedef union { + int64_t dataArr[STATIS_RECORD_NUM_ELEM]; + struct { + int64_t suid; + int64_t uid; + int64_t firstKey; + int64_t lastKey; + int64_t count; + }; +} STbStatisRecord; + +typedef union { + TARRAY2(int64_t) dataArr[STATIS_RECORD_NUM_ELEM]; + struct { + TARRAY2(int64_t) suid[1]; + TARRAY2(int64_t) uid[1]; + TARRAY2(int64_t) firstKey[1]; + TARRAY2(int64_t) lastKey[1]; + TARRAY2(int64_t) count[1]; + }; +} STbStatisBlock; + +typedef struct { + SFDataPtr dp[1]; + TABLEID minTbid; + TABLEID maxTbid; + int32_t numRec; + int32_t size[STATIS_RECORD_NUM_ELEM]; + int8_t cmprAlg; + int8_t rsvd[7]; +} SStatisBlk; + +#define STATIS_BLOCK_SIZE(db) TARRAY2_SIZE((db)->suid) + +int32_t tStatisBlockInit(STbStatisBlock *statisBlock); +int32_t tStatisBlockDestroy(STbStatisBlock *statisBlock); +int32_t tStatisBlockClear(STbStatisBlock *statisBlock); +int32_t tStatisBlockPut(STbStatisBlock *statisBlock, const STbStatisRecord *record); +int32_t tStatisBlockGet(STbStatisBlock *statisBlock, int32_t idx, STbStatisRecord *record); + +// SBrinRecord ---------- +typedef union { + struct { + int64_t dataArr1[10]; + int32_t dataArr2[5]; + }; + struct { + int64_t suid; + int64_t uid; + int64_t firstKey; + int64_t firstKeyVer; + int64_t lastKey; + int64_t lastKeyVer; + int64_t minVer; + int64_t maxVer; + int64_t blockOffset; + int64_t smaOffset; + int32_t blockSize; + int32_t blockKeySize; + int32_t smaSize; + int32_t numRow; + int32_t count; + }; +} SBrinRecord; + +typedef union { + struct { + TARRAY2(int64_t) dataArr1[10]; + TARRAY2(int32_t) dataArr2[5]; + }; + struct { + TARRAY2(int64_t) suid[1]; + TARRAY2(int64_t) uid[1]; + TARRAY2(int64_t) firstKey[1]; + TARRAY2(int64_t) firstKeyVer[1]; + TARRAY2(int64_t) lastKey[1]; + TARRAY2(int64_t) lastKeyVer[1]; + TARRAY2(int64_t) minVer[1]; + TARRAY2(int64_t) maxVer[1]; + TARRAY2(int64_t) blockOffset[1]; + TARRAY2(int64_t) smaOffset[1]; + TARRAY2(int32_t) blockSize[1]; + TARRAY2(int32_t) blockKeySize[1]; + TARRAY2(int32_t) smaSize[1]; + TARRAY2(int32_t) numRow[1]; + TARRAY2(int32_t) count[1]; + }; +} SBrinBlock; + +typedef struct { + SFDataPtr dp[1]; + TABLEID minTbid; + TABLEID maxTbid; + int64_t minVer; + int64_t maxVer; + int32_t numRec; + int32_t size[15]; + int8_t cmprAlg; + int8_t rsvd[7]; +} SBrinBlk; + +typedef TARRAY2(SBrinBlk) TBrinBlkArray; + +#define BRIN_BLOCK_SIZE(db) TARRAY2_SIZE((db)->suid) + +int32_t tBrinBlockInit(SBrinBlock *brinBlock); +int32_t tBrinBlockDestroy(SBrinBlock *brinBlock); +int32_t tBrinBlockClear(SBrinBlock *brinBlock); +int32_t tBrinBlockPut(SBrinBlock *brinBlock, const SBrinRecord *record); +int32_t tBrinBlockGet(SBrinBlock *brinBlock, int32_t idx, SBrinRecord *record); + +// other apis +int32_t tsdbUpdateSkmTb(STsdb *pTsdb, const TABLEID *tbid, SSkmInfo *pSkmTb); +int32_t tsdbUpdateSkmRow(STsdb *pTsdb, const TABLEID *tbid, int32_t sver, SSkmInfo *pSkmRow); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_UTIL_H*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index faa4d2fc5770917b5b2ced38725de403f290a711..2e161d728ffbb9d11ed794bebdff8affcdc6f429 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -49,7 +49,7 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1, .hashBegin = 0, .hashEnd = 0, .hashMethod = 0, - .sttTrigger = TSDB_DEFAULT_STT_FILE, + .sttTrigger = TSDB_DEFAULT_SST_TRIGGER, .tsdbPageSize = TSDB_DEFAULT_PAGE_SIZE}; int vnodeCheckCfg(const SVnodeCfg *pCfg) { @@ -57,7 +57,7 @@ int vnodeCheckCfg(const SVnodeCfg *pCfg) { return 0; } -const char* vnodeRoleToStr(ESyncRole role) { +const char *vnodeRoleToStr(ESyncRole role) { switch (role) { case TAOS_SYNC_ROLE_VOTER: return "true"; @@ -68,11 +68,11 @@ const char* vnodeRoleToStr(ESyncRole role) { } } -const ESyncRole vnodeStrToRole(char* str) { - if(strcmp(str, "true") == 0){ +const ESyncRole vnodeStrToRole(char *str) { + if (strcmp(str, "true") == 0) { return TAOS_SYNC_ROLE_VOTER; } - if(strcmp(str, "false") == 0){ + if (strcmp(str, "false") == 0) { return TAOS_SYNC_ROLE_LEARNER; } @@ -295,10 +295,9 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { char role[10] = {0}; code = tjsonGetStringValue(info, "isReplica", role); if (code < 0) return -1; - if(strlen(role) != 0){ + if (strlen(role) != 0) { pNode->nodeRole = vnodeStrToRole(role); - } - else{ + } else { pNode->nodeRole = TAOS_SYNC_ROLE_VOTER; } vDebug("vgId:%d, decode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, @@ -325,7 +324,7 @@ int vnodeValidateTableHash(SVnode *pVnode, char *tableFName) { if (hashValue < pVnode->config.hashBegin || hashValue > pVnode->config.hashEnd) { terrno = TSDB_CODE_VND_HASH_MISMATCH; - return TSDB_CODE_VND_HASH_MISMATCH; + return -1; } return 0; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 74168591d21ef151b0885e15076dc0842b7ab445..33c6f9d533099a57ebc0ea29e744f832c3d24734 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -16,6 +16,11 @@ #include "vnd.h" #include "vnodeInt.h" +extern int32_t tsdbPreCommit(STsdb *pTsdb); +extern int32_t tsdbCommitBegin(STsdb *pTsdb, SCommitInfo *pInfo); +extern int32_t tsdbCommitCommit(STsdb *pTsdb); +extern int32_t tsdbCommitAbort(STsdb *pTsdb); + #define VND_INFO_FNAME_TMP "vnode_tmp.json" static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData); @@ -290,11 +295,7 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { pInfo->txn = metaGetTxn(pVnode->pMeta); // save info - if (pVnode->pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); - } + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN); vDebug("vgId:%d, save config while prepare commit", TD_VID(pVnode)); if (vnodeSaveInfo(dir, &pInfo->info) < 0) { @@ -302,7 +303,7 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { TSDB_CHECK_CODE(code, lino, _exit); } - tsdbPrepareCommit(pVnode->pTsdb); + tsdbPreCommit(pVnode->pTsdb); metaPrepareAsyncCommit(pVnode->pMeta); @@ -360,7 +361,12 @@ static int32_t vnodeCommitTask(void *arg) { // commit code = vnodeCommitImpl(pInfo); - if (code) goto _exit; + if (code) { + vFatal("vgId:%d, failed to commit vnode since %s", TD_VID(pVnode), terrstr()); + taosMsleep(100); + exit(EXIT_FAILURE); + goto _exit; + } vnodeReturnBufPool(pVnode); @@ -427,20 +433,17 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { return -1; } - if (pVnode->pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); - } + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN); syncBeginSnapshot(pVnode->sync, pInfo->info.state.committed); - // commit each sub-system - code = tsdbCommit(pVnode->pTsdb, pInfo); + code = tsdbCommitBegin(pVnode->pTsdb, pInfo); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbCacheCommit(pVnode->pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); + if (!TSDB_CACHE_NO(pVnode->config)) { + code = tsdbCacheCommit(pVnode->pTsdb); + TSDB_CHECK_CODE(code, lino, _exit); + } if (VND_IS_RSMA(pVnode)) { code = smaCommit(pVnode->pSma, pInfo); @@ -458,7 +461,7 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { TSDB_CHECK_CODE(code, lino, _exit); } - code = tsdbFinishCommit(pVnode->pTsdb); + code = tsdbCommitCommit(pVnode->pTsdb); TSDB_CHECK_CODE(code, lino, _exit); if (VND_IS_RSMA(pVnode)) { @@ -491,16 +494,22 @@ _exit: bool vnodeShouldRollback(SVnode *pVnode) { char tFName[TSDB_FILENAME_LEN] = {0}; - snprintf(tFName, TSDB_FILENAME_LEN, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, - VND_INFO_FNAME_TMP); + int32_t offset = 0; + + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, tFName, TSDB_FILENAME_LEN); + offset = strlen(tFName); + snprintf(tFName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VND_INFO_FNAME_TMP); return taosCheckExistFile(tFName); } void vnodeRollback(SVnode *pVnode) { char tFName[TSDB_FILENAME_LEN] = {0}; - snprintf(tFName, TSDB_FILENAME_LEN, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, - VND_INFO_FNAME_TMP); + int32_t offset = 0; + + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, tFName, TSDB_FILENAME_LEN); + offset = strlen(tFName); + snprintf(tFName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VND_INFO_FNAME_TMP); (void)taosRemoveFile(tFName); } diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c new file mode 100644 index 0000000000000000000000000000000000000000..5c8d563d73f2d98567b1118b676eb607c5937bf9 --- /dev/null +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "meta.h" +#include "storageapi.h" +#include "tstreamUpdate.h" +#include "vnodeInt.h" + +static void initTsdbReaderAPI(TsdReader* pReader); +static void initMetadataAPI(SStoreMeta* pMeta); +static void initTqAPI(SStoreTqReader* pTq); +static void initStateStoreAPI(SStateStore* pStore); +static void initMetaReaderAPI(SStoreMetaReader* pMetaReader); +static void initMetaFilterAPI(SMetaDataFilterAPI* pFilter); +static void initFunctionStateStore(SFunctionStateStore* pStore); +static void initCacheFn(SStoreCacheReader* pCache); +static void initSnapshotFn(SStoreSnapshotFn* pSnapshot); + +void initStorageAPI(SStorageAPI* pAPI) { + initTsdbReaderAPI(&pAPI->tsdReader); + initMetadataAPI(&pAPI->metaFn); + initStateStoreAPI(&pAPI->stateStore); + initMetaReaderAPI(&pAPI->metaReaderFn); + initMetaFilterAPI(&pAPI->metaFilter); + initTqAPI(&pAPI->tqReaderFn); + initFunctionStateStore(&pAPI->functionStore); + initCacheFn(&pAPI->cacheFn); + initSnapshotFn(&pAPI->snapshotFn); +} + +void initTsdbReaderAPI(TsdReader* pReader) { + pReader->tsdReaderOpen = (int32_t(*)(void*, SQueryTableDataCond*, void*, int32_t, SSDataBlock*, void**, const char*, + bool, SHashObj**))tsdbReaderOpen2; + pReader->tsdReaderClose = tsdbReaderClose2; + + pReader->tsdNextDataBlock = tsdbNextDataBlock2; + + pReader->tsdReaderRetrieveDataBlock = tsdbRetrieveDataBlock2; + pReader->tsdReaderReleaseDataBlock = tsdbReleaseDataBlock2; + + pReader->tsdReaderRetrieveBlockSMAInfo = tsdbRetrieveDatablockSMA2; + + pReader->tsdReaderNotifyClosing = tsdbReaderSetCloseFlag; + pReader->tsdReaderResetStatus = tsdbReaderReset2; + + pReader->tsdReaderGetDataBlockDistInfo = tsdbGetFileBlocksDistInfo2; + pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable2; // todo this function should be moved away + + pReader->tsdSetQueryTableList = tsdbSetTableList2; + pReader->tsdSetReaderTaskId = (void (*)(void*, const char*))tsdbReaderSetId2; +} + +void initMetadataAPI(SStoreMeta* pMeta) { + pMeta->isTableExisted = metaIsTableExist; + + pMeta->openTableMetaCursor = metaOpenTbCursor; + pMeta->closeTableMetaCursor = metaCloseTbCursor; + pMeta->pauseTableMetaCursor = metaPauseTbCursor; + pMeta->resumeTableMetaCursor = metaResumeTbCursor; + pMeta->cursorNext = metaTbCursorNext; + pMeta->cursorPrev = metaTbCursorPrev; + + pMeta->getBasicInfo = vnodeGetInfo; + pMeta->getNumOfChildTables = metaGetStbStats; + + pMeta->getChildTableList = vnodeGetCtbIdList; + + pMeta->storeGetIndexInfo = vnodeGetIdx; + pMeta->getInvertIndex = vnodeGetIvtIdx; + + pMeta->extractTagVal = (const void* (*)(const void*, int16_t, STagVal*))metaGetTableTagVal; + pMeta->getTableTags = metaGetTableTags; + pMeta->getTableTagsByUid = metaGetTableTagsByUids; + + pMeta->getTableUidByName = metaGetTableUidByName; + pMeta->getTableTypeByName = metaGetTableTypeByName; + pMeta->getTableNameByUid = metaGetTableNameByUid; + + pMeta->getTableSchema = tsdbGetTableSchema; // todo refactor + pMeta->storeGetTableList = vnodeGetTableList; + + pMeta->getCachedTableList = metaGetCachedTableUidList; + pMeta->putCachedTableList = metaUidFilterCachePut; + + pMeta->metaGetCachedTbGroup = metaGetCachedTbGroup; + pMeta->metaPutTbGroupToCache = metaPutTbGroupToCache; +} + +void initTqAPI(SStoreTqReader* pTq) { + pTq->tqReaderOpen = tqReaderOpen; + pTq->tqReaderSetColIdList = tqReaderSetColIdList; + + pTq->tqReaderClose = tqReaderClose; + pTq->tqReaderSeek = tqReaderSeek; + pTq->tqRetrieveBlock = tqRetrieveDataBlock; + + pTq->tqReaderNextBlockInWal = tqNextBlockInWal; + + pTq->tqNextBlockImpl = tqNextBlockImpl; // todo remove it + + pTq->tqReaderAddTables = tqReaderAddTbUidList; + pTq->tqReaderSetQueryTableList = tqReaderSetTbUidList; + + pTq->tqReaderRemoveTables = tqReaderRemoveTbUidList; + + pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable; + pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed; + + pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it + pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it + + pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it + pTq->tqGetResultBlock = tqGetResultBlock; + + pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; +} + +void initStateStoreAPI(SStateStore* pStore) { + pStore->streamFileStateInit = streamFileStateInit; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + + pStore->streamStateGetByPos = streamStateGetByPos; + + pStore->streamStatePutParName = streamStatePutParName; + pStore->streamStateGetParName = streamStateGetParName; + + pStore->streamStateAddIfNotExist = streamStateAddIfNotExist; + pStore->streamStateReleaseBuf = streamStateReleaseBuf; + pStore->streamStateFreeVal = streamStateFreeVal; + + pStore->streamStatePut = streamStatePut; + pStore->streamStateGet = streamStateGet; + pStore->streamStateCheck = streamStateCheck; + pStore->streamStateGetByPos = streamStateGetByPos; + pStore->streamStateDel = streamStateDel; + pStore->streamStateClear = streamStateClear; + pStore->streamStateSaveInfo = streamStateSaveInfo; + pStore->streamStateGetInfo = streamStateGetInfo; + pStore->streamStateSetNumber = streamStateSetNumber; + + pStore->streamStateFillPut = streamStateFillPut; + pStore->streamStateFillGet = streamStateFillGet; + pStore->streamStateFillDel = streamStateFillDel; + + pStore->streamStateCurNext = streamStateCurNext; + pStore->streamStateCurPrev = streamStateCurPrev; + + pStore->streamStateGetAndCheckCur = streamStateGetAndCheckCur; + pStore->streamStateSeekKeyNext = streamStateSeekKeyNext; + pStore->streamStateFillSeekKeyNext = streamStateFillSeekKeyNext; + pStore->streamStateFillSeekKeyPrev = streamStateFillSeekKeyPrev; + pStore->streamStateFreeCur = streamStateFreeCur; + + pStore->streamStateGetGroupKVByCur = streamStateGetGroupKVByCur; + pStore->streamStateGetKVByCur = streamStateGetKVByCur; + + pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist; + pStore->streamStateSessionPut = streamStateSessionPut; + pStore->streamStateSessionGet = streamStateSessionGet; + pStore->streamStateSessionDel = streamStateSessionDel; + pStore->streamStateSessionClear = streamStateSessionClear; + pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur; + pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist; + pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange; + + pStore->updateInfoInit = updateInfoInit; + pStore->updateInfoFillBlockData = updateInfoFillBlockData; + pStore->updateInfoIsUpdated = updateInfoIsUpdated; + pStore->updateInfoIsTableInserted = updateInfoIsTableInserted; + pStore->updateInfoDestroy = updateInfoDestroy; + pStore->windowSBfDelete = windowSBfDelete; + pStore->windowSBfAdd = windowSBfAdd; + + pStore->updateInfoInitP = updateInfoInitP; + pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF; + pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF; + pStore->updateInfoSerialize = updateInfoSerialize; + pStore->updateInfoDeserialize = updateInfoDeserialize; + + pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext; + pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev; + pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext; + + pStore->streamFileStateInit = streamFileStateInit; + + pStore->streamFileStateDestroy = streamFileStateDestroy; + pStore->streamFileStateClear = streamFileStateClear; + pStore->needClearDiskBuff = needClearDiskBuff; + + pStore->streamStateOpen = streamStateOpen; + pStore->streamStateClose = streamStateClose; + pStore->streamStateBegin = streamStateBegin; + pStore->streamStateCommit = streamStateCommit; + pStore->streamStateDestroy = streamStateDestroy; + pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; + pStore->streamStateReloadInfo = streamStateReloadInfo; +} + +void initMetaReaderAPI(SStoreMetaReader* pMetaReader) { + pMetaReader->initReader = _metaReaderInit; + pMetaReader->clearReader = metaReaderClear; + + pMetaReader->getTableEntryByUid = metaReaderGetTableEntryByUid; + + pMetaReader->getEntryGetUidCache = metaReaderGetTableEntryByUidCache; + pMetaReader->getTableEntryByName = metaGetTableEntryByName; + + pMetaReader->readerReleaseLock = metaReaderReleaseLock; +} + +void initMetaFilterAPI(SMetaDataFilterAPI* pFilter) { + pFilter->metaFilterCreateTime = metaFilterCreateTime; + pFilter->metaFilterTableIds = metaFilterTableIds; + pFilter->metaFilterTableName = metaFilterTableName; + pFilter->metaFilterTtl = metaFilterTtl; +} + +void initFunctionStateStore(SFunctionStateStore* pStore) { + pStore->streamStateFuncPut = streamStateFuncPut; + pStore->streamStateFuncGet = streamStateFuncGet; +} + +void initCacheFn(SStoreCacheReader* pCache) { + pCache->openReader = tsdbCacherowsReaderOpen; + pCache->closeReader = tsdbCacherowsReaderClose; + pCache->retrieveRows = tsdbRetrieveCacheRows; + pCache->reuseReader = tsdbReuseCacherowsReader; +} + +void initSnapshotFn(SStoreSnapshotFn* pSnapshot) { + pSnapshot->setForSnapShot = setForSnapShot; + pSnapshot->destroySnapshot = destroySnapContext; + pSnapshot->getMetaTableInfoFromSnapshot = getMetaTableInfoFromSnapshot; + pSnapshot->getTableInfoFromSnapshot = getTableInfoFromSnapshot; +} diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 782ffd788d27d338743447fb67954a9dd7812ba7..74a8d14a86c6340c3acedec831524b8ae765b91e 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -23,26 +23,24 @@ struct SVnodeTask { void* arg; }; -struct SVnodeGlobal { - int8_t init; - int8_t stop; +typedef struct { int nthreads; TdThread* threads; TdThreadMutex mutex; TdThreadCond hasTask; SVnodeTask queue; +} SVnodeThreadPool; + +struct SVnodeGlobal { + int8_t init; + int8_t stop; + SVnodeThreadPool tp[2]; }; struct SVnodeGlobal vnodeGlobal; static void* loop(void* arg); -static tsem_t canCommit = {0}; - -static void vnodeInitCommit() { tsem_init(&canCommit, 0, 4); }; -void vnode_wait_commit() { tsem_wait(&canCommit); } -void vnode_done_commit() { tsem_wait(&canCommit); } - int vnodeInit(int nthreads) { int8_t init; int ret; @@ -51,28 +49,30 @@ int vnodeInit(int nthreads) { if (init) { return 0; } + vnodeGlobal.stop = 0; - taosThreadMutexInit(&vnodeGlobal.mutex, NULL); - taosThreadCondInit(&vnodeGlobal.hasTask, NULL); + for (int32_t i = 0; i < ARRAY_SIZE(vnodeGlobal.tp); i++) { + taosThreadMutexInit(&vnodeGlobal.tp[i].mutex, NULL); + taosThreadCondInit(&vnodeGlobal.tp[i].hasTask, NULL); - taosThreadMutexLock(&vnodeGlobal.mutex); + taosThreadMutexLock(&vnodeGlobal.tp[i].mutex); - vnodeGlobal.stop = 0; - vnodeGlobal.queue.next = &vnodeGlobal.queue; - vnodeGlobal.queue.prev = &vnodeGlobal.queue; + vnodeGlobal.tp[i].queue.next = &vnodeGlobal.tp[i].queue; + vnodeGlobal.tp[i].queue.prev = &vnodeGlobal.tp[i].queue; - taosThreadMutexUnlock(&(vnodeGlobal.mutex)); + taosThreadMutexUnlock(&(vnodeGlobal.tp[i].mutex)); - vnodeGlobal.nthreads = nthreads; - vnodeGlobal.threads = taosMemoryCalloc(nthreads, sizeof(TdThread)); - if (vnodeGlobal.threads == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - vError("failed to init vnode module since:%s", tstrerror(terrno)); - return -1; - } + vnodeGlobal.tp[i].nthreads = nthreads; + vnodeGlobal.tp[i].threads = taosMemoryCalloc(nthreads, sizeof(TdThread)); + if (vnodeGlobal.tp[i].threads == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + vError("failed to init vnode module since:%s", tstrerror(terrno)); + return -1; + } - for (int i = 0; i < nthreads; i++) { - taosThreadCreate(&(vnodeGlobal.threads[i]), NULL, loop, NULL); + for (int j = 0; j < nthreads; j++) { + taosThreadCreate(&(vnodeGlobal.tp[i].threads[j]), NULL, loop, &vnodeGlobal.tp[i]); + } } if (walInit() < 0) { @@ -92,27 +92,29 @@ void vnodeCleanup() { if (init == 0) return; // set stop - taosThreadMutexLock(&(vnodeGlobal.mutex)); vnodeGlobal.stop = 1; - taosThreadCondBroadcast(&(vnodeGlobal.hasTask)); - taosThreadMutexUnlock(&(vnodeGlobal.mutex)); + for (int32_t i = 0; i < ARRAY_SIZE(vnodeGlobal.tp); i++) { + taosThreadMutexLock(&(vnodeGlobal.tp[i].mutex)); + taosThreadCondBroadcast(&(vnodeGlobal.tp[i].hasTask)); + taosThreadMutexUnlock(&(vnodeGlobal.tp[i].mutex)); + + // wait for threads + for (int j = 0; j < vnodeGlobal.tp[i].nthreads; j++) { + taosThreadJoin(vnodeGlobal.tp[i].threads[j], NULL); + } - // wait for threads - for (int i = 0; i < vnodeGlobal.nthreads; i++) { - taosThreadJoin(vnodeGlobal.threads[i], NULL); + // clear source + taosMemoryFreeClear(vnodeGlobal.tp[i].threads); + taosThreadCondDestroy(&(vnodeGlobal.tp[i].hasTask)); + taosThreadMutexDestroy(&(vnodeGlobal.tp[i].mutex)); } - // clear source - taosMemoryFreeClear(vnodeGlobal.threads); - taosThreadCondDestroy(&(vnodeGlobal.hasTask)); - taosThreadMutexDestroy(&(vnodeGlobal.mutex)); - walCleanUp(); tqCleanUp(); smaCleanUp(); } -int vnodeScheduleTask(int (*execute)(void*), void* arg) { +int vnodeScheduleTaskEx(int tpid, int (*execute)(void*), void* arg) { SVnodeTask* pTask; ASSERT(!vnodeGlobal.stop); @@ -126,35 +128,42 @@ int vnodeScheduleTask(int (*execute)(void*), void* arg) { pTask->execute = execute; pTask->arg = arg; - taosThreadMutexLock(&(vnodeGlobal.mutex)); - pTask->next = &vnodeGlobal.queue; - pTask->prev = vnodeGlobal.queue.prev; - vnodeGlobal.queue.prev->next = pTask; - vnodeGlobal.queue.prev = pTask; - taosThreadCondSignal(&(vnodeGlobal.hasTask)); - taosThreadMutexUnlock(&(vnodeGlobal.mutex)); + taosThreadMutexLock(&(vnodeGlobal.tp[tpid].mutex)); + pTask->next = &vnodeGlobal.tp[tpid].queue; + pTask->prev = vnodeGlobal.tp[tpid].queue.prev; + vnodeGlobal.tp[tpid].queue.prev->next = pTask; + vnodeGlobal.tp[tpid].queue.prev = pTask; + taosThreadCondSignal(&(vnodeGlobal.tp[tpid].hasTask)); + taosThreadMutexUnlock(&(vnodeGlobal.tp[tpid].mutex)); return 0; } +int vnodeScheduleTask(int (*execute)(void*), void* arg) { return vnodeScheduleTaskEx(0, execute, arg); } + /* ------------------------ STATIC METHODS ------------------------ */ static void* loop(void* arg) { - SVnodeTask* pTask; - int ret; - - setThreadName("vnode-commit"); + SVnodeThreadPool* tp = (SVnodeThreadPool*)arg; + SVnodeTask* pTask; + int ret; + + if (tp == &vnodeGlobal.tp[0]) { + setThreadName("vnode-commit"); + } else if (tp == &vnodeGlobal.tp[1]) { + setThreadName("vnode-merge"); + } for (;;) { - taosThreadMutexLock(&(vnodeGlobal.mutex)); + taosThreadMutexLock(&(tp->mutex)); for (;;) { - pTask = vnodeGlobal.queue.next; - if (pTask == &vnodeGlobal.queue) { + pTask = tp->queue.next; + if (pTask == &tp->queue) { // no task if (vnodeGlobal.stop) { - taosThreadMutexUnlock(&(vnodeGlobal.mutex)); + taosThreadMutexUnlock(&(tp->mutex)); return NULL; } else { - taosThreadCondWait(&(vnodeGlobal.hasTask), &(vnodeGlobal.mutex)); + taosThreadCondWait(&(tp->hasTask), &(tp->mutex)); } } else { // has task @@ -164,7 +173,7 @@ static void* loop(void* arg) { } } - taosThreadMutexUnlock(&(vnodeGlobal.mutex)); + taosThreadMutexUnlock(&(tp->mutex)); pTask->execute(pTask->arg); taosMemoryFree(pTask); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index b5e7c6875b3ce044530133f9105ee902628b23c0..65fc552365ff1b9de1d866c098997ed7917dae9a 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -15,7 +15,27 @@ #include "vnd.h" -int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { +int32_t vnodeGetPrimaryDir(const char *relPath, int32_t diskPrimary, STfs *pTfs, char *buf, size_t bufLen) { + if (pTfs) { + SDiskID diskId = {0}; + diskId.id = diskPrimary; + snprintf(buf, bufLen - 1, "%s%s%s", tfsGetDiskPath(pTfs, diskId), TD_DIRSEP, relPath); + } else { + snprintf(buf, bufLen - 1, "%s", relPath); + } + buf[bufLen - 1] = '\0'; + return 0; +} + +static int32_t vnodeMkDir(STfs *pTfs, const char *path) { + if (pTfs) { + return tfsMkdirRecur(pTfs, path); + } else { + return taosMkDir(path); + } +} + +int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs *pTfs) { SVnodeInfo info = {0}; char dir[TSDB_FILENAME_LEN] = {0}; @@ -26,18 +46,11 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { } // create vnode env - if (pTfs) { - if (tfsMkdirAt(pTfs, path, (SDiskID){0}) < 0) { - vError("vgId:%d, failed to create vnode since:%s", pCfg->vgId, tstrerror(terrno)); - return -1; - } - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path); - } else { - if (taosMkDir(path)) { - return TAOS_SYSTEM_ERROR(errno); - } - snprintf(dir, TSDB_FILENAME_LEN, "%s", path); + if (vnodeMkDir(pTfs, path)) { + vError("vgId:%d, failed to prepare vnode dir since %s, path: %s", pCfg->vgId, strerror(errno), path); + return TAOS_SYSTEM_ERROR(errno); } + vnodeGetPrimaryDir(path, diskPrimary, pTfs, dir, TSDB_FILENAME_LEN); if (pCfg) { info.config = *pCfg; @@ -58,16 +71,12 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { return 0; } -int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs) { +int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t diskPrimary, STfs *pTfs) { SVnodeInfo info = {0}; char dir[TSDB_FILENAME_LEN] = {0}; int32_t ret = 0; - if (pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", path); - } + vnodeGetPrimaryDir(path, diskPrimary, pTfs, dir, TSDB_FILENAME_LEN); ret = vnodeLoadInfo(dir, &info); if (ret < 0) { @@ -76,7 +85,7 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *p } SSyncCfg *pCfg = &info.config.syncCfg; - + pCfg->replicaNum = 0; pCfg->totalReplicaNum = 0; memset(&pCfg->nodeInfo, 0, sizeof(pCfg->nodeInfo)); @@ -109,7 +118,7 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *p pCfg->myIndex = pReq->replica + pReq->learnerSelfIndex; } - vInfo("vgId:%d, save config while alter, replicas:%d totalReplicas:%d selfIndex:%d", + vInfo("vgId:%d, save config while alter, replicas:%d totalReplicas:%d selfIndex:%d", pReq->vgId, pCfg->replicaNum, pCfg->totalReplicaNum, pCfg->myIndex); info.config.syncCfg = *pCfg; @@ -129,16 +138,23 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *p return 0; } -int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, STfs *pTfs) { - int32_t ret = tfsRename(pTfs, srcPath, dstPath); - if (ret != 0) return ret; +static int32_t vnodeVgroupIdLen(int32_t vgId) { + char tmp[TSDB_FILENAME_LEN]; + sprintf(tmp, "%d", vgId); + return strlen(tmp); +} + +int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, + int32_t diskPrimary, STfs *pTfs) { + int32_t ret = 0; char oldRname[TSDB_FILENAME_LEN] = {0}; char newRname[TSDB_FILENAME_LEN] = {0}; char tsdbPath[TSDB_FILENAME_LEN] = {0}; char tsdbFilePrefix[TSDB_FILENAME_LEN] = {0}; - snprintf(tsdbPath, TSDB_FILENAME_LEN, "%s%stsdb", dstPath, TD_DIRSEP); + snprintf(tsdbPath, TSDB_FILENAME_LEN, "%s%stsdb", srcPath, TD_DIRSEP); snprintf(tsdbFilePrefix, TSDB_FILENAME_LEN, "tsdb%sv", TD_DIRSEP); + int32_t prefixLen = strlen(tsdbFilePrefix); STfsDir *tsdbDir = tfsOpendir(pTfs, tsdbPath); if (tsdbDir == NULL) return 0; @@ -152,18 +168,17 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr char *tsdbFilePrefixPos = strstr(oldRname, tsdbFilePrefix); if (tsdbFilePrefixPos == NULL) continue; - int32_t tsdbFileVgId = atoi(tsdbFilePrefixPos + 6); + int32_t tsdbFileVgId = atoi(tsdbFilePrefixPos + prefixLen); if (tsdbFileVgId == srcVgId) { - char *tsdbFileSurfixPos = strstr(tsdbFilePrefixPos, "f"); - if (tsdbFileSurfixPos == NULL) continue; + char *tsdbFileSurfixPos = tsdbFilePrefixPos + prefixLen + vnodeVgroupIdLen(srcVgId); - tsdbFilePrefixPos[6] = 0; + tsdbFilePrefixPos[prefixLen] = 0; snprintf(newRname, TSDB_FILENAME_LEN, "%s%d%s", oldRname, dstVgId, tsdbFileSurfixPos); vInfo("vgId:%d, rename file from %s to %s", dstVgId, tsdbFile->rname, newRname); - ret = tfsRename(pTfs, tsdbFile->rname, newRname); + ret = tfsRename(pTfs, diskPrimary, tsdbFile->rname, newRname); if (ret != 0) { - vInfo("vgId:%d, failed to rename file from %s to %s since %s", dstVgId, tsdbFile->rname, newRname, terrstr()); + vError("vgId:%d, failed to rename file from %s to %s since %s", dstVgId, tsdbFile->rname, newRname, terrstr()); tfsClosedir(tsdbDir); return ret; } @@ -171,21 +186,22 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr } tfsClosedir(tsdbDir); - return 0; + + vInfo("vgId:%d, rename dir from %s to %s", dstVgId, srcPath, dstPath); + ret = tfsRename(pTfs, diskPrimary, srcPath, dstPath); + if (ret != 0) { + vError("vgId:%d, failed to rename dir from %s to %s since %s", dstVgId, srcPath, dstPath, terrstr()); + } + return ret; } -int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq, STfs *pTfs) { +int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq, + int32_t diskPrimary, STfs *pTfs) { SVnodeInfo info = {0}; char dir[TSDB_FILENAME_LEN] = {0}; int32_t ret = 0; - if (pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, srcPath); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", srcPath); - } - - // todo add stat file to handle exception while vnode open + vnodeGetPrimaryDir(srcPath, diskPrimary, pTfs, dir, TSDB_FILENAME_LEN); ret = vnodeLoadInfo(dir, &info); if (ret < 0) { @@ -229,7 +245,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod } vInfo("vgId:%d, rename %s to %s", pReq->dstVgId, srcPath, dstPath); - ret = vnodeRenameVgroupId(srcPath, dstPath, pReq->srcVgId, pReq->dstVgId, pTfs); + ret = vnodeRenameVgroupId(srcPath, dstPath, pReq->srcVgId, pReq->dstVgId, diskPrimary, pTfs); if (ret < 0) { vError("vgId:%d, failed to rename vnode from %s to %s since %s", pReq->dstVgId, srcPath, dstPath, tstrerror(terrno)); @@ -240,23 +256,73 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod return 0; } +int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, + int32_t diskPrimary, STfs *pTfs) { + SVnodeInfo info = {0}; + char dir[TSDB_FILENAME_LEN] = {0}; + + vnodeGetPrimaryDir(dstPath, diskPrimary, pTfs, dir, TSDB_FILENAME_LEN); + if (vnodeLoadInfo(dir, &info) == 0) { + if (info.config.vgId != dstVgId) { + vError("vgId:%d, unexpected vnode config.vgId:%d", dstVgId, info.config.vgId); + return -1; + } + return dstVgId; + } + + vnodeGetPrimaryDir(srcPath, diskPrimary, pTfs, dir, TSDB_FILENAME_LEN); + if (vnodeLoadInfo(dir, &info) < 0) { + vError("vgId:%d, failed to read vnode config from %s since %s", srcVgId, srcPath, tstrerror(terrno)); + return -1; + } + + if (info.config.vgId == srcVgId) { + vInfo("vgId:%d, rollback alter hashrange", srcVgId); + return srcVgId; + } else if (info.config.vgId != dstVgId) { + vError("vgId:%d, unexpected vnode config.vgId:%d", dstVgId, info.config.vgId); + return -1; + } + + vInfo("vgId:%d, rename %s to %s", dstVgId, srcPath, dstPath); + if (vnodeRenameVgroupId(srcPath, dstPath, srcVgId, dstVgId, diskPrimary, pTfs) < 0) { + vError("vgId:%d, failed to rename vnode from %s to %s since %s", dstVgId, srcPath, dstPath, tstrerror(terrno)); + return -1; + } + + return dstVgId; +} + void vnodeDestroy(const char *path, STfs *pTfs) { vInfo("path:%s is removed while destroy vnode", path); tfsRmdir(pTfs, path); } -SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { +static int32_t vnodeCheckDisk(int32_t diskPrimary, STfs *pTfs) { + int32_t ndisk = 1; + if (pTfs) { + ndisk = tfsGetDisksAtLevel(pTfs, 0); + } + if (diskPrimary < 0 || diskPrimary >= ndisk) { + vError("disk:%d is unavailable from the %d disks mounted at level 0", diskPrimary, ndisk); + terrno = TSDB_CODE_FS_INVLD_CFG; + return -1; + } + return 0; +} + +SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb) { SVnode *pVnode = NULL; SVnodeInfo info = {0}; char dir[TSDB_FILENAME_LEN] = {0}; char tdir[TSDB_FILENAME_LEN * 2] = {0}; int32_t ret = 0; - if (pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", path); + if (vnodeCheckDisk(diskPrimary, pTfs)) { + vError("failed to open vnode from %s since %s. diskPrimary:%d", path, terrstr(), diskPrimary); + return NULL; } + vnodeGetPrimaryDir(path, diskPrimary, pTfs, dir, TSDB_FILENAME_LEN); info.config = vnodeCfgDefault; @@ -299,6 +365,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { pVnode->state.applied = info.state.committed; pVnode->state.applyTerm = info.state.commitTerm; pVnode->pTfs = pTfs; + pVnode->diskPrimary = diskPrimary; pVnode->msgCb = msgCb; taosThreadMutexInit(&pVnode->lock, NULL); pVnode->blocked = false; @@ -322,18 +389,16 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { goto _err; } + if (metaUpgrade(pVnode, &pVnode->pMeta) < 0) { + vError("vgId:%d, failed to upgrade meta since %s", TD_VID(pVnode), tstrerror(terrno)); + } + // open tsdb if (!VND_IS_RSMA(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, NULL, rollback) < 0) { vError("vgId:%d, failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } - // open sma - if (smaOpen(pVnode, rollback)) { - vError("vgId:%d, failed to open vnode sma since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } - // open wal sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); taosRealPath(tdir, NULL, sizeof(tdir)); @@ -353,6 +418,12 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { goto _err; } + // open sma + if (smaOpen(pVnode, rollback)) { + vError("vgId:%d, failed to open vnode sma since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + // open query if (vnodeQueryOpen(pVnode)) { vError("vgId:%d, failed to open vnode query since %s", TD_VID(pVnode), tstrerror(terrno)); @@ -405,8 +476,8 @@ void vnodeClose(SVnode *pVnode) { tsem_wait(&pVnode->canCommit); vnodeSyncClose(pVnode); vnodeQueryClose(pVnode); - walClose(pVnode->pWal); tqClose(pVnode->pTq); + walClose(pVnode->pWal); if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); smaClose(pVnode->pSma); if (pVnode->pMeta) metaClose(&pVnode->pMeta); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 303d2a9ca449de504a582c8472951a1bf6c6c683..51f4cee40cafd308d6a3dbd601685d9192de5cbb 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -62,7 +62,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { } // query meta - metaReaderInit(&mer1, pVnode->pMeta, 0); + metaReaderDoInit(&mer1, pVnode->pMeta, 0); if (metaGetTableEntryByName(&mer1, infoReq.tbName) < 0) { code = terrno; @@ -79,8 +79,8 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { schemaTag = mer1.me.stbEntry.schemaTag; metaRsp.suid = mer1.me.uid; } else if (mer1.me.type == TSDB_CHILD_TABLE) { - metaReaderInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); - if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; + metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); + if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; strcpy(metaRsp.stbName, mer2.me.name); metaRsp.suid = mer2.me.uid; @@ -175,7 +175,7 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { } // query meta - metaReaderInit(&mer1, pVnode->pMeta, 0); + metaReaderDoInit(&mer1, pVnode->pMeta, 0); if (metaGetTableEntryByName(&mer1, cfgReq.tbName) < 0) { code = terrno; @@ -188,8 +188,8 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { code = TSDB_CODE_VND_HASH_MISMATCH; goto _exit; } else if (mer1.me.type == TSDB_CHILD_TABLE) { - metaReaderInit(&mer2, pVnode->pMeta, 0); - if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; + metaReaderDoInit(&mer2, pVnode->pMeta, 0); + if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; strcpy(cfgRsp.stbName, mer2.me.name); schema = mer2.me.stbEntry.schemaRow; @@ -410,13 +410,32 @@ void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { "nBatchInsertSuccess"); } -void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) { +void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables) { + SVnode* pVnodeObj = pVnode; + SVnodeCfg* pConf = &pVnodeObj->config; + if (dbname) { - *dbname = pVnode->config.dbname; + *dbname = pConf->dbname; } if (vgId) { - *vgId = TD_VID(pVnode); + *vgId = TD_VID(pVnodeObj); + } + + if (numOfTables) { + *numOfTables = pConf->vndStats.numOfNTables + pConf->vndStats.numOfCTables; + } + + if (numOfNormalTables) { + *numOfNormalTables = pConf->vndStats.numOfNTables; + } +} + +int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList) { + if (type == TSDB_SUPER_TABLE) { + return vnodeGetStbIdList(pVnode, 0, pList); + } else { + return TSDB_CODE_INVALID_PARA; } } @@ -440,8 +459,10 @@ int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) { int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg) { return 0; } -int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list) { - SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid, 1); + +int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) { + SVnode *pVnodeObj = pVnode; + SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj->pMeta, suid, 1); while (1) { tb_uid_t id = metaCtbCursorNext(pCur); @@ -475,6 +496,30 @@ int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) { return TSDB_CODE_SUCCESS; } +int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void *arg1), + void *arg) { + SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid); + if (!pCur) { + return TSDB_CODE_FAILED; + } + + while (1) { + tb_uid_t id = metaStbCursorNext(pCur); + if (id == 0) { + break; + } + + if ((*filter) && (*filter)(arg, &id)) { + continue; + } + + taosArrayPush(list, &id); + } + + metaCloseStbCursor(pCur); + return TSDB_CODE_SUCCESS; +} + int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) { SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid, 0); if (!pCur) { @@ -510,6 +555,58 @@ static int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { return TSDB_CODE_SUCCESS; } +#ifdef TD_ENTERPRISE +#define TK_LOG_STB_NUM 19 +static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", + "data_dir", + "dnodes_info", + "d_info", + "grants_info", + "keeper_monitor", + "logs", + "log_dir", + "log_summary", + "m_info", + "taosadapter_restful_http_request_fail", + "taosadapter_restful_http_request_in_flight", + "taosadapter_restful_http_request_summary_milliseconds", + "taosadapter_restful_http_request_total", + "taosadapter_system_cpu_percent", + "taosadapter_system_mem_percent", + "temp_dir", + "vgroups_info", + "vnodes_role"}; + +// exclude stbs of taoskeeper log +static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { + char *dbName = strchr(pVnode->config.dbname, '.'); + if (!dbName || 0 != strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { + return 0; + } + int32_t tbSize = metaSizeOfTbFilterCache(pVnode, 0); + if (tbSize < TK_LOG_STB_NUM) { + for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { + tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]); + if (suid != 0) { + metaPutTbToFilterCache(pVnode, suid, 0); + } + } + tbSize = metaSizeOfTbFilterCache(pVnode, 0); + } + + return tbSize; +} +#endif + +static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) { + SVnode *pVnode = (SVnode *)arg1; + + if (metaTbInFilterCache(pVnode, *(tb_uid_t *)(arg2), 0)) { + return true; + } + return false; +} + int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { SArray *suidList = NULL; @@ -518,7 +615,13 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { return TSDB_CODE_FAILED; } - if (vnodeGetStbIdList(pVnode, 0, suidList) < 0) { + int32_t tbFilterSize = 0; + #ifdef TD_ENTERPRISE + tbFilterSize = vnodeGetTimeSeriesBlackList(pVnode); + #endif + + if ((!tbFilterSize && vnodeGetStbIdList(pVnode, 0, suidList) < 0) || + (tbFilterSize && vnodeGetStbIdListByFilter(pVnode, 0, suidList, vnodeTimeSeriesFilter, pVnode) < 0)) { qError("vgId:%d, failed to get stb id list error: %s", TD_VID(pVnode), terrstr()); taosArrayDestroy(suidList); return TSDB_CODE_FAILED; @@ -529,10 +632,8 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { for (int64_t i = 0; i < arrSize; ++i) { tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); - SMetaStbStats stats = {0}; - metaGetStbStats(pVnode->pMeta, suid, &stats); - int64_t ctbNum = stats.ctbNum; - // vnodeGetCtbNum(pVnode, id, &ctbNum); + int64_t ctbNum = 0; + metaGetStbStats(pVnode, suid, &ctbNum); int numOfCols = 0; vnodeGetStbColumnNum(pVnode, suid, &numOfCols); @@ -567,16 +668,17 @@ int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) { return TSDB_CODE_SUCCESS; } -void *vnodeGetIdx(SVnode *pVnode) { +void *vnodeGetIdx(void *pVnode) { if (pVnode == NULL) { return NULL; } - return metaGetIdx(pVnode->pMeta); + + return metaGetIdx(((SVnode*)pVnode)->pMeta); } -void *vnodeGetIvtIdx(SVnode *pVnode) { +void *vnodeGetIvtIdx(void *pVnode) { if (pVnode == NULL) { return NULL; } - return metaGetIvtIdx(pVnode->pMeta); + return metaGetIvtIdx(((SVnode*)pVnode)->pMeta); } diff --git a/source/dnode/vnode/src/vnd/vnodeRetention.c b/source/dnode/vnode/src/vnd/vnodeRetention.c index 170deb42867d2bdd2df80ff01d84be31115d8ed3..f3344d1d7d0f82cca3872a27d50b5a6ad2680e98 100644 --- a/source/dnode/vnode/src/vnd/vnodeRetention.c +++ b/source/dnode/vnode/src/vnd/vnodeRetention.c @@ -15,116 +15,8 @@ #include "vnd.h" -typedef struct { - SVnode *pVnode; - int64_t now; - int64_t commitID; - SVnodeInfo info; -} SRetentionInfo; +extern int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync); -extern bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now); -extern int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now); -extern int32_t tsdbCommitRetention(STsdb *pTsdb); - -static int32_t vnodePrepareRentention(SVnode *pVnode, SRetentionInfo *pInfo) { - int32_t code = 0; - int32_t lino = 0; - - tsem_wait(&pVnode->canCommit); - - pInfo->commitID = ++pVnode->state.commitID; - - char dir[TSDB_FILENAME_LEN] = {0}; - if (pVnode->pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); - } - - if (vnodeLoadInfo(dir, &pInfo->info) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); - tsem_post(&pVnode->canCommit); - } else { - vInfo("vgId:%d %s done", TD_VID(pVnode), __func__); - } - return code; -} - -static int32_t vnodeRetentionTask(void *param) { - int32_t code = 0; - int32_t lino = 0; - - SRetentionInfo *pInfo = (SRetentionInfo *)param; - SVnode *pVnode = pInfo->pVnode; - char dir[TSDB_FILENAME_LEN] = {0}; - - if (pVnode->pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); - } - - // save info - pInfo->info.state.commitID = pInfo->commitID; - - if (vnodeSaveInfo(dir, &pInfo->info) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } - - // do job - code = tsdbDoRetention(pInfo->pVnode->pTsdb, pInfo->now); - TSDB_CHECK_CODE(code, lino, _exit); - - // commit info - vnodeCommitInfo(dir); - - // commit sub-job - tsdbCommitRetention(pVnode->pTsdb); - -_exit: - if (code) { - vError("vgId:%d %s failed at line %d since %s", TD_VID(pInfo->pVnode), __func__, lino, tstrerror(code)); - } else { - vInfo("vgId:%d %s done", TD_VID(pInfo->pVnode), __func__); - } - tsem_post(&pInfo->pVnode->canCommit); - taosMemoryFree(pInfo); - return code; -} - -int32_t vnodeAsyncRentention(SVnode *pVnode, int64_t now) { - int32_t code = 0; - int32_t lino = 0; - - if (!tsdbShouldDoRetention(pVnode->pTsdb, now)) return code; - - SRetentionInfo *pInfo = (SRetentionInfo *)taosMemoryCalloc(1, sizeof(*pInfo)); - if (pInfo == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - pInfo->pVnode = pVnode; - pInfo->now = now; - - code = vnodePrepareRentention(pVnode, pInfo); - TSDB_CHECK_CODE(code, lino, _exit); - - vnodeScheduleTask(vnodeRetentionTask, pInfo); - -_exit: - if (code) { - vError("vgId:%d %s failed at line %d since %s", TD_VID(pInfo->pVnode), __func__, lino, tstrerror(code)); - if (pInfo) taosMemoryFree(pInfo); - } else { - vInfo("vgId:%d %s done", TD_VID(pInfo->pVnode), __func__); - } - return 0; +int32_t vnodeDoRetention(SVnode *pVnode, int64_t now) { + return tsdbRetention(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1); } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 052e4ab2c18e6dad651cf591962bcb4dd49a0a38..d559783c2f284d40f5ea32fc5cfb1552838e46ad 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -86,17 +86,17 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) { int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) { int32_t code = 0; + SVnode *pVnode = pReader->pVnode; // CONFIG ============== // FIXME: if commit multiple times and the config changed? if (!pReader->cfgDone) { char fName[TSDB_FILENAME_LEN]; - if (pReader->pVnode->pTfs) { - snprintf(fName, TSDB_FILENAME_LEN, "%s%s%s%s%s", tfsGetPrimaryPath(pReader->pVnode->pTfs), TD_DIRSEP, - pReader->pVnode->path, TD_DIRSEP, VND_INFO_FNAME); - } else { - snprintf(fName, TSDB_FILENAME_LEN, "%s%s%s", pReader->pVnode->path, TD_DIRSEP, VND_INFO_FNAME); - } + int32_t offset = 0; + + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, fName, TSDB_FILENAME_LEN); + offset = strlen(fName); + snprintf(fName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VND_INFO_FNAME); TdFilePtr pFile = taosOpenFile(fName, TD_FILE_READ); if (NULL == pFile) { @@ -344,11 +344,7 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * .applyTerm = pWriter->info.state.commitTerm}; pVnode->statis = pWriter->info.statis; char dir[TSDB_FILENAME_LEN] = {0}; - if (pWriter->pVnode->pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", pWriter->pVnode->path); - } + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN); vnodeCommitInfo(dir); } else { @@ -386,7 +382,7 @@ _exit: static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { int32_t code = 0; - + SVnode *pVnode = pWriter->pVnode; SSnapDataHdr *pHdr = (SSnapDataHdr *)pData; // decode info @@ -400,15 +396,9 @@ static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_ // modify info as needed char dir[TSDB_FILENAME_LEN] = {0}; - if (pWriter->pVnode->pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pWriter->pVnode->pTfs), TD_DIRSEP, - pWriter->pVnode->path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", pWriter->pVnode->path); - } + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN); SVnodeStats vndStats = pWriter->info.config.vndStats; - SVnode *pVnode = pWriter->pVnode; pWriter->info.config = pVnode->config; pWriter->info.config.vndStats = vndStats; vDebug("vgId:%d, save config while write snapshot", pWriter->pVnode->config.vgId); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fe1ccf90c872c090e6d0957e742b7879b4f18640..743470aac8372e5647f4fc618685fcc5619a0a9c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -19,25 +19,25 @@ #include "vnode.h" #include "vnodeInt.h" -static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); - -static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t ctime, int64_t *pUid) { +static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); + +static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t btime, int64_t *pUid) { int32_t code = 0; int32_t lino = 0; @@ -66,8 +66,8 @@ static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, i } *(int64_t *)(pCoder->data + pCoder->pos) = uid; - // ctime - *(int64_t *)(pCoder->data + pCoder->pos + 8) = ctime; + // btime + *(int64_t *)(pCoder->data + pCoder->pos + 8) = btime; tEndDecode(pCoder); @@ -84,7 +84,7 @@ static int32_t vnodePreProcessCreateTableMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; int32_t lino = 0; - int64_t ctime = taosGetTimestampMs(); + int64_t btime = taosGetTimestampMs(); SDecoder dc = {0}; int32_t nReqs; @@ -99,7 +99,7 @@ static int32_t vnodePreProcessCreateTableMsg(SVnode *pVnode, SRpcMsg *pMsg) { TSDB_CHECK_CODE(code, lino, _exit); } for (int32_t iReq = 0; iReq < nReqs; iReq++) { - code = vnodePreprocessCreateTableReq(pVnode, &dc, ctime, NULL); + code = vnodePreprocessCreateTableReq(pVnode, &dc, btime, NULL); TSDB_CHECK_CODE(code, lino, _exit); } @@ -107,10 +107,41 @@ static int32_t vnodePreProcessCreateTableMsg(SVnode *pVnode, SRpcMsg *pMsg) { _exit: tDecoderClear(&dc); + if (code) { + vError("vgId:%d, %s:%d failed to preprocess submit request since %s, msg type:%s", TD_VID(pVnode), __func__, lino, + tstrerror(code), TMSG_INFO(pMsg->msgType)); + } + return code; +} + +static int32_t vnodePreProcessAlterTableMsg(SVnode *pVnode, SRpcMsg *pMsg) { + int32_t code = TSDB_CODE_INVALID_MSG; + int32_t lino = 0; + + SDecoder dc = {0}; + tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); + + SVAlterTbReq vAlterTbReq = {0}; + int64_t ctimeMs = taosGetTimestampMs(); + if (tDecodeSVAlterTbReqSetCtime(&dc, &vAlterTbReq, ctimeMs) < 0) { + goto _exit; + } + + code = 0; + +_exit: + tDecoderClear(&dc); + if (code) { + vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } else { + vTrace("vgId:%d %s done, table:%s ctimeMs generated:%" PRId64, TD_VID(pVnode), __func__, vAlterTbReq.tbName, + ctimeMs); + } return code; } + extern int64_t tsMaxKeyByPrecision[]; -static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int64_t ctime) { +static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int64_t btimeMs, int64_t ctimeMs) { int32_t code = 0; int32_t lino = 0; @@ -127,7 +158,7 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int int64_t uid; if (submitTbData.flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { - code = vnodePreprocessCreateTableReq(pVnode, pCoder, ctime, &uid); + code = vnodePreprocessCreateTableReq(pVnode, pCoder, btimeMs, &uid); TSDB_CHECK_CODE(code, lino, _exit); } @@ -153,7 +184,7 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int } // scan and check - TSKEY now = ctime; + TSKEY now = btimeMs; if (pVnode->config.tsdbCfg.precision == TSDB_TIME_PRECISION_MICRO) { now *= 1000; } else if (pVnode->config.tsdbCfg.precision == TSDB_TIME_PRECISION_NANO) { @@ -170,7 +201,6 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int SColData colData = {0}; pCoder->pos += tGetColData(pCoder->data + pCoder->pos, &colData); - if (colData.flag != HAS_VALUE) { code = TSDB_CODE_INVALID_MSG; goto _exit; @@ -182,6 +212,10 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int goto _exit; } } + + for (uint64_t i = 1; i < nColData; i++) { + pCoder->pos += tGetColData(pCoder->data + pCoder->pos, &colData); + } } else { uint64_t nRow; if (tDecodeU64v(pCoder, &nRow) < 0) { @@ -200,6 +234,11 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int } } + if (!tDecodeIsEnd(pCoder)) { + *(int64_t *)(pCoder->data + pCoder->pos) = ctimeMs; + pCoder->pos += sizeof(int64_t); + } + tEndDecode(pCoder); _exit: @@ -229,9 +268,10 @@ static int32_t vnodePreProcessSubmitMsg(SVnode *pVnode, SRpcMsg *pMsg) { TSDB_CHECK_CODE(code, lino, _exit); } - int64_t ctime = taosGetTimestampMs(); + int64_t btimeMs = taosGetTimestampMs(); + int64_t ctimeMs = btimeMs; for (int32_t i = 0; i < nSubmitTbData; i++) { - code = vnodePreProcessSubmitTbData(pVnode, pCoder, ctime); + code = vnodePreProcessSubmitTbData(pVnode, pCoder, btimeMs, ctimeMs); TSDB_CHECK_CODE(code, lino, _exit); } @@ -239,22 +279,29 @@ static int32_t vnodePreProcessSubmitMsg(SVnode *pVnode, SRpcMsg *pMsg) { _exit: tDecoderClear(pCoder); + if (code) { + vError("vgId:%d, %s:%d failed to preprocess submit request since %s, msg type:%s", TD_VID(pVnode), __func__, lino, + tstrerror(code), TMSG_INFO(pMsg->msgType)); + } return code; } static int32_t vnodePreProcessDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; - int32_t size; - int32_t ret; - uint8_t *pCont; - SEncoder *pCoder = &(SEncoder){0}; - SDeleteRes res = {0}; - SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; + int32_t size; + int32_t ret; + uint8_t *pCont; + SEncoder *pCoder = &(SEncoder){0}; + SDeleteRes res = {0}; + + SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; + initStorageAPI(&handle.api); code = qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, &res); if (code) goto _exit; + res.ctimeMs = taosGetTimestampMs(); // malloc and encode tEncodeSize(tEncodeDeleteRes, &res, size, ret); pCont = rpcMallocCont(size + sizeof(SMsgHead)); @@ -276,6 +323,31 @@ _exit: return code; } +static int32_t vnodePreProcessBatchDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) { + int32_t code = 0; + int32_t lino = 0; + + int64_t ctimeMs = taosGetTimestampMs(); + SBatchDeleteReq pReq = {0}; + SDecoder *pCoder = &(SDecoder){0}; + + tDecoderInit(pCoder, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); + + if (tDecodeSBatchDeleteReqSetCtime(pCoder, &pReq, ctimeMs) < 0) { + code = TSDB_CODE_INVALID_MSG; + } + + tDecoderClear(pCoder); + taosArrayDestroy(pReq.deleteReqs); + + if (code) { + vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } else { + vTrace("vgId:%d %s done, ctimeMs generated:%" PRId64, TD_VID(pVnode), __func__, ctimeMs); + } + return code; +} + int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; @@ -283,52 +355,53 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { case TDMT_VND_CREATE_TABLE: { code = vnodePreProcessCreateTableMsg(pVnode, pMsg); } break; + case TDMT_VND_ALTER_TABLE: { + code = vnodePreProcessAlterTableMsg(pVnode, pMsg); + } break; case TDMT_VND_SUBMIT: { code = vnodePreProcessSubmitMsg(pVnode, pMsg); } break; case TDMT_VND_DELETE: { code = vnodePreProcessDeleteMsg(pVnode, pMsg); } break; + case TDMT_VND_BATCH_DEL: { + code = vnodePreProcessBatchDeleteMsg(pVnode, pMsg); + } break; default: break; } _exit: if (code) { - vError("vgId%d failed to preprocess write request since %s, msg type:%d", TD_VID(pVnode), tstrerror(code), - pMsg->msgType); + vError("vgId:%d, failed to preprocess write request since %s, msg type:%s", TD_VID(pVnode), tstrerror(code), + TMSG_INFO(pMsg->msgType)); } return code; } -int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) { +int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg *pRsp) { void *ptr = NULL; void *pReq; int32_t len; int32_t ret; - if (version <= pVnode->state.applied) { - vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version, + if (ver <= pVnode->state.applied) { + vError("vgId:%d, duplicate write request. ver: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), ver, pVnode->state.applied); terrno = TSDB_CODE_VND_DUP_REQUEST; return -1; } - vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), - version); + vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), ver); ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm); - ASSERT(pVnode->state.applied + 1 == version); + ASSERT(pVnode->state.applied + 1 == ver); - atomic_store_64(&pVnode->state.applied, version); + atomic_store_64(&pVnode->state.applied, ver); atomic_store_64(&pVnode->state.applyTerm, pMsg->info.conn.applyTerm); if (!syncUtilUserCommit(pMsg->msgType)) goto _exit; - if (pMsg->msgType == TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE || pMsg->msgType == TDMT_STREAM_TASK_CHECK_RSP) { - if (tqCheckLogInWal(pVnode->pTq, version)) return 0; - } - // skip header pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); len = pMsg->contLen - sizeof(SMsgHead); @@ -337,123 +410,108 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp switch (pMsg->msgType) { /* META */ case TDMT_VND_CREATE_STB: - if (vnodeProcessCreateStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessCreateStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_ALTER_STB: - if (vnodeProcessAlterStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessAlterStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_STB: - if (vnodeProcessDropStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDropStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_CREATE_TABLE: - if (vnodeProcessCreateTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessCreateTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_ALTER_TABLE: - if (vnodeProcessAlterTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessAlterTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_TABLE: - if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDropTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_TTL_TABLE: - if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDropTtlTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_TRIM: - if (vnodeProcessTrimReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessTrimReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_CREATE_SMA: - if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessCreateTSmaReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; /* TSDB */ case TDMT_VND_SUBMIT: - if (vnodeProcessSubmitReq(pVnode, version, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; + if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; break; case TDMT_VND_DELETE: - if (vnodeProcessDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_BATCH_DEL: - if (vnodeProcessBatchDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessBatchDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; break; /* TQ */ case TDMT_VND_TMQ_SUBSCRIBE: - if (tqProcessSubscribeReq(pVnode->pTq, version, pReq, len) < 0) { + if (tqProcessSubscribeReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } break; case TDMT_VND_TMQ_DELETE_SUB: - if (tqProcessDeleteSubReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + if (tqProcessDeleteSubReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } break; case TDMT_VND_TMQ_COMMIT_OFFSET: - if (tqProcessOffsetCommitReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { - goto _err; - } - break; - case TDMT_VND_TMQ_SEEK_TO_OFFSET: - if (tqProcessSeekReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { + if (tqProcessOffsetCommitReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } break; case TDMT_VND_TMQ_ADD_CHECKINFO: - if (tqProcessAddCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) { + if (tqProcessAddCheckInfoReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } break; case TDMT_VND_TMQ_DEL_CHECKINFO: - if (tqProcessDelCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) { + if (tqProcessDelCheckInfoReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } break; case TDMT_STREAM_TASK_DEPLOY: { - if (pVnode->restored && tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) { + if (pVnode->restored && tqProcessTaskDeployReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } } break; case TDMT_STREAM_TASK_DROP: { - if (tqProcessTaskDropReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + if (tqProcessTaskDropReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } } break; case TDMT_STREAM_TASK_PAUSE: { - if (pVnode->restored && tqProcessTaskPauseReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + if (pVnode->restored && tqProcessTaskPauseReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } } break; case TDMT_STREAM_TASK_RESUME: { - if (pVnode->restored && tqProcessTaskResumeReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { - goto _err; - } - } break; - case TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE: { - if (tqProcessTaskRecover2Req(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { - goto _err; - } - } break; - case TDMT_STREAM_TASK_CHECK_RSP: { - if (tqProcessStreamTaskCheckRsp(pVnode->pTq, version, pReq, len) < 0) { + if (pVnode->restored && tqProcessTaskResumeReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } } break; case TDMT_VND_ALTER_CONFIRM: needCommit = pVnode->config.hashChange; - if (vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp) < 0) { + if (vnodeProcessAlterConfirmReq(pVnode, ver, pReq, len, pRsp) < 0) { goto _err; } break; case TDMT_VND_ALTER_CONFIG: - vnodeProcessAlterConfigReq(pVnode, version, pReq, len, pRsp); + vnodeProcessAlterConfigReq(pVnode, ver, pReq, len, pRsp); break; case TDMT_VND_COMMIT: needCommit = true; break; case TDMT_VND_CREATE_INDEX: - vnodeProcessCreateIndexReq(pVnode, version, pReq, len, pRsp); + vnodeProcessCreateIndexReq(pVnode, ver, pReq, len, pRsp); break; case TDMT_VND_DROP_INDEX: - vnodeProcessDropIndexReq(pVnode, version, pReq, len, pRsp); + vnodeProcessDropIndexReq(pVnode, ver, pReq, len, pRsp); break; case TDMT_VND_COMPACT: - vnodeProcessCompactVnodeReq(pVnode, version, pReq, len, pRsp); + vnodeProcessCompactVnodeReq(pVnode, ver, pReq, len, pRsp); goto _exit; default: vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType); @@ -461,18 +519,18 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } vTrace("vgId:%d, process %s request, code:0x%x index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), pRsp->code, - version); + ver); - walApplyVer(pVnode->pWal, version); + walApplyVer(pVnode->pWal, ver); - if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { + if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, ver) < 0) { vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } // commit if need if (needCommit) { - vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); + vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), ver); if (vnodeAsyncCommit(pVnode) < 0) { vError("vgId:%d, failed to vnode async commit since %s.", TD_VID(pVnode), tstrerror(terrno)); goto _err; @@ -489,8 +547,8 @@ _exit: return 0; _err: - vError("vgId:%d, process %s request failed since %s, version:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), - tstrerror(terrno), version); + vError("vgId:%d, process %s request failed since %s, ver:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + tstrerror(terrno), ver); return -1; } @@ -504,7 +562,9 @@ int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in vnode query queue is processing"); - if ((pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_VND_TMQ_CONSUME) && !syncIsReadyForRead(pVnode->sync)) { + if ((pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_VND_TMQ_CONSUME || + pMsg->msgType == TDMT_VND_TMQ_CONSUME_PUSH) && + !syncIsReadyForRead(pVnode->sync)) { vnodeRedirectRpcMsg(pVnode, pMsg, terrno); return 0; } @@ -514,7 +574,9 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { return 0; } - SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; + SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; + initStorageAPI(&handle.api); + switch (pMsg->msgType) { case TDMT_SCH_QUERY: case TDMT_SCH_MERGE_QUERY: @@ -523,6 +585,8 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg, 0); case TDMT_VND_TMQ_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg); + case TDMT_VND_TMQ_CONSUME_PUSH: + return tqProcessPollPush(pVnode->pTq, pMsg); default: vError("unknown msg type:%d in query queue", pMsg->msgType); return TSDB_CODE_APP_ERROR; @@ -556,37 +620,60 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return vnodeGetTableCfg(pVnode, pMsg, true); case TDMT_VND_BATCH_META: return vnodeGetBatchMeta(pVnode, pMsg); - case TDMT_VND_TMQ_CONSUME: - return tqProcessPollReq(pVnode->pTq, pMsg); + // case TDMT_VND_TMQ_CONSUME: + // return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_VND_TMQ_VG_WALINFO: return tqProcessVgWalInfoReq(pVnode->pTq, pMsg); + case TDMT_VND_TMQ_VG_COMMITTEDINFO: + return tqProcessVgCommittedInfoReq(pVnode->pTq, pMsg); + case TDMT_VND_TMQ_SEEK: + return tqProcessSeekReq(pVnode->pTq, pMsg); + + default: + vError("unknown msg type:%d in fetch queue", pMsg->msgType); + return TSDB_CODE_APP_ERROR; + } +} + +int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { + vTrace("vgId:%d, msg:%p in fetch queue is processing", pVnode->config.vgId, pMsg); + if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG || + pMsg->msgType == TDMT_VND_BATCH_META) && + !syncIsReadyForRead(pVnode->sync)) { + vnodeRedirectRpcMsg(pVnode, pMsg, terrno); + return 0; + } + + switch (pMsg->msgType) { case TDMT_STREAM_TASK_RUN: return tqProcessTaskRunReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH: return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true); - case TDMT_STREAM_TASK_CHECK: - return tqProcessStreamTaskCheckReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg); + case TDMT_STREAM_TASK_CHECK: + return tqProcessStreamTaskCheckReq(pVnode->pTq, pMsg); + case TDMT_STREAM_TASK_CHECK_RSP: + return tqProcessStreamTaskCheckRsp(pVnode->pTq, 0, pMsg); case TDMT_STREAM_RETRIEVE: return tqProcessTaskRetrieveReq(pVnode->pTq, pMsg); case TDMT_STREAM_RETRIEVE_RSP: return tqProcessTaskRetrieveRsp(pVnode->pTq, pMsg); - case TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE: - return tqProcessTaskRecover1Req(pVnode->pTq, pMsg); - case TDMT_STREAM_RECOVER_FINISH: - return tqProcessTaskRecoverFinishReq(pVnode->pTq, pMsg); - case TDMT_STREAM_RECOVER_FINISH_RSP: - return tqProcessTaskRecoverFinishRsp(pVnode->pTq, pMsg); + case TDMT_VND_STREAM_SCAN_HISTORY: + return tqProcessTaskScanHistory(pVnode->pTq, pMsg); + case TDMT_STREAM_TRANSFER_STATE: + return tqProcessTaskTransferStateReq(pVnode->pTq, pMsg); + case TDMT_STREAM_SCAN_HISTORY_FINISH: + return tqProcessTaskScanHistoryFinishReq(pVnode->pTq, pMsg); + case TDMT_STREAM_SCAN_HISTORY_FINISH_RSP: + return tqProcessTaskScanHistoryFinishRsp(pVnode->pTq, pMsg); default: - vError("unknown msg type:%d in fetch queue", pMsg->msgType); + vError("unknown msg type:%d in stream queue", pMsg->msgType); return TSDB_CODE_APP_ERROR; } } -// TODO: remove the function void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { - // TODO // blockDebugShowDataBlocks(data, __func__); tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); } @@ -602,8 +689,9 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { pMetaRsp->precision = pVnode->config.tsdbCfg.precision; } -extern int32_t vnodeAsyncRentention(SVnode *pVnode, int64_t now); -static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +extern int32_t vnodeDoRetention(SVnode *pVnode, int64_t now); + +static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SVTrimDbReq trimReq = {0}; @@ -615,16 +703,13 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, vInfo("vgId:%d, trim vnode request will be processed, time:%d", pVnode->config.vgId, trimReq.timestamp); - // process - vnodeAsyncRentention(pVnode, trimReq.timestamp); - tsem_wait(&pVnode->canCommit); - tsem_post(&pVnode->canCommit); + code = vnodeDoRetention(pVnode, trimReq.timestamp); _exit: return code; } -static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY; @@ -634,8 +719,8 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p goto end; } - vDebug("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp); - int32_t ret = metaTtlDropTable(pVnode->pMeta, ttlReq.timestamp, tbUids); + vDebug("vgId:%d, drop ttl table req will be processed, time:%" PRId32, pVnode->config.vgId, ttlReq.timestampSec); + int32_t ret = metaTtlDropTable(pVnode->pMeta, (int64_t)ttlReq.timestampSec * 1000, tbUids); if (ret != 0) { goto end; } @@ -643,14 +728,14 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p tqUpdateTbUidList(pVnode->pTq, tbUids, false); } - vnodeAsyncRentention(pVnode, ttlReq.timestamp); + vnodeDoRetention(pVnode, ttlReq.timestampSec); end: taosArrayDestroy(tbUids); return ret; } -static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateStbReq req = {0}; SDecoder coder; @@ -667,7 +752,7 @@ static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *p goto _err; } - if (metaCreateSTable(pVnode->pMeta, version, &req) < 0) { + if (metaCreateSTable(pVnode->pMeta, ver, &req) < 0) { pRsp->code = terrno; goto _err; } @@ -685,7 +770,7 @@ _err: return -1; } -static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SDecoder decoder = {0}; SEncoder encoder = {0}; int32_t rcode = 0; @@ -742,7 +827,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pR } // do create table - if (metaCreateTable(pVnode->pMeta, version, pCreateReq, &cRsp.pMeta) < 0) { + if (metaCreateTable(pVnode->pMeta, ver, pCreateReq, &cRsp.pMeta) < 0) { if (pCreateReq->flags & TD_CREATE_IF_NOT_EXISTS && terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { cRsp.code = TSDB_CODE_SUCCESS; } else { @@ -790,7 +875,7 @@ _exit: return rcode; } -static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateStbReq req = {0}; SDecoder dc = {0}; @@ -808,7 +893,7 @@ static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pR return -1; } - if (metaAlterSTable(pVnode->pMeta, version, &req) < 0) { + if (metaAlterSTable(pVnode->pMeta, ver, &req) < 0) { pRsp->code = terrno; tDecoderClear(&dc); return -1; @@ -819,7 +904,7 @@ static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pR return 0; } -static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVDropStbReq req = {0}; int32_t rcode = TSDB_CODE_SUCCESS; SDecoder decoder = {0}; @@ -839,7 +924,7 @@ static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pRe // process request tbUidList = taosArrayInit(8, sizeof(int64_t)); if (tbUidList == NULL) goto _exit; - if (metaDropSTable(pVnode->pMeta, version, &req, tbUidList) < 0) { + if (metaDropSTable(pVnode->pMeta, ver, &req, tbUidList) < 0) { rcode = terrno; goto _exit; } @@ -862,7 +947,7 @@ _exit: return 0; } -static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVAlterTbReq vAlterTbReq = {0}; SVAlterTbRsp vAlterTbRsp = {0}; SDecoder dc = {0}; @@ -887,7 +972,7 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pRe } // process - if (metaAlterTable(pVnode->pMeta, version, &vAlterTbReq, &vMetaRsp) < 0) { + if (metaAlterTable(pVnode->pMeta, ver, &vAlterTbReq, &vMetaRsp) < 0) { vAlterTbRsp.code = terrno; tDecoderClear(&dc); rcode = -1; @@ -912,7 +997,7 @@ _exit: return 0; } -static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVDropTbBatchReq req = {0}; SVDropTbBatchRsp rsp = {0}; SDecoder decoder = {0}; @@ -946,7 +1031,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq tb_uid_t tbUid = 0; /* code */ - ret = metaDropTable(pVnode->pMeta, version, pDropTbReq, tbUids, &tbUid); + ret = metaDropTable(pVnode->pMeta, ver, pDropTbReq, tbUids, &tbUid); if (ret < 0) { if (pDropTbReq->igNotExists && terrno == TSDB_CODE_TDB_TABLE_NOT_EXIST) { dropTbRsp.code = TSDB_CODE_SUCCESS; @@ -1189,7 +1274,7 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) { return code; } -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; terrno = 0; @@ -1251,7 +1336,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq for (int32_t iRow = 0; iRow < pColData->nVal; iRow++) { if (aKey[iRow] < minKey || aKey[iRow] > maxKey || (iRow > 0 && aKey[iRow] <= aKey[iRow - 1])) { code = TSDB_CODE_INVALID_MSG; - vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), version); + vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), ver); goto _exit; } } @@ -1263,7 +1348,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq for (int32_t iRow = 0; iRow < nRow; ++iRow) { if (aRow[iRow]->ts < minKey || aRow[iRow]->ts > maxKey || (iRow > 0 && aRow[iRow]->ts <= aRow[iRow - 1]->ts)) { code = TSDB_CODE_INVALID_MSG; - vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), version); + vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), ver); goto _exit; } } @@ -1291,7 +1376,8 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq } if (info.suid) { - metaGetInfo(pVnode->pMeta, info.suid, &info, NULL); + code = metaGetInfo(pVnode->pMeta, info.suid, &info, NULL); + ASSERT(code == 0); } if (pSubmitTbData->sver != info.skmVer) { @@ -1350,7 +1436,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq SVCreateTbRsp *pCreateTbRsp = taosArrayReserve(pSubmitRsp->aCreateTbRsp, 1); // create table - if (metaCreateTable(pVnode->pMeta, version, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == 0) { + if (metaCreateTable(pVnode->pMeta, ver, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == 0) { // create table success if (newTbUids == NULL && @@ -1376,7 +1462,10 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq // insert data int32_t affectedRows; - code = tsdbInsertTableData(pVnode->pTsdb, version, pSubmitTbData, &affectedRows); + code = tsdbInsertTableData(pVnode->pTsdb, ver, pSubmitTbData, &affectedRows); + if (code) goto _exit; + + code = metaUpdateChangeTime(pVnode->pMeta, pSubmitTbData->uid, pSubmitTbData->ctimeMs); if (code) goto _exit; pSubmitRsp->affectedRows += affectedRows; @@ -1404,7 +1493,7 @@ _exit: atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); if (code == 0) { atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); - tdProcessRSmaSubmit(pVnode->pSma, version, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); + tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); } // clear @@ -1419,7 +1508,7 @@ _exit: return code; } -static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateTSmaReq req = {0}; SDecoder coder = {0}; @@ -1439,20 +1528,20 @@ static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void * goto _err; } - if (tdProcessTSmaCreate(pVnode->pSma, version, (const char *)&req) < 0) { + if (tdProcessTSmaCreate(pVnode->pSma, ver, (const char *)&req) < 0) { if (pRsp) pRsp->code = terrno; goto _err; } tDecoderClear(&coder); vDebug("vgId:%d, success to create tsma %s:%" PRIi64 " version %" PRIi64 " for table %" PRIi64, TD_VID(pVnode), - req.indexName, req.indexUid, version, req.tableUid); + req.indexName, req.indexUid, ver, req.tableUid); return 0; _err: tDecoderClear(&coder); vError("vgId:%d, failed to create tsma %s:%" PRIi64 " version %" PRIi64 "for table %" PRIi64 " since %s", - TD_VID(pVnode), req.indexName, req.indexUid, version, req.tableUid, terrstr()); + TD_VID(pVnode), req.indexName, req.indexUid, ver, req.tableUid, terrstr()); return -1; } @@ -1468,28 +1557,28 @@ int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen) { return vnodeProcessCreateTSmaReq(pVnode, 1, pCont, contLen, NULL); } -static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t version) { +static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t ver) { int32_t code = TSDB_CODE_SUCCESS; vInfo("vgId:%d, trim meta of tables per hash range [%" PRIu32 ", %" PRIu32 "]. apply-index:%" PRId64, TD_VID(pVnode), - pVnode->config.hashBegin, pVnode->config.hashEnd, version); + pVnode->config.hashBegin, pVnode->config.hashEnd, ver); // TODO: trim meta of tables from TDB per hash range [pVnode->config.hashBegin, pVnode->config.hashEnd] + code = metaTrimTables(pVnode->pMeta); return code; } -static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { vInfo("vgId:%d, vnode handle msgType:alter-confirm, alter confim msg is processed", TD_VID(pVnode)); int32_t code = TSDB_CODE_SUCCESS; if (!pVnode->config.hashChange) { goto _exit; } - code = vnodeConsolidateAlterHashRange(pVnode, version); + code = vnodeConsolidateAlterHashRange(pVnode, ver); if (code < 0) { - vError("vgId:%d, failed to consolidate alter hashrange since %s. version:%" PRId64, TD_VID(pVnode), terrstr(), - version); + vError("vgId:%d, failed to consolidate alter hashrange since %s. version:%" PRId64, TD_VID(pVnode), terrstr(), ver); goto _exit; } pVnode->config.hashChange = false; @@ -1503,7 +1592,7 @@ _exit: return code; } -static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { bool walChanged = false; bool tsdbChanged = false; @@ -1606,14 +1695,14 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void return 0; } -static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SBatchDeleteReq deleteReq; SDecoder decoder; tDecoderInit(&decoder, pReq, len); tDecodeSBatchDeleteReq(&decoder, &deleteReq); SMetaReader mr = {0}; - metaReaderInit(&mr, pVnode->pMeta, META_READER_NOLOCK); + metaReaderDoInit(&mr, pVnode->pMeta, META_READER_NOLOCK); int32_t sz = taosArrayGetSize(deleteReq.deleteReqs); for (int32_t i = 0; i < sz; i++) { @@ -1626,13 +1715,21 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void int64_t uid = mr.me.uid; - int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs); + int32_t code = tsdbDeleteTableData(pVnode->pTsdb, ver, deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs); if (code < 0) { terrno = code; vError("vgId:%d, delete error since %s, suid:%" PRId64 ", uid:%" PRId64 ", start ts:%" PRId64 ", end ts:%" PRId64, TD_VID(pVnode), terrstr(), deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs); } + code = metaUpdateChangeTime(pVnode->pMeta, uid, deleteReq.ctimeMs); + if (code < 0) { + terrno = code; + vError("vgId:%d, update change time error since %s, suid:%" PRId64 ", uid:%" PRId64 ", start ts:%" PRId64 + ", end ts:%" PRId64, + TD_VID(pVnode), terrstr(), deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs); + } + tDecoderClear(&mr.coder); } metaReaderClear(&mr); @@ -1640,7 +1737,7 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void return 0; } -static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SDecoder *pCoder = &(SDecoder){0}; SDeleteRes *pRes = &(SDeleteRes){0}; @@ -1661,8 +1758,10 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq ASSERT(taosArrayGetSize(pRes->uidList) == 0 || (pRes->skey != 0 && pRes->ekey != 0)); for (int32_t iUid = 0; iUid < taosArrayGetSize(pRes->uidList); iUid++) { - code = tsdbDeleteTableData(pVnode->pTsdb, version, pRes->suid, *(uint64_t *)taosArrayGet(pRes->uidList, iUid), - pRes->skey, pRes->ekey); + uint64_t uid = *(uint64_t *)taosArrayGet(pRes->uidList, iUid); + code = tsdbDeleteTableData(pVnode->pTsdb, ver, pRes->suid, uid, pRes->skey, pRes->ekey); + if (code) goto _err; + code = metaUpdateChangeTime(pVnode->pMeta, uid, pRes->ctimeMs); if (code) goto _err; } @@ -1682,7 +1781,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq _err: return code; } -static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateStbReq req = {0}; SDecoder dc = {0}; @@ -1698,7 +1797,7 @@ static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void tDecoderClear(&dc); return -1; } - if (metaAddIndexToSTable(pVnode->pMeta, version, &req) < 0) { + if (metaAddIndexToSTable(pVnode->pMeta, ver, &req) < 0) { pRsp->code = terrno; goto _err; } @@ -1708,7 +1807,7 @@ _err: tDecoderClear(&dc); return -1; } -static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { SDropIndexReq req = {0}; pRsp->msgType = TDMT_VND_DROP_INDEX_RSP; pRsp->code = TSDB_CODE_SUCCESS; @@ -1720,21 +1819,21 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *p return -1; } - if (metaDropIndexFromSTable(pVnode->pMeta, version, &req) < 0) { + if (metaDropIndexFromSTable(pVnode->pMeta, ver, &req) < 0) { pRsp->code = terrno; return -1; } return TSDB_CODE_SUCCESS; } -extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - return vnodeProcessCompactVnodeReqImpl(pVnode, version, pReq, len, pRsp); +static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { + return vnodeProcessCompactVnodeReqImpl(pVnode, ver, pReq, len, pRsp); } #ifndef TD_ENTERPRISE -int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; } #endif diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 4ea5e3c6ec269012f60e3b0187db5741ec5c4831..360da41482c43cc13236b104052220c87bd216cf 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -431,7 +431,7 @@ static int32_t vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsm return tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg); } -static int32_t vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +static int32_t vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { if (pMsg->code == 0) { return vnodeSyncApplyMsg(pFsm, pMsg, pMeta); } @@ -451,7 +451,7 @@ static int32_t vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFs return 0; } -static int32_t vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +static int32_t vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { if (pMeta->isWeak == 1) { return vnodeSyncApplyMsg(pFsm, pMsg, pMeta); } @@ -463,7 +463,7 @@ static SyncIndex vnodeSyncAppliedIndex(const SSyncFSM *pFSM) { return atomic_load_64(&pVnode->state.applied); } -static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { SVnode *pVnode = pFsm->data; vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", weak:%d, code:%d, state:%d %s, type:%s", pVnode->config.vgId, pFsm, pMeta->index, pMeta->isWeak, pMeta->code, pMeta->state, syncStr(pMeta->state), @@ -551,10 +551,10 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) // start to restore all stream tasks if (tsDisableStream) { - vInfo("vgId:%d, not restore stream tasks, since disabled", pVnode->config.vgId); + vInfo("vgId:%d, not launch stream tasks, since stream tasks are disabled", pVnode->config.vgId); } else { - vInfo("vgId:%d start to restore stream tasks", pVnode->config.vgId); - tqStartStreamTasks(pVnode->pTq); + vInfo("vgId:%d start to launch stream tasks", pVnode->config.vgId); + tqCheckStreamStatus(pVnode->pTq); } } diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index e18be94ace94c35a486a7ebd8c54e4eb3ae2e009..4a95629d59ec2ce34388814f2eb2b566cfd9a2b2 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -17,4 +17,5 @@ add_subdirectory(executor) add_subdirectory(stream) add_subdirectory(planner) add_subdirectory(qworker) +add_subdirectory(geometry) add_subdirectory(command) \ No newline at end of file diff --git a/source/libs/cache/test/cacheTests.cpp b/source/libs/cache/test/cacheTests.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/source/libs/catalog/CMakeLists.txt b/source/libs/catalog/CMakeLists.txt index bc56ff4c7f766862f2f70057d09af56d2a9f8895..6f09af8a8f38392ed674636cf0574fddb8143c2b 100644 --- a/source/libs/catalog/CMakeLists.txt +++ b/source/libs/catalog/CMakeLists.txt @@ -11,6 +11,6 @@ target_link_libraries( PRIVATE os util transport qcom nodes ) -if(${BUILD_TEST}) - ADD_SUBDIRECTORY(test) -endif(${BUILD_TEST}) +# if(${BUILD_TEST}) +# ADD_SUBDIRECTORY(test) +# endif(${BUILD_TEST}) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index c548a6c696e96601e306a7891ec9280985654a76..7d47e82164fdbd5b77a5eb70637957ae048ef7bb 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -23,6 +23,8 @@ extern "C" { #include "catalog.h" #include "query.h" #include "tcommon.h" +#include "ttimer.h" +#include "tglobal.h" #define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 #define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100 @@ -34,6 +36,8 @@ extern "C" { #define CTG_DEFAULT_BATCH_NUM 64 #define CTG_DEFAULT_FETCH_NUM 8 #define CTG_MAX_COMMAND_LEN 512 +#define CTG_DEFAULT_CACHE_MON_MSEC 5000 +#define CTG_CLEAR_CACHE_ROUND_TB_NUM 3000 #define CTG_RENT_SLOT_SECOND 1.5 @@ -82,6 +86,7 @@ enum { enum { CTG_OP_UPDATE_VGROUP = 0, + CTG_OP_UPDATE_DB_CFG, CTG_OP_UPDATE_TB_META, CTG_OP_DROP_DB_CACHE, CTG_OP_DROP_DB_VGROUP, @@ -130,6 +135,7 @@ typedef struct SCtgDebug { typedef struct SCtgCacheStat { uint64_t cacheNum[CTG_CI_MAX_VALUE]; + uint64_t cacheSize[CTG_CI_MAX_VALUE]; uint64_t cacheHit[CTG_CI_MAX_VALUE]; uint64_t cacheNHit[CTG_CI_MAX_VALUE]; } SCtgCacheStat; @@ -238,8 +244,8 @@ typedef STableIndexRsp STableIndex; typedef struct SCtgTbCache { SRWLatch metaLock; - STableMeta* pMeta; SRWLatch indexLock; + STableMeta* pMeta; STableIndex* pIndex; } SCtgTbCache; @@ -248,20 +254,27 @@ typedef struct SCtgVgCache { SDBVgInfo* vgInfo; } SCtgVgCache; +typedef struct SCtgCfgCache { + SRWLatch cfgLock; + SDbCfgInfo* cfgInfo; +} SCtgCfgCache; + typedef struct SCtgDBCache { - SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads - uint64_t dbId; - int8_t deleted; - SCtgVgCache vgCache; - SHashObj* tbCache; // key:tbname, value:SCtgTbCache - SHashObj* stbCache; // key:suid, value:char* - uint64_t dbCacheNum[CTG_CI_MAX_VALUE]; + SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads + uint64_t dbId; + int8_t deleted; + SCtgVgCache vgCache; + SCtgCfgCache cfgCache; + SHashObj* tbCache; // key:tbname, value:SCtgTbCache + SHashObj* stbCache; // key:suid, value:char* + uint64_t dbCacheNum[CTG_CI_MAX_VALUE]; + uint64_t dbCacheSize; } SCtgDBCache; typedef struct SCtgRentSlot { SRWLatch lock; bool needSort; - SArray* meta; // element is SDbVgVersion or SSTableVersion + SArray* meta; // element is SDbCacheInfo or SSTableVersion } SCtgRentSlot; typedef struct SCtgRentMgmt { @@ -269,12 +282,15 @@ typedef struct SCtgRentMgmt { uint16_t slotNum; uint16_t slotRIdx; int64_t lastReadMsec; + uint64_t rentCacheSize; + int32_t metaSize; SCtgRentSlot* slots; } SCtgRentMgmt; typedef struct SCtgUserAuth { SRWLatch lock; SGetUserAuthRsp userAuth; + uint64_t userCacheSize; } SCtgUserAuth; typedef struct SCatalog { @@ -405,6 +421,7 @@ typedef struct SCtgRuntimeStat { uint64_t numOfOpAbort; uint64_t numOfOpEnqueue; uint64_t numOfOpDequeue; + uint64_t numOfOpClearMeta; uint64_t numOfOpClearCache; } SCtgRuntimeStat; @@ -425,6 +442,13 @@ typedef struct SCtgUpdateVgMsg { SDBVgInfo* dbInfo; } SCtgUpdateVgMsg; +typedef struct SCtgUpdateDbCfgMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + uint64_t dbId; + SDbCfgInfo* cfgInfo; +} SCtgUpdateDbCfgMsg; + typedef struct SCtgUpdateTbMetaMsg { SCatalog* pCtg; STableMetaOutput* pMeta; @@ -474,6 +498,7 @@ typedef struct SCtgDropTbIndexMsg { typedef struct SCtgClearCacheMsg { SCatalog* pCtg; + bool clearMeta; bool freeCtg; } SCtgClearCacheMsg; @@ -512,6 +537,8 @@ typedef struct SCatalogMgmt { int32_t jobPool; SRWLatch lock; SCtgQueue queue; + void *timer; + tmr_h cacheTimer; TdThread updateThread; SHashObj* pCluster; // key: clusterId, value: SCatalog* SCatalogStat statInfo; @@ -528,8 +555,8 @@ typedef struct SCtgOperation { } SCtgOperation; typedef struct SCtgCacheItemInfo { - char* name; - int32_t flag; + char* name; + int32_t flag; } SCtgCacheItemInfo; #define CTG_AUTH_READ(_t) ((_t) == AUTH_TYPE_READ || (_t) == AUTH_TYPE_READ_OR_WRITE) @@ -542,11 +569,6 @@ typedef struct SCtgCacheItemInfo { #define CTG_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n) #define CTG_STAT_GET(_item) atomic_load_64(&(_item)) -#define CTG_DB_NUM_INC(_item) dbCache->dbCacheNum[_item] += 1 -#define CTG_DB_NUM_DEC(_item) dbCache->dbCacheNum[_item] -= 1 -#define CTG_DB_NUM_SET(_item) dbCache->dbCacheNum[_item] = 1 -#define CTG_DB_NUM_RESET(_item) dbCache->dbCacheNum[_item] = 0 - #define CTG_STAT_API_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.api.item, n)) #define CTG_STAT_RT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.runtime.item, n)) #define CTG_STAT_NUM_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.cache.cacheNum[item], n)) @@ -561,6 +583,11 @@ typedef struct SCtgCacheItemInfo { #define CTG_CACHE_HIT_INC(item, n) (CTG_STAT_INC(pCtg->cacheStat.cacheHit[item], n)) #define CTG_CACHE_NHIT_INC(item, n) (CTG_STAT_INC(pCtg->cacheStat.cacheNHit[item], n)) +#define CTG_DB_NUM_INC(_item) dbCache->dbCacheNum[_item] += 1 +#define CTG_DB_NUM_DEC(_item) dbCache->dbCacheNum[_item] -= 1 +#define CTG_DB_NUM_SET(_item) dbCache->dbCacheNum[_item] = 1 +#define CTG_DB_NUM_RESET(_item) dbCache->dbCacheNum[_item] = 0 + #define CTG_META_NUM_INC(type) \ do { \ switch (type) { \ @@ -671,6 +698,10 @@ typedef struct SCtgCacheItemInfo { #define CTG_DB_NOT_EXIST(code) \ (code == TSDB_CODE_MND_DB_NOT_EXIST || code == TSDB_CODE_MND_DB_IN_CREATING || code == TSDB_CODE_MND_DB_IN_DROPPING) +#define CTG_CACHE_OVERFLOW(_csize, _maxsize) ((_maxsize >= 0) ? ((_csize) >= (_maxsize) * 1048576L * 0.9) : false) +#define CTG_CACHE_LOW(_csize, _maxsize) ((_maxsize >= 0) ? ((_csize) <= (_maxsize) * 1048576L * 0.75) : true) + + #define ctgFatal(param, ...) qFatal("CTG:%p " param, pCtg, __VA_ARGS__) #define ctgError(param, ...) qError("CTG:%p " param, pCtg, __VA_ARGS__) #define ctgWarn(param, ...) qWarn("CTG:%p " param, pCtg, __VA_ARGS__) @@ -773,6 +804,12 @@ typedef struct SCtgCacheItemInfo { CTG_RET(__code); \ } while (0) +#define CTG_API_NLEAVE() \ + do { \ + CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); \ + CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); \ + } while (0) + #define CTG_API_ENTER() \ do { \ CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); \ @@ -782,6 +819,15 @@ typedef struct SCtgCacheItemInfo { } \ } while (0) +#define CTG_API_NENTER() \ + do { \ + CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); \ + CTG_LOCK(CTG_READ, &gCtgMgmt.lock); \ + if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { \ + CTG_API_NLEAVE(); \ + } \ + } while (0) + #define CTG_API_JENTER() \ do { \ CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); \ @@ -815,8 +861,10 @@ int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t* fetchIdx, int32_t baseResIdx, SArray* pList); +void* ctgCloneDbCfgInfo(void* pSrc); int32_t ctgOpUpdateVgroup(SCtgCacheOperation* action); +int32_t ctgOpUpdateDbCfg(SCtgCacheOperation *operation); int32_t ctgOpUpdateTbMeta(SCtgCacheOperation* action); int32_t ctgOpDropDbCache(SCtgCacheOperation* action); int32_t ctgOpDropDbVgroup(SCtgCacheOperation* action); @@ -838,12 +886,13 @@ int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, bool syncReq); int32_t ctgDropTbMetaEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, const char* tbName, bool syncReq); int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq); +int32_t ctgUpdateDbCfgEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SDbCfgInfo *cfgInfo, bool syncOp); int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput* output, bool syncReq); int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp* pAuth, bool syncReq); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char* dbFName, int32_t vgId, SEpSet* pEpSet); int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex** pIndex, bool syncOp); -int32_t ctgClearCacheEnqueue(SCatalog* pCtg, bool freeCtg, bool stopQueue, bool syncOp); -int32_t ctgMetaRentInit(SCtgRentMgmt* mgmt, uint32_t rentSec, int8_t type); +int32_t ctgClearCacheEnqueue(SCatalog* pCtg, bool clearMeta, bool freeCtg, bool stopQueue, bool syncOp); +int32_t ctgMetaRentInit(SCtgRentMgmt* mgmt, uint32_t rentSec, int8_t type, int32_t size); int32_t ctgMetaRentAdd(SCtgRentMgmt* mgmt, void* meta, int64_t id, int32_t size); int32_t ctgMetaRentGet(SCtgRentMgmt* mgmt, void** res, uint32_t* num, int32_t size); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); @@ -889,7 +938,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const void* param); int32_t ctgLaunchJob(SCtgJob* pJob); int32_t ctgMakeAsyncRes(SCtgJob* pJob); -int32_t ctgLaunchSubTask(SCtgTask* pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param); +int32_t ctgLaunchSubTask(SCtgTask** ppTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param); int32_t ctgGetTbCfgCb(SCtgTask* pTask); void ctgFreeHandle(SCatalog* pCatalog); @@ -901,17 +950,17 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) int32_t ctgGenerateVgList(SCatalog* pCtg, SHashObj* vgHash, SArray** pList); void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); -int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); -int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, +int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); +int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update); int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, int32_t* vgId); void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); -int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2); +int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2); int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); -int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2); +int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); @@ -924,7 +973,7 @@ void ctgFreeSTableIndex(void* info); void ctgClearSubTaskRes(SCtgSubRes* pRes); void ctgFreeQNode(SCtgQNode* node); void ctgClearHandle(SCatalog* pCtg); -void ctgFreeTbCacheImpl(SCtgTbCache* pCache); +void ctgFreeTbCacheImpl(SCtgTbCache* pCache, bool lock); int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); @@ -932,6 +981,9 @@ SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); int32_t ctgdGetOneHandle(SCatalog** pHandle); int ctgVgInfoComp(const void* lp, const void* rp); int32_t ctgMakeVgArray(SDBVgInfo* dbInfo); +int32_t ctgChkSetAuthRes(SCatalog *pCtg, SCtgAuthReq *req, SCtgAuthRsp* res); +int32_t ctgReadDBCfgFromCache(SCatalog *pCtg, const char* dbFName, SDbCfgInfo* pDbCfg); + int32_t ctgAcquireVgMetaFromCache(SCatalog* pCtg, const char* dbFName, const char* tbName, SCtgDBCache** pDb, SCtgTbCache** pTb); int32_t ctgCopyTbMeta(SCatalog* pCtg, SCtgTbMetaCtx* ctx, SCtgDBCache** pDb, SCtgTbCache** pTb, STableMeta** pTableMeta, @@ -940,6 +992,16 @@ void ctgReleaseVgMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCach void ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCache* pCache); void ctgGetGlobalCacheStat(SCtgCacheStat* pStat); int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res); +void ctgGetGlobalCacheSize(uint64_t *pSize); +uint64_t ctgGetTbIndexCacheSize(STableIndex *pIndex); +uint64_t ctgGetTbMetaCacheSize(STableMeta *pMeta); +uint64_t ctgGetDbVgroupCacheSize(SDBVgInfo *pVg); +uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth); +uint64_t ctgGetClusterCacheSize(SCatalog *pCtg); +void ctgClearHandleMeta(SCatalog* pCtg, int64_t *pClearedSize, int64_t *pCleardNum, bool *roundDone); +void ctgClearAllHandleMeta(int64_t *clearedSize, int64_t *clearedNum, bool *roundDone); +void ctgProcessTimerEvent(void *param, void *tmrId); + int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName); int32_t ctgGetTbTagCb(SCtgTask* pTask); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 976a38c03d0d2a857802d18a9a30adf3227cf907..f975517669d4fff1ce3125fa18cc058dbbe9ea42 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -341,13 +341,10 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pReq, SCtgAuthReq req = {0}; req.pRawReq = pReq; req.pConn = pConn; - req.onlyCache = exists ? true : false; + req.onlyCache = false; CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pReq->user, &req.authInfo, NULL)); CTG_ERR_JRET(ctgChkSetAuthRes(pCtg, &req, &rsp)); - if (rsp.metaNotExists && exists) { - *exists = false; - } _return: @@ -568,7 +565,7 @@ int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* return TSDB_CODE_SUCCESS; } - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pConn ? &pConn->mgmtEps : NULL, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup)); _return: @@ -629,7 +626,7 @@ int32_t ctgGetCachedTbVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInf return TSDB_CODE_SUCCESS; } - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, NULL, dbCache->vgCache.vgInfo, pTableName, pVgroup)); ctgRUnlockVgInfo(dbCache); @@ -668,7 +665,55 @@ _return: CTG_RET(code); } +void ctgProcessTimerEvent(void *param, void *tmrId) { + CTG_API_NENTER(); + + int32_t cacheMaxSize = atomic_load_32(&tsMetaCacheMaxSize); + if (cacheMaxSize >= 0) { + uint64_t cacheSize = 0; + ctgGetGlobalCacheSize(&cacheSize); + bool overflow = CTG_CACHE_OVERFLOW(cacheSize, cacheMaxSize); + + qDebug("catalog cache size: %" PRIu64"B, maxCaseSize:%dMB, %s", cacheSize, cacheMaxSize, overflow ? "overflow" : "NO overflow"); + + if (overflow) { + int32_t code = ctgClearCacheEnqueue(NULL, true, false, false, false); + if (code) { + qError("clear cache enqueue failed, error:%s", tstrerror(code)); + taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + } + + goto _return; + } + } + + qTrace("reset catalog timer"); + taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + +_return: + + CTG_API_NLEAVE(); +} + +int32_t ctgGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, SDbCfgInfo* pDbCfg) { + CTG_ERR_RET(ctgReadDBCfgFromCache(pCtg, dbFName, pDbCfg)); + + if (pDbCfg->cfgVersion < 0) { + CTG_ERR_RET(ctgGetDBCfgFromMnode(pCtg, pConn, dbFName, pDbCfg, NULL)); + SDbCfgInfo *pCfg = ctgCloneDbCfgInfo(pDbCfg); + if (NULL == pCfg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + CTG_ERR_RET(ctgUpdateDbCfgEnqueue(pCtg, dbFName, pDbCfg->dbId, pCfg, false)); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t catalogInit(SCatalogCfg* cfg) { + qDebug("catalogInit start"); if (gCtgMgmt.pCluster) { qError("catalog already initialized"); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); @@ -726,6 +771,18 @@ int32_t catalogInit(SCatalogCfg* cfg) { CTG_ERR_RET(terrno); } + gCtgMgmt.timer = taosTmrInit(0, 0, 0, "catalog"); + if (NULL == gCtgMgmt.timer) { + qError("init timer failed, error:%s", tstrerror(terrno)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + gCtgMgmt.cacheTimer = taosTmrStart(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer); + if (NULL == gCtgMgmt.cacheTimer) { + qError("start cache timer failed"); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgStartUpdateThread()); qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, @@ -769,8 +826,8 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { clusterCtg->clusterId = clusterId; - CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB)); - CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); + CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo))); + CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion))); clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); @@ -958,6 +1015,23 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogUpdateDbCfg(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDbCfgInfo* cfgInfo) { + CTG_API_ENTER(); + + int32_t code = 0; + + if (NULL == pCtg || NULL == dbFName || NULL == cfgInfo) { + freeDbCfgInfo(cfgInfo); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + + code = ctgUpdateDbCfgEnqueue(pCtg, dbFName, dbId, cfgInfo, false); + +_return: + + CTG_API_LEAVE(code); +} + int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { CTG_API_ENTER(); @@ -1105,6 +1179,22 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogAsyncUpdateTableMeta(SCatalog* pCtg, STableMetaRsp* pMsg) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pMsg) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgUpdateTbMeta(pCtg, pMsg, false)); + +_return: + + CTG_API_LEAVE(code); +} + + int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* pTables) { CTG_API_ENTER(); @@ -1371,14 +1461,14 @@ int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableVersion** stables, uint3 CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void**)stables, num, sizeof(SSTableVersion))); } -int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion** dbs, uint32_t* num) { +int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbCacheInfo** dbs, uint32_t* num) { CTG_API_ENTER(); if (NULL == pCtg || NULL == dbs || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void**)dbs, num, sizeof(SDbVgVersion))); + CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void**)dbs, num, sizeof(SDbCacheInfo))); } int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num) { @@ -1426,9 +1516,7 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbF CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_CACHE_NHIT_INC(CTG_CI_DB_CFG, 1); - - CTG_API_LEAVE(ctgGetDBCfgFromMnode(pCtg, pConn, dbFName, pDbCfg, NULL)); + CTG_API_LEAVE(ctgGetDBCfg(pCtg, pConn, dbFName, pDbCfg)); } int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char* indexName, SIndexInfo* pInfo) { @@ -1568,11 +1656,11 @@ int32_t catalogClearCache(void) { qInfo("start to clear catalog cache"); - if (NULL == gCtgMgmt.pCluster || atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + if (NULL == gCtgMgmt.pCluster) { CTG_API_LEAVE_NOLOCK(TSDB_CODE_SUCCESS); } - int32_t code = ctgClearCacheEnqueue(NULL, false, false, true); + int32_t code = ctgClearCacheEnqueue(NULL, false, false, false, true); qInfo("clear catalog cache end, code: %s", tstrerror(code)); @@ -1586,10 +1674,17 @@ void catalogDestroy(void) { return; } + if (gCtgMgmt.cacheTimer) { + taosTmrStop(gCtgMgmt.cacheTimer); + gCtgMgmt.cacheTimer = NULL; + taosTmrCleanUp(gCtgMgmt.timer); + gCtgMgmt.timer = NULL; + } + atomic_store_8((int8_t*)&gCtgMgmt.exit, true); if (!taosCheckCurrentInDll()) { - ctgClearCacheEnqueue(NULL, true, true, true); + ctgClearCacheEnqueue(NULL, false, true, true, true); taosThreadJoin(gCtgMgmt.updateThread, NULL); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 9e654e89d93f3046064ca671f06965570f3c7134..fb5ecf7ad219b029c6d2d256192dc4b8f5d983b4 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1112,7 +1112,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, pOut->dbVgroup, pName, &vgInfo)); ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); @@ -1132,7 +1132,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, dbCache->vgCache.vgInfo, pName, &vgInfo)); ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); @@ -1282,7 +1282,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, pOut->dbVgroup, pName, &vgInfo)); ctgTaskDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); @@ -1302,7 +1302,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, dbCache->vgCache.vgInfo, pName, &vgInfo)); ctgTaskDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); @@ -1501,7 +1501,7 @@ int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, ctx->pName, (SVgroupInfo*)pTask->res)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pTask->pJob->conn.mgmtEps, pOut->dbVgroup, ctx->pName, (SVgroupInfo*)pTask->res)); CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, ctx->dbFName, pOut->dbId, pOut->dbVgroup, false)); pOut->dbVgroup = NULL; @@ -1536,7 +1536,7 @@ int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); - CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, tReq, pOut->dbVgroup, ctx, pMsgCtx->target, pReq->pTables, true)); + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, &pTask->pJob->conn.mgmtEps, tReq, pOut->dbVgroup, ctx, pMsgCtx->target, pReq->pTables, true)); CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, pMsgCtx->target, pOut->dbId, pOut->dbVgroup, false)); pOut->dbVgroup = NULL; @@ -1587,7 +1587,6 @@ int32_t ctgHandleGetTbIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu CTG_ERR_JRET(ctgCloneTableIndex(pOut->pIndex, &pInfo)); pTask->res = pInfo; - SCtgTbIndexCtx* ctx = pTask->taskCtx; CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pTask->pJob->pCtg, (STableIndex**)&pTask->msgCtx.out, false)); _return: @@ -1660,8 +1659,14 @@ _return: int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + SCtgDbCfgCtx* ctx = pTask->taskCtx; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + SDbCfgInfo* pCfg = ctgCloneDbCfgInfo(pTask->msgCtx.out); + + CTG_ERR_RET(ctgUpdateDbCfgEnqueue(pTask->pJob->pCtg, ctx->dbFName, pCfg->dbId, pCfg, false)); + TSWAP(pTask->res, pTask->msgCtx.out); _return: @@ -1794,7 +1799,7 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTaskReq* tReq, int32_t flag, SName* pName, int CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, dbCache->vgCache.vgInfo, pName, &vgInfo)); ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); @@ -1943,7 +1948,7 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask* pTask) { if (NULL == pTask->res) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, dbCache->vgCache.vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); ctgReleaseVgInfoToCache(pCtg, dbCache); dbCache = NULL; @@ -1991,7 +1996,7 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask* pTask) { tReq.pTask = pTask; tReq.msgIdx = -1; CTG_ERR_JRET( - ctgGetVgInfosFromHashValue(pCtg, &tReq, dbCache->vgCache.vgInfo, pCtx, pReq->dbFName, pReq->pTables, false)); + ctgGetVgInfosFromHashValue(pCtg, &pConn->mgmtEps, &tReq, dbCache->vgCache.vgInfo, pCtx, pReq->dbFName, pReq->pTables, false)); ctgReleaseVgInfoToCache(pCtg, dbCache); dbCache = NULL; @@ -2085,25 +2090,25 @@ int32_t ctgLaunchGetTbCfgTask(SCtgTask* pTask) { } CTG_CACHE_NHIT_INC(CTG_CI_TBL_CFG, 1); - + if (pCtx->tbType <= 0) { CTG_ERR_JRET(ctgReadTbTypeFromCache(pCtg, dbFName, pCtx->pName->tname, &pCtx->tbType)); if (pCtx->tbType <= 0) { SCtgTbMetaParam param; param.pName = pCtx->pName; param.flag = 0; - CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, ¶m)); + CTG_ERR_JRET(ctgLaunchSubTask(&pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, ¶m)); return TSDB_CODE_SUCCESS; } } - if (TSDB_SUPER_TABLE == pCtx->tbType) { + if (TSDB_SUPER_TABLE == pCtx->tbType || TSDB_SYSTEM_TABLE == pCtx->tbType) { CTG_ERR_JRET(ctgGetTableCfgFromMnode(pCtg, pConn, pCtx->pName, NULL, pTask)); } else { if (NULL == pCtx->pVgInfo) { CTG_ERR_JRET(ctgGetTbHashVgroupFromCache(pCtg, pCtx->pName, &pCtx->pVgInfo)); if (NULL == pCtx->pVgInfo) { - CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbCfgCb, dbFName)); + CTG_ERR_JRET(ctgLaunchSubTask(&pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbCfgCb, dbFName)); return TSDB_CODE_SUCCESS; } } @@ -2140,7 +2145,7 @@ int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) { if (NULL == pCtx->pVgInfo) { CTG_ERR_JRET(ctgGetTbHashVgroupFromCache(pCtg, pCtx->pName, &pCtx->pVgInfo)); if (NULL == pCtx->pVgInfo) { - CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbTagCb, dbFName)); + CTG_ERR_JRET(ctgLaunchSubTask(&pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbTagCb, dbFName)); return TSDB_CODE_SUCCESS; } } @@ -2195,13 +2200,24 @@ int32_t ctgLaunchGetDbCfgTask(SCtgTask* pTask) { SCtgDbCfgCtx* pCtx = (SCtgDbCfgCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + SDbCfgInfo cfgInfo; if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } - CTG_CACHE_NHIT_INC(CTG_CI_DB_CFG, 1); + CTG_ERR_RET(ctgReadDBCfgFromCache(pCtg, pCtx->dbFName, &cfgInfo)); - CTG_ERR_RET(ctgGetDBCfgFromMnode(pCtg, pConn, pCtx->dbFName, NULL, pTask)); + if (cfgInfo.cfgVersion < 0) { + CTG_ERR_RET(ctgGetDBCfgFromMnode(pCtg, pConn, pCtx->dbFName, NULL, pTask)); + } else { + pTask->res = taosMemoryCalloc(1, sizeof(SDbCfgInfo)); + if (NULL == pTask->res) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(pTask->res, &cfgInfo, sizeof(cfgInfo)); + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + } return TSDB_CODE_SUCCESS; } @@ -2315,7 +2331,7 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) { SCtgTbMetaParam param; param.pName = &pCtx->user.tbName; param.flag = CTG_FLAG_SYNC_OP; - CTG_ERR_RET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetUserCb, ¶m)); + CTG_ERR_RET(ctgLaunchSubTask(&pTask, CTG_TASK_GET_TB_META, ctgGetUserCb, ¶m)); } else { CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pCtx->user.user, NULL, pTask)); } @@ -2359,7 +2375,7 @@ int32_t ctgGetTbCfgCb(SCtgTask* pTask) { SDBVgInfo* pDb = (SDBVgInfo*)pTask->subRes.res; pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, pDb, pCtx->pName, pCtx->pVgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, &pTask->pJob->conn.mgmtEps, pDb, pCtx->pName, pCtx->pVgInfo)); } CTG_RET(ctgLaunchGetTbCfgTask(pTask)); @@ -2379,7 +2395,7 @@ int32_t ctgGetTbTagCb(SCtgTask* pTask) { if (NULL == pCtx->pVgInfo) { pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, pDb, pCtx->pName, pCtx->pVgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, &pTask->pJob->conn.mgmtEps, pDb, pCtx->pName, pCtx->pVgInfo)); } CTG_RET(ctgLaunchGetTbTagTask(pTask)); @@ -2525,19 +2541,35 @@ _return: CTG_RET(code); } -int32_t ctgLaunchSubTask(SCtgTask* pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param) { - SCtgJob* pJob = pTask->pJob; +SCtgTask* ctgGetTask(SCtgJob* pJob, int32_t taskId) { + int32_t taskNum = taosArrayGetSize(pJob->pTasks); + + for (int32_t i = 0; i < taskNum; ++i) { + SCtgTask* pTask = taosArrayGet(pJob->pTasks, i); + if (pTask->taskId == taskId) { + return pTask; + } + } + + return NULL; +} + + +int32_t ctgLaunchSubTask(SCtgTask** ppTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param) { + SCtgJob* pJob = (*ppTask)->pJob; int32_t subTaskId = -1; bool newTask = false; + int32_t taskId = (*ppTask)->taskId; - ctgClearSubTaskRes(&pTask->subRes); - pTask->subRes.type = type; - pTask->subRes.fp = fp; + ctgClearSubTaskRes(&(*ppTask)->subRes); + (*ppTask)->subRes.type = type; + (*ppTask)->subRes.fp = fp; CTG_ERR_RET(ctgSearchExistingTask(pJob, type, param, &subTaskId)); if (subTaskId < 0) { CTG_ERR_RET(ctgInitTask(pJob, type, param, &subTaskId)); newTask = true; + *ppTask = ctgGetTask(pJob, taskId); } SCtgTask* pSub = taosArrayGet(pJob->pTasks, subTaskId); @@ -2545,10 +2577,10 @@ int32_t ctgLaunchSubTask(SCtgTask* pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, pSub->subTask = true; } - CTG_ERR_RET(ctgSetSubTaskCb(pSub, pTask)); + CTG_ERR_RET(ctgSetSubTaskCb(pSub, *ppTask)); if (newTask) { - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(*ppTask, -1); SCtgMsgCtx* pSubMsgCtx = CTG_GET_TASK_MSGCTX(pSub, -1); pSubMsgCtx->pBatchs = pMsgCtx->pBatchs; @@ -2568,6 +2600,7 @@ int32_t ctgLaunchJob(SCtgJob* pJob) { qDebug("QID:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); + pTask = taosArrayGet(pJob->pTasks, i); pTask->status = CTG_TASK_LAUNCHED; } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 433eacd30e92c8d6347ff7c91a496f328fa4c554..44de83b7ef142c5cfc277854d196cde332b3895c 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -20,6 +20,7 @@ #include "trpc.h" SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = {{CTG_OP_UPDATE_VGROUP, "update vgInfo", ctgOpUpdateVgroup}, + {CTG_OP_UPDATE_DB_CFG, "update dbCfg", ctgOpUpdateDbCfg}, {CTG_OP_UPDATE_TB_META, "update tbMeta", ctgOpUpdateTbMeta}, {CTG_OP_DROP_DB_CACHE, "drop DB", ctgOpDropDbCache}, {CTG_OP_DROP_DB_VGROUP, "drop DBVgroup", ctgOpDropDbVgroup}, @@ -32,24 +33,25 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = {{CTG_OP_UPDATE_VGROUP, "update v {CTG_OP_CLEAR_CACHE, "clear cache", ctgOpClearCache}}; SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = { - {"Cluster ", CTG_CI_FLAG_LEVEL_GLOBAL}, // CTG_CI_CLUSTER - {"Dnode ", CTG_CI_FLAG_LEVEL_CLUSTER}, // CTG_CI_DNODE, - {"Qnode ", CTG_CI_FLAG_LEVEL_CLUSTER}, // CTG_CI_QNODE, - {"DB ", CTG_CI_FLAG_LEVEL_CLUSTER}, // CTG_CI_DB, - {"DbVgroup ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_DB_VGROUP, - {"DbCfg ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_DB_CFG, - {"DbInfo ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_DB_INFO, - {"StbMeta ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_STABLE_META, - {"NtbMeta ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_NTABLE_META, - {"CtbMeta ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_CTABLE_META, - {"SysTblMeta", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_SYSTABLE_META, - {"OthTblMeta", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_OTHERTABLE_META, - {"TblSMA ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_TBL_SMA, - {"TblCfg ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_TBL_CFG, - {"IndexInfo ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_INDEX_INFO, - {"User ", CTG_CI_FLAG_LEVEL_CLUSTER}, // CTG_CI_USER, - {"UDF ", CTG_CI_FLAG_LEVEL_CLUSTER}, // CTG_CI_UDF, - {"SvrVer ", CTG_CI_FLAG_LEVEL_CLUSTER} // CTG_CI_SVR_VER, + {"Cluster ", CTG_CI_FLAG_LEVEL_GLOBAL}, //CTG_CI_CLUSTER + {"Dnode ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_DNODE, + {"Qnode ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_QNODE, + {"DB ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_DB, + {"DbVgroup ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_DB_VGROUP, + {"DbCfg ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_DB_CFG, + {"DbInfo ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_DB_INFO, + {"StbMeta ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_STABLE_META, + {"NtbMeta ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_NTABLE_META, + {"CtbMeta ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_CTABLE_META, + {"SysTblMeta", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_SYSTABLE_META, + {"OthTblMeta", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_OTHERTABLE_META, + {"TblSMA ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_TBL_SMA, + {"TblCfg ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_TBL_CFG, + {"TblTag ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_TBL_TAG, + {"IndexInfo ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_INDEX_INFO, + {"User ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_USER, + {"UDF ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_UDF, + {"SvrVer ", CTG_CI_FLAG_LEVEL_CLUSTER} //CTG_CI_SVR_VER, }; int32_t ctgRLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { @@ -89,10 +91,15 @@ int32_t ctgWLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { return TSDB_CODE_SUCCESS; } -void ctgRUnlockVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); } +void ctgRLockDbCfgInfo(SCtgDBCache *dbCache) { CTG_LOCK(CTG_READ, &dbCache->cfgCache.cfgLock); } +void ctgWLockDbCfgInfo(SCtgDBCache *dbCache) { CTG_LOCK(CTG_WRITE, &dbCache->cfgCache.cfgLock); } +void ctgRUnlockVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); } void ctgWUnlockVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); } +void ctgRUnlockDbCfgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->cfgCache.cfgLock); } +void ctgWUnlockDbCfgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_WRITE, &dbCache->cfgCache.cfgLock); } + void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->dbLock); taosHashRelease(pCtg->dbCache, dbCache); @@ -703,6 +710,43 @@ _return: CTG_RET(code); } +int32_t ctgReadDBCfgFromCache(SCatalog *pCtg, const char* dbFName, SDbCfgInfo* pDbCfg) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + pDbCfg->cfgVersion = -1; + CTG_CACHE_NHIT_INC(CTG_CI_DB_CFG, 1); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->cfgCache.cfgLock); + + if (dbCache->cfgCache.cfgInfo) { + SDbCfgInfo *pInfo = ctgCloneDbCfgInfo(dbCache->cfgCache.cfgInfo); + if (NULL == pInfo) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(pDbCfg, pInfo, sizeof(*pInfo)); + taosMemoryFree(pInfo); + CTG_CACHE_HIT_INC(CTG_CI_DB_CFG, 1); + } else { + pDbCfg->cfgVersion = -1; + CTG_CACHE_NHIT_INC(CTG_CI_DB_CFG, 1); + } + +_return: + + if (dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->cfgCache.cfgLock); + ctgReleaseDBCache(pCtg, dbCache); + } + + return code; +} + int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName) { *stbName = NULL; @@ -726,14 +770,9 @@ int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid return TSDB_CODE_SUCCESS; } + int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp *pRes) { int32_t code = 0; - if (IS_SYS_DBNAME(pReq->tbName.dbname)) { - *inCache = true; - pRes->pRawRes->pass = true; - ctgDebug("sysdb %s, pass", pReq->tbName.dbname); - return TSDB_CODE_SUCCESS; - } SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, pReq->user, strlen(pReq->user)); if (NULL == pUser) { @@ -816,7 +855,7 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); - ctgDebug("action [%s] added into queue", opName); + ctgDebug("%sync action [%s] added into queue", syncOp ? "S": "As", opName); CTG_QUEUE_INC(); CTG_STAT_RT_INC(numOfOpEnqueue, 1); @@ -1006,6 +1045,44 @@ _return: CTG_RET(code); } + +int32_t ctgUpdateDbCfgEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SDbCfgInfo *cfgInfo, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + op->opId = CTG_OP_UPDATE_DB_CFG; + op->syncOp = syncOp; + + SCtgUpdateDbCfgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateDbCfgMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateDbCfgMsg)); + taosMemoryFree(op); + freeDbCfgInfo(cfgInfo); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + char *p = strchr(dbFName, '.'); + if (p && IS_SYS_DBNAME(p + 1)) { + dbFName = p + 1; + } + + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->pCtg = pCtg; + msg->dbId = dbId; + msg->cfgInfo = cfgInfo; + + op->data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; + +_return: + + freeDbCfgInfo(cfgInfo); + CTG_RET(code); +} + + int32_t ctgUpdateTbMetaEnqueue(SCatalog *pCtg, STableMetaOutput *output, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); @@ -1160,7 +1237,7 @@ _return: CTG_RET(code); } -int32_t ctgClearCacheEnqueue(SCatalog *pCtg, bool freeCtg, bool stopQueue, bool syncOp) { +int32_t ctgClearCacheEnqueue(SCatalog *pCtg, bool clearMeta, bool freeCtg, bool stopQueue, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_CLEAR_CACHE; @@ -1176,6 +1253,7 @@ int32_t ctgClearCacheEnqueue(SCatalog *pCtg, bool freeCtg, bool stopQueue, bool } msg->pCtg = pCtg; + msg->clearMeta = clearMeta; msg->freeCtg = freeCtg; op->data = msg; @@ -1188,10 +1266,11 @@ _return: CTG_RET(code); } -int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { +int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type, int32_t size) { mgmt->slotRIdx = 0; mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; mgmt->type = type; + mgmt->metaSize = size; size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; @@ -1201,6 +1280,8 @@ int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + mgmt->rentCacheSize = msgSize; + qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum); return TSDB_CODE_SUCCESS; @@ -1227,6 +1308,7 @@ int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + mgmt->rentCacheSize += size; slot->needSort = true; qDebug("add meta to rent, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); @@ -1307,6 +1389,7 @@ int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortComp } taosArrayRemove(slot->meta, idx); + mgmt->rentCacheSize -= mgmt->metaSize; qDebug("meta in rent removed, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); @@ -1419,15 +1502,15 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { CTG_CACHE_NUM_INC(CTG_CI_DB, 1); - SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1, .stateTs = 0}; - tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + SDbCacheInfo dbCacheInfo = {.dbId = newDBCache.dbId, .vgVersion = -1, .stateTs = 0, .cfgVersion = -1}; + tstrncpy(dbCacheInfo.dbFName, dbFName, sizeof(dbCacheInfo.dbFName)); ctgDebug("db added to cache, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId); if (!IS_SYS_DBNAME(dbFName)) { - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &dbCacheInfo, dbId, sizeof(SDbCacheInfo))); - ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, dbId); + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%" PRIx64, dbFName, dbCacheInfo.vgVersion, dbId); } return TSDB_CODE_SUCCESS; @@ -1471,7 +1554,7 @@ int32_t ctgRemoveDBFromCache(SCatalog *pCtg, SCtgDBCache *dbCache, const char *d CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock); - CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbId, ctgDbCacheInfoSortCompare, ctgDbCacheInfoSearchCompare)); ctgDebug("db removed from rent, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId); if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { @@ -1574,10 +1657,15 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam } if (origType == TSDB_SUPER_TABLE) { - if (taosHashRemove(dbCache->stbCache, &orig->suid, sizeof(orig->suid))) { - ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:0x%" PRIx64, dbFName, tbName, orig->suid); - } else { - ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:0x%" PRIx64, dbFName, tbName, orig->suid); + char *stbName = taosHashGet(dbCache->stbCache, &orig->suid, sizeof(orig->suid)); + if (stbName) { + uint64_t metaSize = strlen(stbName) + 1 + sizeof(orig->suid); + if (taosHashRemove(dbCache->stbCache, &orig->suid, sizeof(orig->suid))) { + ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:0x%" PRIx64, dbFName, tbName, orig->suid); + } else { + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:0x%" PRIx64, dbFName, tbName, orig->suid); + atomic_sub_fetch_64(&dbCache->dbCacheSize, metaSize); + } } } } @@ -1591,14 +1679,20 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(meta)); + pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); } else { CTG_LOCK(CTG_WRITE, &pCache->metaLock); if (orig) { CTG_META_NUM_DEC(origType); } + + atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(meta) - ctgGetTbMetaCacheSize(pCache->pMeta)); + taosMemoryFree(pCache->pMeta); pCache->pMeta = meta; + CTG_UNLOCK(CTG_WRITE, &pCache->metaLock); } @@ -1616,12 +1710,12 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + atomic_add_fetch_64(&dbCache->dbCacheSize, sizeof(meta->suid) + strlen(tbName) + 1); + ctgDebug("stb 0x%" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, meta->tableType); - if (pCache) { - CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); - } + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); return TSDB_CODE_SUCCESS; } @@ -1648,6 +1742,8 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(SCtgTbCache) + ctgGetTbIndexCacheSize(pIndex)); + CTG_DB_NUM_INC(CTG_CI_TBL_SMA); *index = NULL; @@ -1664,6 +1760,7 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa CTG_LOCK(CTG_WRITE, &pCache->indexLock); if (pCache->pIndex) { + atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbIndexCacheSize(pCache->pIndex)); if (0 == suid) { suid = pCache->pIndex->suid; } @@ -1674,6 +1771,8 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa pCache->pIndex = pIndex; CTG_UNLOCK(CTG_WRITE, &pCache->indexLock); + atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbIndexCacheSize(pIndex)); + *index = NULL; ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, @@ -1703,7 +1802,7 @@ _return: CTG_RET(code); } -void ctgClearAllInstance(void) { +void ctgClearAllHandles(void) { SCatalog *pCtg = NULL; void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); @@ -1718,7 +1817,7 @@ void ctgClearAllInstance(void) { } } -void ctgFreeAllInstance(void) { +void ctgFreeAllHandles(void) { SCatalog *pCtg = NULL; void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); @@ -1759,15 +1858,15 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { goto _return; } - if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { + if (dbInfo->vgVersion < 0 || (taosHashGetSize(dbInfo->vgHash) <= 0 && !IS_SYS_DBNAME(dbFName))) { ctgDebug("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } bool newAdded = false; - SDbVgVersion vgVersion = { - .dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable, .stateTs = dbInfo->stateTs}; + SDbCacheInfo dbCacheInfo = { + .dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .cfgVersion = -1, .numOfTable = dbInfo->numOfTable, .stateTs = dbInfo->stateTs}; SCtgDBCache *dbCache = NULL; CTG_ERR_JRET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); @@ -1777,6 +1876,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { } SCtgVgCache *vgCache = &dbCache->vgCache; + CTG_ERR_JRET(ctgWLockVgInfo(msg->pCtg, dbCache)); if (vgCache->vgInfo) { @@ -1799,25 +1899,38 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { goto _return; } + uint64_t groupCacheSize = ctgGetDbVgroupCacheSize(vgCache->vgInfo); + ctgDebug("sub dbGroupCacheSize %" PRIu64 " from db, dbFName:%s", groupCacheSize, dbFName); + + atomic_sub_fetch_64(&dbCache->dbCacheSize, groupCacheSize); + freeVgInfo(vgInfo); CTG_DB_NUM_RESET(CTG_CI_DB_VGROUP); } + if (dbCache->cfgCache.cfgInfo) { + dbCacheInfo.cfgVersion = dbCache->cfgCache.cfgInfo->cfgVersion; + } + vgCache->vgInfo = dbInfo; msg->dbInfo = NULL; CTG_DB_NUM_SET(CTG_CI_DB_VGROUP); ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, stateTs:%" PRId64 ", dbId:0x%" PRIx64, dbFName, - vgVersion.vgVersion, vgVersion.stateTs, vgVersion.dbId); + dbCacheInfo.vgVersion, dbCacheInfo.stateTs, dbCacheInfo.dbId); ctgWUnlockVgInfo(dbCache); + uint64_t groupCacheSize = ctgGetDbVgroupCacheSize(vgCache->vgInfo); + atomic_add_fetch_64(&dbCache->dbCacheSize, groupCacheSize); + ctgDebug("add dbGroupCacheSize %" PRIu64 " from db, dbFName:%s", groupCacheSize, dbFName); + dbCache = NULL; // if (!IS_SYS_DBNAME(dbFName)) { - tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), - ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + tstrncpy(dbCacheInfo.dbFName, dbFName, sizeof(dbCacheInfo.dbFName)); + CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &dbCacheInfo, dbCacheInfo.dbId, sizeof(SDbCacheInfo), + ctgDbCacheInfoSortCompare, ctgDbCacheInfoSearchCompare)); //} _return: @@ -1828,6 +1941,67 @@ _return: CTG_RET(code); } +int32_t ctgOpUpdateDbCfg(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgUpdateDbCfgMsg *msg = operation->data; + SDbCfgInfo *cfgInfo = msg->cfgInfo; + char *dbFName = msg->dbFName; + SCatalog *pCtg = msg->pCtg; + + if (pCtg->stopUpdate || NULL == cfgInfo) { + goto _return; + } + + if (cfgInfo->cfgVersion < 0) { + ctgDebug("invalid db cfgInfo, dbFName:%s, cfgVersion:%d", dbFName, cfgInfo->cfgVersion); + CTG_ERR_JRET(TSDB_CODE_APP_ERROR); + } + + SCtgDBCache *dbCache = NULL; + CTG_ERR_JRET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%" PRIx64, dbFName, msg->dbId); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SDbCacheInfo cacheInfo = {0}; + cacheInfo.dbId = dbCache->dbId; + tstrncpy(cacheInfo.dbFName, dbFName, sizeof(cacheInfo.dbFName)); + cacheInfo.cfgVersion = cfgInfo->cfgVersion; + + SCtgVgCache *vgCache = &dbCache->vgCache; + if (vgCache->vgInfo) { + cacheInfo.vgVersion = vgCache->vgInfo->vgVersion; + cacheInfo.numOfTable = vgCache->vgInfo->numOfTable; + cacheInfo.stateTs = vgCache->vgInfo->stateTs; + } else { + cacheInfo.vgVersion = -1; + } + + ctgWLockDbCfgInfo(dbCache); + + freeDbCfgInfo(dbCache->cfgCache.cfgInfo); + dbCache->cfgCache.cfgInfo = cfgInfo; + cfgInfo = NULL; + + ctgWUnlockDbCfgInfo(dbCache); + + ctgDebug("db cfgInfo updated, dbFName:%s, cfgVer:%d", dbFName, dbCache->cfgCache.cfgInfo->cfgVersion); + + // if (!IS_SYS_DBNAME(dbFName)) { + CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &cacheInfo, cacheInfo.dbId, sizeof(SDbCacheInfo), + ctgDbCacheInfoSortCompare, ctgDbCacheInfoSearchCompare)); + //} + +_return: + + freeDbCfgInfo(cfgInfo); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + int32_t ctgOpDropDbCache(SCtgCacheOperation *operation) { int32_t code = 0; SCtgDropDBMsg *msg = operation->data; @@ -1875,6 +2049,8 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { CTG_ERR_JRET(ctgWLockVgInfo(pCtg, dbCache)); + atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetDbVgroupCacheSize(dbCache->vgCache.vgInfo)); + freeVgInfo(dbCache->vgCache.vgInfo); dbCache->vgCache.vgInfo = NULL; @@ -1966,26 +2142,32 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { goto _return; } - if (taosHashRemove(dbCache->stbCache, &msg->suid, sizeof(msg->suid))) { - ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:0x%" PRIx64, msg->dbFName, - msg->stbName, msg->suid); + char *stbName = taosHashGet(dbCache->stbCache, &msg->suid, sizeof(msg->suid)); + if (stbName) { + uint64_t metaSize = strlen(stbName) + 1 + sizeof(msg->suid); + if (taosHashRemove(dbCache->stbCache, &msg->suid, sizeof(msg->suid))) { + ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:0x%" PRIx64, msg->dbFName, + msg->stbName, msg->suid); + } else { + atomic_sub_fetch_64(&dbCache->dbCacheSize, metaSize); + } } - + SCtgTbCache *pTbCache = taosHashGet(dbCache->tbCache, msg->stbName, strlen(msg->stbName)); if (NULL == pTbCache) { ctgDebug("stb %s already not in cache", msg->stbName); goto _return; } - CTG_LOCK(CTG_WRITE, &pTbCache->metaLock); tblType = pTbCache->pMeta->tableType; - ctgFreeTbCacheImpl(pTbCache); - CTG_UNLOCK(CTG_WRITE, &pTbCache->metaLock); + atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(pTbCache->pMeta) + ctgGetTbIndexCacheSize(pTbCache->pIndex)); + ctgFreeTbCacheImpl(pTbCache, true); if (taosHashRemove(dbCache->tbCache, msg->stbName, strlen(msg->stbName))) { ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:0x%" PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_META_NUM_DEC(tblType); + atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(*pTbCache) + strlen(msg->stbName)); } ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:0x%" PRIx64, msg->dbFName, msg->stbName, msg->suid); @@ -2029,15 +2211,15 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { goto _return; } - CTG_LOCK(CTG_WRITE, &pTbCache->metaLock); tblType = pTbCache->pMeta->tableType; - ctgFreeTbCacheImpl(pTbCache); - CTG_UNLOCK(CTG_WRITE, &pTbCache->metaLock); + atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(pTbCache->pMeta) + ctgGetTbIndexCacheSize(pTbCache->pIndex)); + ctgFreeTbCacheImpl(pTbCache, true); if (taosHashRemove(dbCache->tbCache, msg->tbName, strlen(msg->tbName))) { ctgError("tb %s not exist in cache, dbFName:%s", msg->tbName, msg->dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } else { + atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(*pTbCache) + strlen(msg->tbName)); CTG_META_NUM_DEC(tblType); } @@ -2064,7 +2246,8 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { SCtgUserAuth userAuth = {0}; memcpy(&userAuth.userAuth, &msg->userAuth, sizeof(msg->userAuth)); - + userAuth.userCacheSize = ctgGetUserCacheSize(&userAuth.userAuth); + if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); @@ -2080,30 +2263,24 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { CTG_LOCK(CTG_WRITE, &pUser->lock); taosHashCleanup(pUser->userAuth.createdDbs); - pUser->userAuth.createdDbs = msg->userAuth.createdDbs; - msg->userAuth.createdDbs = NULL; - taosHashCleanup(pUser->userAuth.readDbs); - pUser->userAuth.readDbs = msg->userAuth.readDbs; - msg->userAuth.readDbs = NULL; - taosHashCleanup(pUser->userAuth.writeDbs); - pUser->userAuth.writeDbs = msg->userAuth.writeDbs; - msg->userAuth.writeDbs = NULL; - taosHashCleanup(pUser->userAuth.readTbs); - pUser->userAuth.readTbs = msg->userAuth.readTbs; - msg->userAuth.readTbs = NULL; - taosHashCleanup(pUser->userAuth.writeTbs); - pUser->userAuth.writeTbs = msg->userAuth.writeTbs; - msg->userAuth.writeTbs = NULL; - taosHashCleanup(pUser->userAuth.useDbs); - pUser->userAuth.useDbs = msg->userAuth.useDbs; + + memcpy(&pUser->userAuth, &msg->userAuth, sizeof(msg->userAuth)); + + msg->userAuth.createdDbs = NULL; + msg->userAuth.readDbs = NULL; + msg->userAuth.writeDbs = NULL; + msg->userAuth.readTbs = NULL; + msg->userAuth.writeTbs = NULL; msg->userAuth.useDbs = NULL; CTG_UNLOCK(CTG_WRITE, &pUser->lock); + + atomic_store_64(&pUser->userCacheSize, ctgGetUserCacheSize(&pUser->userAuth)); _return: @@ -2239,33 +2416,78 @@ _return: CTG_RET(code); } -int32_t ctgOpClearCache(SCtgCacheOperation *operation) { - int32_t code = 0; +void ctgClearFreeCache(SCtgCacheOperation *operation) { SCtgClearCacheMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; CTG_LOCK(CTG_WRITE, &gCtgMgmt.lock); - + if (pCtg) { if (msg->freeCtg) { ctgFreeHandle(pCtg); } else { ctgClearHandle(pCtg); } + } else if (msg->freeCtg) { + ctgFreeAllHandles(); + } else { + ctgClearAllHandles(); + } + + CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.lock); +} - goto _return; +void ctgClearMetaCache(SCtgCacheOperation *operation) { + SCtgClearCacheMsg *msg = operation->data; + SCatalog *pCtg = msg->pCtg; + int64_t clearedSize = 0; + int64_t clearedNum = 0; + int64_t remainSize = 0; + bool roundDone = false; + + if (pCtg) { + ctgClearHandleMeta(pCtg, &clearedSize, &clearedNum, &roundDone); + } else { + ctgClearAllHandleMeta(&clearedSize, &clearedNum, &roundDone); + } + + qDebug("catalog finish one round meta clear, clearedSize:%" PRId64 ", clearedNum:%" PRId64 ", done:%d", clearedSize, clearedNum, roundDone); + + ctgGetGlobalCacheSize(&remainSize); + int32_t cacheMaxSize = atomic_load_32(&tsMetaCacheMaxSize); + + if (CTG_CACHE_LOW(remainSize, cacheMaxSize)) { + qDebug("catalog finish meta clear, remainSize:%" PRId64 ", cacheMaxSize:%dMB", remainSize, cacheMaxSize); + taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + return; + } + + if (!roundDone) { + qDebug("catalog all meta cleared, remainSize:%" PRId64 ", cacheMaxSize:%dMB, to clear handle", remainSize, cacheMaxSize); + ctgClearFreeCache(operation); + taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + return; + } + + int32_t code = ctgClearCacheEnqueue(NULL, true, false, false, false); + if (code) { + qError("clear cache enqueue failed, error:%s", tstrerror(code)); + taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); } +} - if (msg->freeCtg) { - ctgFreeAllInstance(); +int32_t ctgOpClearCache(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgClearCacheMsg *msg = operation->data; + + if (msg->clearMeta) { + ctgClearMetaCache(operation); } else { - ctgClearAllInstance(); + ctgClearFreeCache(operation); } _return: - CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.lock); - taosMemoryFreeClear(msg); CTG_RET(code); @@ -2759,7 +2981,7 @@ int32_t ctgGetTbHashVgroupFromCache(SCatalog *pCtg, const SName *pTableName, SVg } *pVgroup = taosMemoryCalloc(1, sizeof(SVgroupInfo)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pTableName, *pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, NULL, dbCache->vgCache.vgInfo, pTableName, *pVgroup)); _return: diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 12ff8a7b388a994e8fd0249a557e993768934342..7cba6ddf0da240ce17e2794487c0d70181cecf27 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -501,6 +501,25 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) { } } + if (dbCache->cfgCache.cfgInfo) { + SDbCfgInfo *pCfg = dbCache->cfgCache.cfgInfo; + ctgDebug("[%d] db [%.*s][0x%" PRIx64 + "] %s: cfgVersion:%d, numOfVgroups:%d, numOfStables:%d, buffer:%d, cacheSize:%d, pageSize:%d, pages:%d" + ", daysPerFile:%d, daysToKeep0:%d, daysToKeep1:%d, daysToKeep2:%d, minRows:%d, maxRows:%d, walFsyncPeriod:%d" + ", hashPrefix:%d, hashSuffix:%d, walLevel:%d, precision:%d, compression:%d, replications:%d, strict:%d" + ", cacheLast:%d, tsdbPageSize:%d, walRetentionPeriod:%d, walRollPeriod:%d, walRetentionSize:%" PRId64 "" + ", walSegmentSize:%" PRId64 ", numOfRetensions:%d, schemaless:%d, sstTrigger:%d", + i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", + pCfg->cfgVersion, pCfg->numOfVgroups, pCfg->numOfStables, pCfg->buffer, + pCfg->cacheSize, pCfg->pageSize, pCfg->pages, pCfg->daysPerFile, pCfg->daysToKeep0, + pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->minRows, pCfg->maxRows, pCfg->walFsyncPeriod, + pCfg->hashPrefix, pCfg->hashSuffix, pCfg->walLevel, pCfg->precision, pCfg->compression, + pCfg->replications, pCfg->strict, pCfg->cacheLast, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, + pCfg->walRollPeriod, pCfg->walRetentionSize, pCfg->walSegmentSize, pCfg->numOfRetensions, + pCfg->schemaless, pCfg->sstTrigger); + } + + ++i; pIter = taosHashIterate(dbHash, pIter); } } @@ -528,7 +547,10 @@ int32_t ctgdShowStatInfo(void) { CTG_API_ENTER(); SCtgCacheStat cache; + uint64_t cacheSize = 0; + ctgGetGlobalCacheStat(&cache); + ctgGetGlobalCacheSize(&cacheSize); qDebug("## Global Stat Info %s ##", "begin"); qDebug("## \t%s \t%s \t%s ##", "Num", "Hit", "Nhit"); @@ -536,6 +558,7 @@ int32_t ctgdShowStatInfo(void) { qDebug("# %s \t%" PRIu64 " \t%" PRIu64 " \t%" PRIu64 " #", gCtgStatItem[i].name, cache.cacheNum[i], cache.cacheHit[i], cache.cacheNHit[i]); } qDebug("## Global Stat Info %s ##", "end"); + qDebug("## Global Cache Size: %" PRIu64, cacheSize); CTG_API_LEAVE(TSDB_CODE_SUCCESS); } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 9274c1ef9254b4f15010b912dda571d0a5e40bed..dab007aa47729e7e32f4550fca7f3d60e502d149 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -199,6 +199,7 @@ void ctgFreeMetaRent(SCtgRentMgmt* mgmt) { } taosMemoryFreeClear(mgmt->slots); + mgmt->rentCacheSize = 0; } void ctgFreeStbMetaCache(SCtgDBCache* dbCache) { @@ -211,12 +212,26 @@ void ctgFreeStbMetaCache(SCtgDBCache* dbCache) { dbCache->stbCache = NULL; } -void ctgFreeTbCacheImpl(SCtgTbCache* pCache) { - qDebug("tbMeta freed, p:%p", pCache->pMeta); - taosMemoryFreeClear(pCache->pMeta); +void ctgFreeTbCacheImpl(SCtgTbCache* pCache, bool lock) { + if (pCache->pMeta) { + if (lock) { + CTG_LOCK(CTG_WRITE, &pCache->metaLock); + } + taosMemoryFreeClear(pCache->pMeta); + if (lock) { + CTG_UNLOCK(CTG_WRITE, &pCache->metaLock); + } + } + if (pCache->pIndex) { + if (lock) { + CTG_LOCK(CTG_WRITE, &pCache->indexLock); + } taosArrayDestroyEx(pCache->pIndex->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(pCache->pIndex); + if (lock) { + CTG_UNLOCK(CTG_WRITE, &pCache->indexLock); + } } } @@ -228,7 +243,7 @@ void ctgFreeTbCache(SCtgDBCache* dbCache) { int32_t tblNum = taosHashGetSize(dbCache->tbCache); SCtgTbCache* pCache = taosHashIterate(dbCache->tbCache, NULL); while (NULL != pCache) { - ctgFreeTbCacheImpl(pCache); + ctgFreeTbCacheImpl(pCache, false); pCache = taosHashIterate(dbCache->tbCache, pCache); } taosHashCleanup(dbCache->tbCache); @@ -236,6 +251,7 @@ void ctgFreeTbCache(SCtgDBCache* dbCache) { } void ctgFreeVgInfoCache(SCtgDBCache* dbCache) { freeVgInfo(dbCache->vgCache.vgInfo); } +void ctgFreeCfgInfoCache(SCtgDBCache* dbCache) { freeDbCfgInfo(dbCache->cfgCache.cfgInfo); } void ctgFreeDbCache(SCtgDBCache* dbCache) { if (NULL == dbCache) { @@ -243,6 +259,7 @@ void ctgFreeDbCache(SCtgDBCache* dbCache) { } ctgFreeVgInfoCache(dbCache); + ctgFreeCfgInfoCache(dbCache); ctgFreeStbMetaCache(dbCache); ctgFreeTbCache(dbCache); } @@ -314,21 +331,86 @@ void ctgFreeHandle(SCatalog* pCtg) { ctgInfo("handle freed, clusterId:0x%" PRIx64, clusterId); } +void ctgClearHandleMeta(SCatalog* pCtg, int64_t *pClearedSize, int64_t *pCleardNum, bool *roundDone) { + int64_t cacheSize = 0; + void* pIter = taosHashIterate(pCtg->dbCache, NULL); + while (pIter) { + SCtgDBCache* dbCache = pIter; + + SCtgTbCache* pCache = taosHashIterate(dbCache->tbCache, NULL); + while (NULL != pCache) { + size_t len = 0; + void* key = taosHashGetKey(pCache, &len); + + if (pCache->pMeta && TSDB_SUPER_TABLE == pCache->pMeta->tableType) { + pCache = taosHashIterate(dbCache->tbCache, pCache); + continue; + } + + taosHashRemove(dbCache->tbCache, key, len); + cacheSize = len + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(pCache->pMeta) + ctgGetTbIndexCacheSize(pCache->pIndex); + atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); + *pClearedSize += cacheSize; + (*pCleardNum)++; + + if (pCache->pMeta) { + CTG_META_NUM_DEC(pCache->pMeta->tableType); + } + + ctgFreeTbCacheImpl(pCache, true); + + if (*pCleardNum >= CTG_CLEAR_CACHE_ROUND_TB_NUM) { + taosHashCancelIterate(dbCache->tbCache, pCache); + goto _return; + } + + pCache = taosHashIterate(dbCache->tbCache, pCache); + } + + pIter = taosHashIterate(pCtg->dbCache, pIter); + } + +_return: + + if (*pCleardNum >= CTG_CLEAR_CACHE_ROUND_TB_NUM) { + *roundDone = true; + } +} + +void ctgClearAllHandleMeta(int64_t *clearedSize, int64_t *clearedNum, bool *roundDone) { + SCatalog *pCtg = NULL; + + void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + while (pIter) { + pCtg = *(SCatalog **)pIter; + + if (pCtg) { + ctgClearHandleMeta(pCtg, clearedSize, clearedNum, roundDone); + if (*roundDone) { + taosHashCancelIterate(gCtgMgmt.pCluster, pIter); + break; + } + } + + pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); + } +} + void ctgClearHandle(SCatalog* pCtg) { if (NULL == pCtg) { return; } uint64_t clusterId = pCtg->clusterId; - + ctgFreeMetaRent(&pCtg->dbRent); ctgFreeMetaRent(&pCtg->stbRent); ctgFreeInstDbCache(pCtg->dbCache); ctgFreeInstUserCache(pCtg->userCache); - ctgMetaRentInit(&pCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB); - ctgMetaRentInit(&pCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE); + ctgMetaRentInit(&pCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo)); + ctgMetaRentInit(&pCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion)); pCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); @@ -844,7 +926,6 @@ int32_t ctgGenerateVgList(SCatalog* pCtg, SHashObj* vgHash, SArray** pList) { } pIter = taosHashIterate(vgHash, pIter); - vgInfo = NULL; } *pList = vgList; @@ -887,7 +968,7 @@ int32_t ctgHashValueComp(void const* lp, void const* rp) { return 0; } -int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) { +int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) { int32_t code = 0; CTG_ERR_RET(ctgMakeVgArray(dbInfo)); @@ -895,6 +976,14 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); + if (IS_SYS_DBNAME(pTableName->dbname)) { + pVgroup->vgId = MNODE_HANDLE; + if (pMgmtEps) { + memcpy(&pVgroup->epSet, pMgmtEps, sizeof(pVgroup->epSet)); + } + return TSDB_CODE_SUCCESS; + } + if (vgNum <= 0) { ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", db, vgNum); CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); @@ -938,23 +1027,53 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName CTG_RET(code); } -int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, +int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; SMetaRes res = {0}; + SVgroupInfo* vgInfo = NULL; CTG_ERR_RET(ctgMakeVgArray(dbInfo)); + int32_t tbNum = taosArrayGetSize(pNames); + + char* pSep = strchr(dbFName, '.'); + if (pSep && IS_SYS_DBNAME(pSep + 1)) { + SVgroupInfo mgmtInfo = {0}; + mgmtInfo.vgId = MNODE_HANDLE; + if (pMgmgEpSet) { + memcpy(&mgmtInfo.epSet, pMgmgEpSet, sizeof(mgmtInfo.epSet)); + } + for (int32_t i = 0; i < tbNum; ++i) { + vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo)); + if (NULL == vgInfo) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(vgInfo, &mgmtInfo, sizeof(mgmtInfo)); + + ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps, + vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); + + if (update) { + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); + SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + pRes->pRes = vgInfo; + } else { + res.pRes = vgInfo; + taosArrayPush(pCtx->pResList, &res); + } + } + return TSDB_CODE_SUCCESS; + } + int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); if (vgNum <= 0) { ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - SVgroupInfo* vgInfo = NULL; - int32_t tbNum = taosArrayGetSize(pNames); - if (1 == vgNum) { for (int32_t i = 0; i < tbNum; ++i) { vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo)); @@ -1073,10 +1192,10 @@ int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { } } -int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2) { - if (*(int64_t*)key1 < ((SDbVgVersion*)key2)->dbId) { +int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2) { + if (*(int64_t*)key1 < ((SDbCacheInfo*)key2)->dbId) { return -1; - } else if (*(int64_t*)key1 > ((SDbVgVersion*)key2)->dbId) { + } else if (*(int64_t*)key1 > ((SDbCacheInfo*)key2)->dbId) { return 1; } else { return 0; @@ -1093,10 +1212,10 @@ int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { } } -int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2) { - if (((SDbVgVersion*)key1)->dbId < ((SDbVgVersion*)key2)->dbId) { +int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2) { + if (((SDbCacheInfo*)key1)->dbId < ((SDbCacheInfo*)key2)->dbId) { return -1; - } else if (((SDbVgVersion*)key1)->dbId > ((SDbVgVersion*)key2)->dbId) { + } else if (((SDbCacheInfo*)key1)->dbId > ((SDbCacheInfo*)key2)->dbId) { return 1; } else { return 0; @@ -1267,16 +1386,20 @@ static void* ctgCloneDbVgroup(void* pSrc) { return taosArrayDup((const SArray*)p static void ctgFreeDbVgroup(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } -static void* ctgCloneDbCfgInfo(void* pSrc) { +void* ctgCloneDbCfgInfo(void* pSrc) { SDbCfgInfo* pDst = taosMemoryMalloc(sizeof(SDbCfgInfo)); if (NULL == pDst) { return NULL; } memcpy(pDst, pSrc, sizeof(SDbCfgInfo)); + pDst->pRetensions = taosArrayDup(((SDbCfgInfo *)pSrc)->pRetensions, NULL); return pDst; } -static void ctgFreeDbCfgInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } +static void ctgFreeDbCfgInfo(void* p) { + SDbCfgInfo* pDst = (SDbCfgInfo *)((SMetaRes*)p)->pRes; + freeDbCfgInfo(pDst); +} static void* ctgCloneDbInfo(void* pSrc) { SDbInfo* pDst = taosMemoryMalloc(sizeof(SDbInfo)); @@ -1466,17 +1589,26 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { return TSDB_CODE_SUCCESS; } + if (IS_SYS_DBNAME(pReq->tbName.dbname)) { + pRes->pass = true; + ctgDebug("sysdb %s, pass", pReq->tbName.dbname); + return TSDB_CODE_SUCCESS; + } + char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(&pReq->tbName, dbFName); + // since that we add read/write previliges when create db, there is no need to check createdDbs +#if 0 if (pInfo->createdDbs && taosHashGet(pInfo->createdDbs, dbFName, strlen(dbFName))) { pRes->pass = true; return TSDB_CODE_SUCCESS; } +#endif switch (pReq->type) { case AUTH_TYPE_READ: { - if (pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { + if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { req->singleType = AUTH_TYPE_READ; CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); if (pRes->pass || res->metaNotExists) { @@ -1492,7 +1624,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { break; } case AUTH_TYPE_WRITE: { - if (pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { + if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { req->singleType = AUTH_TYPE_WRITE; CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); if (pRes->pass || res->metaNotExists) { @@ -1618,6 +1750,130 @@ void catalogFreeMetaData(SMetaData* pData) { } #endif +uint64_t ctgGetTbIndexCacheSize(STableIndex *pIndex) { + if (NULL == pIndex) { + return 0; + } + + return sizeof(*pIndex) + pIndex->indexSize; +} + +FORCE_INLINE uint64_t ctgGetTbMetaCacheSize(STableMeta *pMeta) { + if (NULL == pMeta) { + return 0; + } + + switch (pMeta->tableType) { + case TSDB_SUPER_TABLE: + return sizeof(*pMeta) + (pMeta->tableInfo.numOfColumns + pMeta->tableInfo.numOfTags) * sizeof(SSchema); + case TSDB_CHILD_TABLE: + return sizeof(SCTableMeta); + default: + return sizeof(*pMeta) + pMeta->tableInfo.numOfColumns * sizeof(SSchema); + } + + return 0; +} + +uint64_t ctgGetDbVgroupCacheSize(SDBVgInfo *pVg) { + if (NULL == pVg) { + return 0; + } + + return sizeof(*pVg) + taosHashGetSize(pVg->vgHash) * (sizeof(SVgroupInfo) + sizeof(int32_t)) + + taosArrayGetSize(pVg->vgArray) * sizeof(SVgroupInfo); +} + +uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) { + if (NULL == pAuth) { + return 0; + } + + uint64_t cacheSize = 0; + char* p = taosHashIterate(pAuth->createdDbs, NULL); + while (p != NULL) { + size_t len = 0; + void* key = taosHashGetKey(p, &len); + cacheSize += len + strlen(p); + + p = taosHashIterate(pAuth->createdDbs, p); + } + + p = taosHashIterate(pAuth->readDbs, NULL); + while (p != NULL) { + size_t len = 0; + void* key = taosHashGetKey(p, &len); + cacheSize += len + strlen(p); + + p = taosHashIterate(pAuth->readDbs, p); + } + + p = taosHashIterate(pAuth->writeDbs, NULL); + while (p != NULL) { + size_t len = 0; + void* key = taosHashGetKey(p, &len); + cacheSize += len + strlen(p); + + p = taosHashIterate(pAuth->writeDbs, p); + } + + p = taosHashIterate(pAuth->readTbs, NULL); + while (p != NULL) { + size_t len = 0; + void* key = taosHashGetKey(p, &len); + cacheSize += len + strlen(p); + + p = taosHashIterate(pAuth->readTbs, p); + } + + p = taosHashIterate(pAuth->writeTbs, NULL); + while (p != NULL) { + size_t len = 0; + void* key = taosHashGetKey(p, &len); + cacheSize += len + strlen(p); + + p = taosHashIterate(pAuth->writeTbs, p); + } + + int32_t *ref = taosHashIterate(pAuth->useDbs, NULL); + while (ref != NULL) { + size_t len = 0; + void* key = taosHashGetKey(ref, &len); + cacheSize += len + sizeof(*ref); + + ref = taosHashIterate(pAuth->useDbs, ref); + } + + return cacheSize; +} + +uint64_t ctgGetClusterCacheSize(SCatalog *pCtg) { + uint64_t cacheSize = sizeof(SCatalog); + + SCtgUserAuth* pAuth = taosHashIterate(pCtg->userCache, NULL); + while (pAuth != NULL) { + size_t len = 0; + void* key = taosHashGetKey(pAuth, &len); + cacheSize += len + sizeof(SCtgUserAuth) + atomic_load_64(&pAuth->userCacheSize); + + pAuth = taosHashIterate(pCtg->userCache, pAuth); + } + + SCtgDBCache* pDb = taosHashIterate(pCtg->dbCache, NULL); + while (pDb != NULL) { + size_t len = 0; + void* key = taosHashGetKey(pDb, &len); + cacheSize += len + sizeof(SCtgDBCache) + atomic_load_64(&pDb->dbCacheSize); + + pDb = taosHashIterate(pCtg->dbCache, pDb); + } + + cacheSize += pCtg->dbRent.rentCacheSize; + cacheSize += pCtg->stbRent.rentCacheSize; + + return cacheSize; +} + void ctgGetClusterCacheStat(SCatalog* pCtg) { for (int32_t i = 0; i < CTG_CI_MAX_VALUE; ++i) { if (0 == (gCtgStatItem[i].flag & CTG_CI_FLAG_LEVEL_DB)) { @@ -1682,3 +1938,23 @@ void ctgGetGlobalCacheStat(SCtgCacheStat* pStat) { memcpy(pStat, &gCtgMgmt.statInfo.cache, sizeof(gCtgMgmt.statInfo.cache)); } + +void ctgGetGlobalCacheSize(uint64_t *pSize) { + *pSize = 0; + + SCatalog* pCtg = NULL; + void* pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + while (pIter) { + size_t len = 0; + void* key = taosHashGetKey(pIter, &len); + *pSize += len + POINTER_BYTES; + + pCtg = *(SCatalog**)pIter; + if (pCtg) { + *pSize += ctgGetClusterCacheSize(pCtg); + } + + pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); + } +} + diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 2cba433e849f89b161b7b3e2c3e58d9694aacdb0..e0e456402b454aeca23d95fd8d09a467c77b3502 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -37,6 +37,7 @@ #include "tglobal.h" #include "trpc.h" #include "tvariant.h" +#include "ttimer.h" namespace { @@ -150,6 +151,7 @@ void ctgTestInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; + tmrDebugFlag = 159; strcpy(tsLogDir, TD_LOG_DIR_PATH); ctgdEnableDebug("api", true); @@ -1331,7 +1333,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - SDbVgVersion *dbs = NULL; + SDbCacheInfo *dbs = NULL; SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; @@ -1443,7 +1445,7 @@ TEST(tableMeta, childTableCase) { taosMemoryFree(tableMeta); - SDbVgVersion *dbs = NULL; + SDbCacheInfo *dbs = NULL; SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; @@ -1584,7 +1586,7 @@ TEST(tableMeta, superTableCase) { taosMemoryFree(tableMeta); - SDbVgVersion *dbs = NULL; + SDbCacheInfo *dbs = NULL; SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; @@ -1746,6 +1748,8 @@ TEST(tableMeta, updateStbMeta) { code = catalogUpdateTableMeta(pCtg, &rsp); ASSERT_EQ(code, 0); + code = catalogAsyncUpdateTableMeta(pCtg, &rsp); + ASSERT_EQ(code, 0); taosMemoryFreeClear(rsp.pSchemas); while (true) { @@ -2680,7 +2684,7 @@ TEST(rentTest, allRent) { SDBVgInfo dbVgroup = {0}; SArray *vgList = NULL; ctgTestStop = false; - SDbVgVersion *dbs = NULL; + SDbCacheInfo *dbs = NULL; SSTableVersion *stable = NULL; uint32_t num = 0; diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index 2a435b43e8bd645c18a3122d6ab042ac57ec9077..2a7aeb00603160a2f7a1e6042d6bdcd1d1530414 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -145,7 +145,7 @@ typedef struct SExplainCtx { SHashObj *groupHash; // Hash } SExplainCtx; -#define EXPLAIN_ORDER_STRING(_order) ((ORDER_ASC == _order) ? "asc" : "desc") +#define EXPLAIN_ORDER_STRING(_order) ((ORDER_ASC == _order) ? "asc" : ORDER_DESC == _order ? "desc" : "unknown") #define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") #define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u))) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 6efdd8d8ebddbaea1032de9c583ff37147587b1a..8ddf730d5a884a31377bcb4278ac3927a67e9e31 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -48,6 +48,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe static int32_t getSchemaBytes(const SSchema* pSchema) { switch (pSchema->type) { case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_GEOMETRY: return (pSchema->bytes - VARSTR_HEADER_SIZE); case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_JSON: @@ -86,7 +87,7 @@ static int32_t buildDescResultDataBlock(SSDataBlock** pOutput) { return code; } -static void setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta) { +static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta) { blockDataEnsureCapacity(pBlock, numOfRows); pBlock->info.rows = 0; @@ -113,6 +114,11 @@ static void setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, in colDataSetVal(pCol4, pBlock->info.rows, buf, false); ++(pBlock->info.rows); } + if (pBlock->info.rows <= 0) { + qError("no permission to view any columns"); + return TSDB_CODE_PAR_PERMISSION_DENIED; + } + return TSDB_CODE_SUCCESS; } static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp) { @@ -122,7 +128,7 @@ static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** SSDataBlock* pBlock = NULL; int32_t code = buildDescResultDataBlock(&pBlock); if (TSDB_CODE_SUCCESS == code) { - setDescResultIntoDataBlock(sysInfoUser, pBlock, numOfRows, pDesc->pMeta); + code = setDescResultIntoDataBlock(sysInfoUser, pBlock, numOfRows, pDesc->pMeta); } if (TSDB_CODE_SUCCESS == code) { code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp); @@ -285,21 +291,24 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch hashPrefix = pCfg->hashPrefix + dbFNameLen + 1; } - len += sprintf( - buf2 + VARSTR_HEADER_SIZE, - "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm " - "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " - "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d " - "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " WAL_ROLL_PERIOD %d WAL_SEGMENT_SIZE %" PRId64, - dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile, - pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, - pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups, - 1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, - pCfg->walRetentionSize, pCfg->walRollPeriod, pCfg->walSegmentSize); - - if (retentions) { - len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions); - taosMemoryFree(retentions); + if (IS_SYS_DBNAME(dbName)) { + len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s`", dbName); + } else { + len += sprintf( + buf2 + VARSTR_HEADER_SIZE, + "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm " + "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " + "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d " + "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64, + dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile, + pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, + pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups, + 1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize); + + if (retentions) { + len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions); + taosMemoryFree(retentions); + } } (varDataLen(buf2)) = len; @@ -391,7 +400,7 @@ static int32_t setAliveResultIntoDataBlock(int64_t* pConnId, SSDataBlock* pBlock int32_t status = 0; int32_t code = getAliveStatusFromApi(pConnId, dbName, &status); if (code == TSDB_CODE_SUCCESS) { - colDataAppend(pCol1, 0, (const char*)&status, false); + colDataSetVal(pCol1, 0, (const char*)&status, false); } return code; } @@ -448,7 +457,7 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { SSchema* pSchema = pCfg->pSchemas + i; char type[32]; sprintf(type, "%s", tDataTypes[pSchema->type].name); - if (TSDB_DATA_TYPE_VARCHAR == pSchema->type) { + if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); @@ -463,7 +472,7 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; char type[32]; sprintf(type, "%s", tDataTypes[pSchema->type].name); - if (TSDB_DATA_TYPE_VARCHAR == pSchema->type) { + if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); @@ -539,7 +548,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { } /* - if (type == TSDB_DATA_TYPE_BINARY) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_GEOMETRY) { if (pTagVal->nData > 0) { if (num) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", "); @@ -606,6 +615,31 @@ void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* if (pCfg->ttl > 0) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " TTL %d", pCfg->ttl); } + + if (TSDB_SUPER_TABLE == pCfg->tableType || TSDB_NORMAL_TABLE == pCfg->tableType) { + int32_t nSma = 0; + for (int32_t i = 0; i < pCfg->numOfColumns; ++i) { + if (IS_BSMA_ON(pCfg->pSchemas + i)) { + ++nSma; + } + } + + if (nSma < pCfg->numOfColumns) { + bool smaOn = false; + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " SMA("); + for (int32_t i = 0; i < pCfg->numOfColumns; ++i) { + if (IS_BSMA_ON(pCfg->pSchemas + i)) { + if (smaOn) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ",`%s`", (pCfg->pSchemas + i)->name); + } else { + smaOn = true; + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "`%s`", (pCfg->pSchemas + i)->name); + } + } + } + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ")"); + } + } } static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* pDbCfg, char* tbName, STableCfg* pCfg) { @@ -748,7 +782,7 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { return terrno; } - if (taosSetCfg(tsCfg, pStmt->config)) { + if (taosApplyLocalCfg(tsCfg, pStmt->config)) { return terrno; } @@ -768,15 +802,19 @@ static int32_t buildLocalVariablesResultDataBlock(SSDataBlock** pOutput) { pBlock->pDataBlock = taosArrayInit(SHOW_LOCAL_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData)); SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN; taosArrayPush(pBlock->pDataBlock, &infoData); + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN; + taosArrayPush(pBlock->pDataBlock, &infoData); + *pOutput = pBlock; return TSDB_CODE_SUCCESS; } @@ -789,6 +827,7 @@ int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) { for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) { SConfigItem* pItem = taosArrayGet(tsCfg->array, i); GRANT_CFG_SKIP; + char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++); @@ -801,6 +840,12 @@ int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) { pColInfo = taosArrayGet(pBlock->pDataBlock, c++); colDataSetVal(pColInfo, i, value, false); + char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0}; + cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen); + varDataSetLen(scope, valueLen); + pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + colDataSetVal(pColInfo, i, scope, false); + numOfRows++; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index c5b9eb7475b0bdebe19451f1d3ac69a4d248abb5..e917de33dd3ef81333231096c92efac278db516f 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -499,6 +499,9 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pPrjNode->pProjections->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pPrjNode->node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pPrjNode->node.inputTsOrder)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); @@ -544,6 +547,9 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pJoinNode->pTargets->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pJoinNode->node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pJoinNode->node.inputTsOrder)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); @@ -597,6 +603,9 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_GROUPS_FORMAT, pAggNode->pGroupKeys->length); } + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pAggNode->node.inputTsOrder)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); @@ -716,6 +725,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i case QUERY_NODE_PHYSICAL_PLAN_SORT: { SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSortNode->node.inputTsOrder)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_OUTPUT_ORDER_TYPE_FORMAT, EXPLAIN_ORDER_STRING(pSortNode->node.outputTsOrder)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); @@ -796,9 +810,10 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pIntNode->window.inputTsOrder)); + EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pIntNode->window.node.inputTsOrder)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_OUTPUT_ORDER_TYPE_FORMAT, EXPLAIN_ORDER_STRING(pIntNode->window.node.outputTsOrder)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_OUTPUT_ORDER_TYPE_FORMAT, EXPLAIN_ORDER_STRING(pIntNode->window.outputTsOrder)); EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); @@ -847,6 +862,10 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pIntNode->window.pFuncs->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pIntNode->window.node.inputTsOrder)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_OUTPUT_ORDER_TYPE_FORMAT, EXPLAIN_ORDER_STRING(pIntNode->window.node.outputTsOrder)); EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); @@ -895,6 +914,9 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_MODE_FORMAT, nodesGetFillModeString(pFillNode->mode)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pFillNode->node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pFillNode->node.inputTsOrder)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); @@ -987,7 +1009,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i SStateWinodwPhysiNode *pStateNode = (SStateWinodwPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_STATE_WINDOW_FORMAT, - nodesGetNameFromColumnNode(((STargetNode *)pStateNode->pStateKey)->pExpr)); + nodesGetNameFromColumnNode(pStateNode->pStateKey)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); @@ -1080,6 +1102,10 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pDescNode->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pDescNode->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pMergeNode->node.inputTsOrder)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_OUTPUT_ORDER_TYPE_FORMAT, EXPLAIN_ORDER_STRING(pMergeNode->node.outputTsOrder)); EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 8b3d04e32c35e7143ccdc46fdcdf9f56ec09d0fc..d2c39aba7483adf5d3088789180c3e436ffd3ae7 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -1,10 +1,9 @@ aux_source_directory(src EXECUTOR_SRC) -#add_library(executor ${EXECUTOR_SRC}) add_library(executor STATIC ${EXECUTOR_SRC}) target_link_libraries(executor - PRIVATE os util common function parser planner qcom vnode scalar nodes index stream + PRIVATE os util common function parser planner qcom scalar nodes index wal tdb ) target_include_directories( diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 57a771b275b219b776dbee34193e2ba644e73a1f..9893b4eb764814f4fa07f20190de499dc650086a 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -22,6 +22,7 @@ extern "C" { #include "dataSinkMgt.h" #include "plannodes.h" +#include "storageapi.h" #include "tcommon.h" struct SDataSink; @@ -29,6 +30,7 @@ struct SDataSinkHandle; typedef struct SDataSinkManager { SDataSinkMgtCfg cfg; + SStorageAPI* pAPI; } SDataSinkManager; typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 879395cd5787c9e5e435cbf0bbad176bf5efd3df..33c9d845b9bded28acf0c3ec796266bc5476371d 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -12,17 +12,17 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef TDENGINE_QUERYUTIL_H -#define TDENGINE_QUERYUTIL_H +#ifndef TDENGINE_EXECUTIL_H +#define TDENGINE_EXECUTIL_H #include "executor.h" #include "function.h" #include "nodes.h" #include "plannodes.h" +#include "storageapi.h" #include "tcommon.h" #include "tpagedbuf.h" #include "tsimplehash.h" -#include "vnode.h" #define T_LONG_JMP(_obj, _c) \ do { \ @@ -75,10 +75,11 @@ typedef struct SResultRowInfo { } SResultRowInfo; typedef struct SColMatchItem { - int32_t colId; - int32_t srcSlotId; - int32_t dstSlotId; - bool needOutput; + int32_t colId; + int32_t srcSlotId; + int32_t dstSlotId; + bool needOutput; + SDataType dataType; } SColMatchItem; typedef struct SColMatchInfo { @@ -86,8 +87,25 @@ typedef struct SColMatchInfo { int32_t matchType; // determinate the source according to col id or slot id } SColMatchInfo; -typedef struct SExecTaskInfo SExecTaskInfo; -typedef struct STableListInfo STableListInfo; +typedef struct SExecTaskInfo SExecTaskInfo; + +typedef struct STableListIdInfo { + uint64_t suid; + uint64_t uid; + int32_t tableType; +} STableListIdInfo; + +// If the numOfOutputGroups is 1, the data blocks that belongs to different groups will be provided randomly +// The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups +typedef struct STableListInfo { + bool oneTableForEachGroup; + int32_t numOfOuputGroups; // the data block will be generated one by one + int32_t* groupOffset; // keep the offset value for each group in the tableList + SArray* pTableList; + SHashObj* map; // speedup acquire the tableQueryInfo by table uid + STableListIdInfo idInfo; // this maybe the super table or ordinary table +} STableListInfo; + struct SqlFunctionCtx; int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, @@ -118,6 +136,10 @@ struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t i static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) { SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); + if (!bufPage) { + uFatal("failed to get the buffer page:%d since %s", pos->pageId, terrstr()); + return NULL; + } if (forUpdate) { setBufPageDirty(bufPage, true); } @@ -137,7 +159,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); -int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId); +int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId, SStorageAPI* pAPI); size_t getTableTagsBufLen(const SNodeList* pGroups); SArray* createSortInfo(SNodeList* pNodeList); @@ -149,7 +171,7 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId); void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode); SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); -SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset); +SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, SFunctionStateStore* pStore); void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn); void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); @@ -161,8 +183,11 @@ void cleanupQueryTableDataCond(SQueryTableDataCond* pCond); int32_t convertFillType(int32_t mode); int32_t resultrowComparAsc(const void* p1, const void* p2); -int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified); +int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified, SStorageAPI *pAPI); void printDataBlock(SSDataBlock* pBlock, const char* flag); -#endif // TDENGINE_QUERYUTIL_H +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); + +#endif // TDENGINE_EXECUTIL_H diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 3f365c70480d3cb9773599056dc44f0c15c03ff8..fbca5e29f9b733d71bea68c540f981c09903e3a3 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -25,6 +25,7 @@ extern "C" { #include "tsort.h" #include "ttszip.h" #include "tvariant.h" +#include "theap.h" #include "dataSinkMgt.h" #include "executil.h" @@ -38,13 +39,15 @@ extern "C" { #include "tlockfree.h" #include "tmsg.h" #include "tpagedbuf.h" -#include "tstream.h" -#include "tstreamUpdate.h" - -#include "vnode.h" +//#include "tstream.h" +//#include "tstreamUpdate.h" +#include "tlrucache.h" typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); +typedef struct STsdbReader STsdbReader; +typedef struct STqReader STqReader; + #define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0) #define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN) #define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0) @@ -78,7 +81,6 @@ enum { STREAM_RECOVER_STEP__PREPARE1, STREAM_RECOVER_STEP__PREPARE2, STREAM_RECOVER_STEP__SCAN1, - STREAM_RECOVER_STEP__SCAN2, }; extern int32_t exchangeObjRefPool; @@ -206,6 +208,7 @@ typedef struct STableScanBase { SLimitInfo limitInfo; // there are more than one table list exists in one task, if only one vnode exists. STableListInfo* pTableListInfo; + TsdReader readerAPI; } STableScanBase; typedef struct STableScanInfo { @@ -213,6 +216,7 @@ typedef struct STableScanInfo { SScanInfo scanInfo; int32_t scanTimes; SSDataBlock* pResBlock; + SHashObj* pIgnoreTables; SSampleExecInfo sample; // sample execution info int32_t currentGroupId; int32_t currentTable; @@ -220,6 +224,7 @@ typedef struct STableScanInfo { int8_t assignBlockUid; bool hasGroupByTag; bool countOnly; +// TsdReader readerAPI; } STableScanInfo; typedef struct STableMergeScanInfo { @@ -227,19 +232,20 @@ typedef struct STableMergeScanInfo { int32_t tableEndIndex; bool hasGroupId; uint64_t groupId; - SArray* queryConds; // array of queryTableDataCond STableScanBase base; int32_t bufPageSize; uint32_t sortBufSize; // max buffer size for in-memory sort SArray* pSortInfo; SSortHandle* pSortHandle; SSDataBlock* pSortInputBlock; + SSDataBlock* pReaderBlock; int64_t startTs; // sort start time SArray* sortSourceParams; SLimitInfo limitInfo; int64_t numOfRows; SScanInfo scanInfo; int32_t scanTimes; + int32_t readIdx; SSDataBlock* pResBlock; SSampleExecInfo sample; // sample execution info SSortExecInfo sortExecInfo; @@ -279,6 +285,9 @@ typedef struct SStreamAggSupporter { int32_t stateKeySize; int16_t stateKeyType; SDiskbasedBuf* pResultBuf; + SStateStore stateStore; + STimeWindow winRange; + SStorageAPI* pSessionAPI; } SStreamAggSupporter; typedef struct SWindowSupporter { @@ -298,7 +307,6 @@ typedef struct SPartitionBySupporter { typedef struct SPartitionDataInfo { uint64_t groupId; char* tbname; - SArray* tags; SArray* rowIds; } SPartitionDataInfo; @@ -335,7 +343,7 @@ typedef struct SStreamScanInfo { STqReader* tqReader; uint64_t groupId; - SUpdateInfo* pUpdateInfo; + struct SUpdateInfo* pUpdateInfo; EStreamScanMode scanMode; struct SOperatorInfo* pStreamScanOp; @@ -359,22 +367,24 @@ typedef struct SStreamScanInfo { SNode* pTagIndexCond; // recover - int32_t blockRecoverContiCnt; int32_t blockRecoverTotCnt; SSDataBlock* pRecoverRes; SSDataBlock* pCreateTbRes; int8_t igCheckUpdate; int8_t igExpired; - SStreamState* pState; + void* pState; //void + SStoreTqReader readerFn; + SStateStore stateStore; } SStreamScanInfo; typedef struct { - SVnode* vnode; - SSDataBlock pRes; // result SSDataBlock - STsdbReader* dataReader; - SSnapContext* sContext; - STableListInfo* pTableListInfo; + struct SVnode* vnode; // todo remove this + SSDataBlock pRes; // result SSDataBlock + STsdbReader* dataReader; + struct SSnapContext* sContext; + SStorageAPI* pAPI; + STableListInfo* pTableListInfo; } SStreamRawScanInfo; typedef struct STableCountScanSupp { @@ -391,6 +401,8 @@ typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; SSDataBlock* pRes; bool mergeResultBlock; + int32_t inputTsOrder; + int32_t outputTsOrder; } SOptrBasicInfo; typedef struct SIntervalAggOperatorInfo { @@ -403,11 +415,17 @@ typedef struct SIntervalAggOperatorInfo { STimeWindow win; // query time range bool timeWindowInterpo; // interpolation needed or not SArray* pInterpCols; // interpolation columns - int32_t resultTsOrder; // result timestamp order - int32_t inputOrder; // input data ts order EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] STimeWindowAggSupp twAggSup; SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. + // for limit optimization + bool limited; + int64_t limit; + bool slimited; + int64_t slimit; + uint64_t curGroupId; // initialize to UINT64_MAX + uint64_t handledGroupNum; + BoundedQueue* pBQ; } SIntervalAggOperatorInfo; typedef struct SMergeAlignedIntervalAggOperatorInfo { @@ -441,12 +459,15 @@ typedef struct SStreamIntervalOperatorInfo { bool isFinal; SArray* pChildren; int32_t numOfChild; - SStreamState* pState; + SStreamState* pState; // void SWinKey delKey; uint64_t numOfDatapack; SArray* pUpdated; SSHashObj* pUpdatedMap; int64_t dataVersion; + SStateStore statestore; + bool recvGetAll; + SHashObj* pFinalPullDataMap; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { @@ -492,6 +513,8 @@ typedef struct SStreamSessionAggOperatorInfo { SArray* pUpdated; SSHashObj* pStUpdated; int64_t dataVersion; + SArray* historyWins; + bool isHistoryOp; } SStreamSessionAggOperatorInfo; typedef struct SStreamStateAggOperatorInfo { @@ -511,6 +534,8 @@ typedef struct SStreamStateAggOperatorInfo { SArray* pUpdated; SSHashObj* pSeUpdated; int64_t dataVersion; + bool isHistoryOp; + SArray* historyWins; } SStreamStateAggOperatorInfo; typedef struct SStreamPartitionOperatorInfo { @@ -543,6 +568,7 @@ typedef struct SStreamFillSupporter { int32_t rowSize; SSHashObj* pResMap; bool hasDelete; + SStorageAPI* pAPI; } SStreamFillSupporter; typedef struct SStreamFillOperatorInfo { @@ -569,12 +595,11 @@ void cleanupQueriedTableScanInfo(SSchemaInfo* pSchemaInfo); void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); void cleanupBasicInfo(SOptrBasicInfo* pInfo); -int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr); +int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr, SFunctionStateStore* pStore); void cleanupExprSupp(SExprSupp* pSup); - int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, - const char* pkey, void* pState); + const char* pkey, void* pState, SFunctionStateStore* pStore); void cleanupAggSup(SAggSupporter* pAggSup); void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); @@ -602,7 +627,7 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* de extern void doDestroyExchangeOperatorInfo(void* param); -void doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo); +int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, int32_t rows, const char* idStr, STableMetaCacheInfo* pCache); @@ -636,7 +661,7 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); bool functionNeedToExecute(SqlFunctionCtx* pCtx); bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); -bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup); +bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, SStateStore* pStore); void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); @@ -645,20 +670,17 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); bool groupbyTbname(SNodeList* pGroupList); -int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, +int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo); -int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size); -int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, +int32_t saveSessionDiscBuf(void* pState, SSessionKey* key, void* buf, int32_t size, SStateStore* pAPI); +int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo); -int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, - SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup); -int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult); -int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize); +int32_t releaseOutputBuf(void* pState, SWinKey* pKey, SResultRow* pResult, SStateStore* pAPI); void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, int64_t* pData); -void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, - SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock); +void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI); SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag); SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs); @@ -670,6 +692,8 @@ void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExpr void doClearBufferedBlocks(SStreamScanInfo* pInfo); uint64_t calcGroupId(char* pData, int32_t len); +void streamOpReleaseState(struct SOperatorInfo* pOperator); +void streamOpReloadState(struct SOperatorInfo* pOperator); #ifdef __cplusplus } diff --git a/source/libs/executor/inc/operator.h b/source/libs/executor/inc/operator.h index 632b817a0754d2d8d4482200df5ac28e18bcf02c..e6c3405d7ff1b75326601adc0b436ca8295ee78d 100644 --- a/source/libs/executor/inc/operator.h +++ b/source/libs/executor/inc/operator.h @@ -35,6 +35,7 @@ typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr); typedef void (*__optr_close_fn_t)(void* param); typedef int32_t (*__optr_explain_fn_t)(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); typedef int32_t (*__optr_reqBuf_fn_t)(struct SOperatorInfo* pOptr); +typedef void (*__optr_state_fn_t)(struct SOperatorInfo* pOptr); typedef struct SOperatorFpSet { __optr_open_fn_t _openFn; // DO NOT invoke this function directly @@ -45,6 +46,8 @@ typedef struct SOperatorFpSet { __optr_encode_fn_t encodeResultRow; __optr_decode_fn_t decodeResultRow; __optr_explain_fn_t getExplainFn; + __optr_state_fn_t releaseStreamStateFn; + __optr_state_fn_t reloadStreamStateFn; } SOperatorFpSet; enum { @@ -126,13 +129,13 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle); -SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); +SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle); SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); @@ -143,6 +146,7 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_explain_fn_t explain); +void setOperatorStreamStateFn(SOperatorInfo* pOperator, __optr_state_fn_t relaseFn, __optr_state_fn_t reloadFn); int32_t optrDummyOpenFn(SOperatorInfo* pOperator); int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); void setOperatorCompleted(SOperatorInfo* pOperator); @@ -156,7 +160,7 @@ void destroyOperator(SOperatorInfo* pOperator); SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder); -int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr); +int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr, SStorageAPI* pAPI); int32_t getOperatorExplainExecInfo(struct SOperatorInfo* operatorInfo, SArray* pExecInfoList); #ifdef __cplusplus diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h index 37c93fef5c3fdad15ac2d6855e3d0b7ca323274c..7241b015a09321db59af5f212efae85af56959ca 100644 --- a/source/libs/executor/inc/querytask.h +++ b/source/libs/executor/inc/querytask.h @@ -56,20 +56,21 @@ typedef struct STaskStopInfo { } STaskStopInfo; typedef struct { - STqOffsetVal currentOffset; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta - int64_t snapshotVer; -// SPackedData submit; // todo remove it - SSchemaWrapper* schema; - char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor - int8_t recoverStep; - int8_t recoverScanFinished; - SQueryTableDataCond tableCond; - int64_t fillHistoryVer1; - int64_t fillHistoryVer2; - SStreamState* pState; - int64_t dataVersion; - int64_t checkPointId; + STqOffsetVal currentOffset; // for tmq + SMqMetaRsp metaRsp; // for tmq fetching meta + int64_t snapshotVer; + SSchemaWrapper* schema; + char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor + int8_t recoverStep; +// bool recoverStep1Finished; +// bool recoverStep2Finished; + int8_t recoverScanFinished; + SQueryTableDataCond tableCond; + SVersionRange fillHistoryVer; + STimeWindow fillHistoryWindow; + SStreamState* pState; + int64_t dataVersion; + int64_t checkPointId; } SStreamTaskInfo; struct SExecTaskInfo { @@ -92,10 +93,11 @@ struct SExecTaskInfo { SArray* pResultBlockList; // result block list STaskStopInfo stopInfo; SRWLatch lock; // secure the access of STableListInfo + SStorageAPI storageAPI; }; void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst); -SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model); +SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI); void doDestroyTask(SExecTaskInfo* pTaskInfo); bool isTaskKilled(SExecTaskInfo* pTaskInfo); void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index 78b3cd2f40b839dddd39d12bf0b0d6fc1ba82555..79837480d79ad265e8e6ce307a09828a3dc5e1d7 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -120,6 +120,8 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); +void taosFillUpdateStartTimestampInfo(SFillInfo* pFillInfo, int64_t ts); +bool taosFillNotStarted(const SFillInfo* pFillInfo); SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr, int32_t numOfNotFillCols, const struct SNodeListNode* val); bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index d51a24bb43253e09d6ab21e6a1785d6ef6b5e5ff..57c8bce27553ab9bb9035e1b55a04c7b2f6bbe97 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -26,6 +26,7 @@ extern "C" { enum { SORT_MULTISOURCE_MERGE = 0x1, SORT_SINGLESOURCE_SORT = 0x2, + SORT_BLOCK_TS_MERGE = 0x3 }; typedef struct SMultiMergeSource { @@ -53,6 +54,12 @@ typedef struct SMsortComparParam { int32_t numOfSources; SArray* orderInfo; // SArray bool cmpGroupId; + + int32_t sortType; + // the following field to speed up when sortType == SORT_BLOCK_TS_MERGE + int32_t tsSlotId; + int32_t order; + __compar_fn_t cmpFn; } SMsortComparParam; typedef struct SSortHandle SSortHandle; @@ -64,10 +71,16 @@ typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void* /** * * @param type + * @param maxRows keep maxRows at most, if 0, pq sort will not be used + * @param maxTupleLength max len of one tuple, for check if pq sort is applicable + * @param sortBufSize sort memory buf size, for check if heap sort is applicable * @return */ SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, int32_t type, int32_t pageSize, int32_t numOfPages, - SSDataBlock* pBlock, const char* idstr); + SSDataBlock* pBlock, const char* idstr, uint64_t pqMaxRows, uint32_t pqMaxTupleLength, + uint32_t pqSortBufSize); + +void tsortSetForceUsePQSort(SSortHandle* pHandle); /** * @@ -104,6 +117,10 @@ int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fetc */ int32_t tsortSetComparFp(SSortHandle* pHandle, _sort_merge_compar_fn_t fp); +/** + * +*/ +void tsortSetMergeLimit(SSortHandle* pHandle, int64_t mergeLimit); /** * */ @@ -146,7 +163,7 @@ void* tsortGetValue(STupleHandle* pVHandle, int32_t colId); * @return */ uint64_t tsortGetGroupId(STupleHandle* pVHandle); - +void* tsortGetBlockInfo(STupleHandle* pVHandle); /** * * @param pSortHandle @@ -170,6 +187,12 @@ SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle); */ int32_t getProperSortPageSize(size_t rowSize, uint32_t numOfCols); + +bool tsortIsClosed(SSortHandle* pHandle); +void tsortSetClosed(SSortHandle* pHandle); + +void setSingleTableMerge(SSortHandle* pHandle); + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index 5cd95d33114d8bcb9edf0638a990b4116478d5bd..176c4b53be828a0ad356953793f722cc39d43893 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -21,7 +21,6 @@ #include "tname.h" #include "executorInt.h" -#include "index.h" #include "operator.h" #include "query.h" #include "querytask.h" @@ -30,6 +29,7 @@ #include "tglobal.h" #include "thash.h" #include "ttypes.h" +#include "index.h" typedef struct { bool hasAgg; @@ -45,6 +45,8 @@ typedef struct SAggOperatorInfo { SGroupResInfo groupResInfo; SExprSupp scalarExprSup; bool groupKeyOptimized; + bool hasValidBlock; + SSDataBlock* pNewGroupBlock; } SAggOperatorInfo; static void destroyAggOperatorInfo(void* param); @@ -53,7 +55,6 @@ static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, u static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock); static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock); -static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator); static int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx); static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator); @@ -84,7 +85,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -95,7 +96,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr); } - code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr); + code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -108,10 +109,12 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; pInfo->groupKeyOptimized = pAggNode->groupKeyOptimized; pInfo->groupId = UINT64_MAX; + pInfo->binfo.inputTsOrder = pAggNode->node.inputTsOrder; + pInfo->binfo.outputTsOrder = pAggNode->node.outputTsOrder; - setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, - pTaskInfo); - pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, + setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, + !pAggNode->node.forceCreateNonBlockingOptr, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, getAggregateResult, NULL, destroyAggOperatorInfo, optrDefaultBufFn, NULL); if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { @@ -151,30 +154,42 @@ void destroyAggOperatorInfo(void* param) { taosMemoryFreeClear(param); } -// this is a blocking operator -int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { - if (OPTR_IS_OPENED(pOperator)) { - return TSDB_CODE_SUCCESS; - } - +/** + * @brief get blocks from downstream and fill results into groupedRes after aggragation + * @retval false if no more groups + * @retval true if there could have new groups coming + * @note if pOperator.blocking is true, scan all blocks from downstream, all groups are handled + * if false, fill results of ONE GROUP + * */ +static bool nextGroupedResult(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SAggOperatorInfo* pAggInfo = pOperator->info; + if (pOperator->blocking && pAggInfo->hasValidBlock) return false; + SExprSupp* pSup = &pOperator->exprSupp; SOperatorInfo* downstream = pOperator->pDownstream[0]; - int64_t st = taosGetTimestampUs(); - - int32_t order = TSDB_ORDER_ASC; - int32_t scanFlag = MAIN_SCAN; - - bool hasValidBlock = false; + int64_t st = taosGetTimestampUs(); + int32_t code = TSDB_CODE_SUCCESS; + int32_t order = pAggInfo->binfo.inputTsOrder; + SSDataBlock* pBlock = pAggInfo->pNewGroupBlock; + if (pBlock) { + pAggInfo->pNewGroupBlock = NULL; + tSimpleHashClear(pAggInfo->aggSup.pResultRowHashTable); + setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.id.groupId); + setInputDataBlock(pSup, pBlock, order, pBlock->info.scanFlag, true); + code = doAggregateImpl(pOperator, pSup->pCtx); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, code); + } + } while (1) { bool blockAllocated = false; - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - if (!hasValidBlock) { + if (!pAggInfo->hasValidBlock) { createDataBlockForEmptyInput(pOperator, &pBlock); if (pBlock == NULL) { break; @@ -184,13 +199,8 @@ int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { break; } } - hasValidBlock = true; - - int32_t code = getTableScanInfo(pOperator, &order, &scanFlag, false); - if (code != TSDB_CODE_SUCCESS) { - destroyDataBlockForEmptyInput(blockAllocated, &pBlock); - T_LONG_JMP(pTaskInfo->env, code); - } + pAggInfo->hasValidBlock = true; + pAggInfo->binfo.pRes->info.scanFlag = pBlock->info.scanFlag; // there is an scalar expression that needs to be calculated before apply the group aggregation. if (pAggInfo->scalarExprSup.pExprInfo != NULL && !blockAllocated) { @@ -201,10 +211,14 @@ int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } } - + // if non-blocking mode and new group arrived, save the block and break + if (!pOperator->blocking && pAggInfo->groupId != UINT64_MAX && pBlock->info.id.groupId != pAggInfo->groupId) { + pAggInfo->pNewGroupBlock = pBlock; + break; + } // the pDataBlock are always the same one, no need to call this again setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.id.groupId); - setInputDataBlock(pSup, pBlock, order, scanFlag, true); + setInputDataBlock(pSup, pBlock, order, pBlock->info.scanFlag, true); code = doAggregateImpl(pOperator, pSup->pCtx); if (code != 0) { destroyDataBlockForEmptyInput(blockAllocated, &pBlock); @@ -220,10 +234,7 @@ int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { } initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0); - OPTR_SET_OPENED(pOperator); - - pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - return pTaskInfo->code; + return pBlock != NULL; } SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { @@ -235,26 +246,25 @@ SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { } SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - setOperatorCompleted(pOperator); - return NULL; - } + bool hasNewGroups = false; + do { + hasNewGroups = nextGroupedResult(pOperator); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - while (1) { - doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); - doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); + while (1) { + doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); + doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); - if (!hasRemainResults(&pAggInfo->groupResInfo)) { - setOperatorCompleted(pOperator); - break; - } + if (!hasRemainResults(&pAggInfo->groupResInfo)) { + if (!hasNewGroups) setOperatorCompleted(pOperator); + break; + } - if (pInfo->pRes->info.rows > 0) { - break; + if (pInfo->pRes->info.rows > 0) { + break; + } } - } + } while (pInfo->pRes->info.rows == 0 && hasNewGroups); size_t rows = blockDataGetNumOfRows(pInfo->pRes); pOperator->resultInfo.totalRows += rows; @@ -461,8 +471,12 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n uint32_t defaultPgsz = 0; uint32_t defaultBufsz = 0; - getBufferPgSize(pAggSup->resultRowSize, &defaultPgsz, &defaultBufsz); - + code = getBufferPgSize(pAggSup->resultRowSize, &defaultPgsz, &defaultBufsz); + if (code) { + qError("failed to get buff page size, rowSize:%d", pAggSup->resultRowSize); + return code; + } + if (!osTempSpaceAvailable()) { code = TSDB_CODE_NO_DISKSPACE; qError("Init stream agg supporter failed since %s, key:%s, tempDir:%s", terrstr(code), pKey, tsTempDir); @@ -485,8 +499,8 @@ void cleanupAggSup(SAggSupporter* pAggSup) { } int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, - const char* pkey, void* pState) { - int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); + const char* pkey, void* pState, SFunctionStateStore* pStore) { + int32_t code = initExprSupp(pSup, pExprInfo, numOfCols, pStore); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 430b0b11ed7419bb284a9e80afb92f3cd1eb23e3..ce39ebab5937c6576994458e05963fe0f373bf6a 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "function.h" #include "os.h" +#include "function.h" #include "tname.h" #include "tdatablock.h" @@ -27,6 +27,8 @@ #include "thash.h" #include "ttypes.h" +#include "storageapi.h" + typedef struct SCacheRowsScanInfo { SSDataBlock* pRes; SReadHandle readHandle; @@ -102,9 +104,9 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); uint64_t suid = tableListGetSuid(pTableListInfo); - code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables, - taosArrayGetSize(pInfo->matchInfo.pList), pCidList, pInfo->pSlotIds, suid, - &pInfo->pLastrowReader, pTaskInfo->id.str); + code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables, + taosArrayGetSize(pInfo->matchInfo.pList), pCidList, pInfo->pSlotIds, + suid, &pInfo->pLastrowReader, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -124,7 +126,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe if (pScanNode->scan.pScanPseudoCols != NULL) { SExprSupp* p = &pInfo->pseudoExprSup; p->pExprInfo = createExprInfo(pScanNode->scan.pScanPseudoCols, NULL, &p->numOfExprs); - p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset); + p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); } setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED, @@ -172,7 +174,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pBufferredRes); taosArrayClear(pInfo->pUidList); - int32_t code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds, + int32_t code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds, pInfo->pDstSlotIds, pInfo->pUidList); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -206,6 +208,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pRes->info.id.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); pRes->info.rows = 1; + pRes->info.scanFlag = MAIN_SCAN; SExprSupp* pSup = &pInfo->pseudoExprSup; int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pRes, @@ -238,18 +241,22 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } - code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, - taosArrayGetSize(pInfo->matchInfo.pList), pInfo->pCidList, pInfo->pSlotIds, suid, - &pInfo->pLastrowReader, pTaskInfo->id.str); - if (code != TSDB_CODE_SUCCESS) { - pInfo->currentGroupIndex += 1; - taosArrayClear(pInfo->pUidList); - continue; + if (NULL == pInfo->pLastrowReader) { + code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, + taosArrayGetSize(pInfo->matchInfo.pList), pInfo->pCidList, pInfo->pSlotIds, suid, &pInfo->pLastrowReader, + pTaskInfo->id.str); + if (code != TSDB_CODE_SUCCESS) { + pInfo->currentGroupIndex += 1; + taosArrayClear(pInfo->pUidList); + continue; + } + } else { + pInfo->readHandle.api.cacheFn.reuseReader(pInfo->pLastrowReader, pList, num); } taosArrayClear(pInfo->pUidList); - code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pDstSlotIds, + code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pDstSlotIds, pInfo->pUidList); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -276,13 +283,14 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } - pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); + //pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); return pInfo->pRes; } else { - pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); + //pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); } } + pInfo->pLastrowReader = pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader); setOperatorCompleted(pOperator); return NULL; } @@ -300,7 +308,7 @@ void destroyCacheScanOperator(void* param) { tableListDestroy(pInfo->pTableList); if (pInfo->pLastrowReader != NULL) { - pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); + pInfo->pLastrowReader = pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader); } cleanupExprSupp(&pInfo->pseudoExprSup); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index f7167343d02dfbafda590c894faf5020ff30748e..2a22656d8ca70be98bffe008d1a455d25a0523a5 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -198,9 +198,6 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; - // ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); - // ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); - atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index d31ac0bc517f810732fc868a6371a97d356d78a7..646964ebf4eaf38b464bba680301b882ceb9d449 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -17,6 +17,7 @@ #include "dataSinkMgt.h" #include "executorInt.h" #include "planner.h" +#include "storageapi.h" #include "tcompression.h" #include "tdatablock.h" #include "tglobal.h" @@ -428,8 +429,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat inserter->explain = pInserterNode->explain; int64_t suid = 0; - int32_t code = - tsdbGetTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid); + int32_t code = pManager->pAPI->metaFn.getTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid); if (code) { destroyDataSinker((SDataSinkHandle*)inserter); taosMemoryFree(inserter); diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index b3cb57325ba5dee1d2d2f3b0d07304a1d1ea5ae2..3a972c1c20406d8d61fde988e4ed93140c46c0fa 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -21,8 +21,9 @@ static SDataSinkManager gDataSinkManager = {0}; SDataSinkStat gDataSinkStat = {0}; -int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg) { +int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI) { gDataSinkManager.cfg = *cfg; + gDataSinkManager.pAPI = pAPI; return 0; // to avoid compiler eror } diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c index 956d5b714d1fbf98f7aec024605befc31621eeef..bbdc50183e35bf2f40d80f59eb98e79662a0d98d 100644 --- a/source/libs/executor/src/eventwindowoperator.c +++ b/source/libs/executor/src/eventwindowoperator.c @@ -92,7 +92,7 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo if (pEventWindowNode->window.pExprs != NULL) { int32_t numOfScalarExpr = 0; SExprInfo* pScalarExprInfo = createExprInfo(pEventWindowNode->window.pExprs, NULL, &numOfScalarExpr); - code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -110,7 +110,7 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo initResultSizeInfo(&pOperator->resultInfo, 4096); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -120,6 +120,8 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo initBasicInfo(&pInfo->binfo, pResBlock); initResultRowInfo(&pInfo->binfo.resultRowInfo); + pInfo->binfo.inputTsOrder = physiNode->inputTsOrder; + pInfo->binfo.outputTsOrder = physiNode->outputTsOrder; pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pEventWindowNode->window.watermark, .calTrigger = pEventWindowNode->window.triggerType}; @@ -174,6 +176,7 @@ void destroyEWindowOperatorInfo(void* param) { colDataDestroy(&pInfo->twAggSup.timeWindowData); cleanupAggSup(&pInfo->aggSup); + cleanupExprSupp(&pInfo->scalarSup); taosMemoryFreeClear(param); } @@ -182,7 +185,7 @@ static SSDataBlock* eventWindowAggregate(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExprSupp* pSup = &pOperator->exprSupp; - int32_t order = TSDB_ORDER_ASC; + int32_t order = pInfo->binfo.inputTsOrder; SSDataBlock* pRes = pInfo->binfo.pRes; @@ -195,6 +198,7 @@ static SSDataBlock* eventWindowAggregate(SOperatorInfo* pOperator) { break; } + pRes->info.scanFlag = pBlock->info.scanFlag; setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true); blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId); diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 94041140d4e60030a4b93a1d6a6b7b4df9eacd85..4fbe6785a3b964395c56392af4ef417b48aa7cf6 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -520,6 +520,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pCo // data from mnode pRes->info.dataLoad = 1; pRes->info.rows = pBlock->info.rows; + pRes->info.scanFlag = MAIN_SCAN; relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); blockDataDestroy(pBlock); } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c8b16ad83b9c9eb07484718937d34ba2a8066db6..aa0c7945b022da5ddd453542eb24975675213853 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -26,25 +26,9 @@ #include "executil.h" #include "executorInt.h" #include "querytask.h" +#include "storageapi.h" #include "tcompression.h" -typedef struct STableListIdInfo { - uint64_t suid; - uint64_t uid; - int32_t tableType; -} STableListIdInfo; - -// If the numOfOutputGroups is 1, the data blocks that belongs to different groups will be provided randomly -// The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups -struct STableListInfo { - bool oneTableForEachGroup; - int32_t numOfOuputGroups; // the data block will be generated one by one - int32_t* groupOffset; // keep the offset value for each group in the tableList - SArray* pTableList; - SHashObj* map; // speedup acquire the tableQueryInfo by table uid - STableListIdInfo idInfo; // this maybe the super table or ordinary table -}; - typedef struct tagFilterAssist { SHashObj* colHash; int32_t index; @@ -58,13 +42,13 @@ typedef enum { } FilterCondType; static FilterCondType checkTagCond(SNode* cond); -static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond); -static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond); +static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond, SStorageAPI* pAPI); +static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond, SStorageAPI* pStoreAPI); -static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, - SNode* pTagIndexCond, STableListInfo* pListInfo, const char* idstr); -static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, - void* metaHandle); +static int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, + STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI); +static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode, + SStorageAPI* pStorageAPI); static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; } static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; } @@ -143,6 +127,10 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in if (pGroupResInfo->pRows != NULL) { taosArrayDestroy(pGroupResInfo->pRows); } + if (pGroupResInfo->pBuf) { + taosMemoryFree(pGroupResInfo->pBuf); + pGroupResInfo->pBuf = NULL; + } // extract the result rows information from the hash map int32_t size = tSimpleHashGetSize(pHashmap); @@ -152,7 +140,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in size_t keyLen = 0; int32_t iter = 0; - int32_t bufLen = 0, offset = 0; + int64_t bufLen = 0, offset = 0; // todo move away and record this during create window while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { @@ -279,7 +267,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { STagVal tagVal = {0}; tagVal.cid = pSColumnNode->colId; - const char* p = metaGetTableTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal); + const char* p = mr->pAPI->extractTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal); if (p == NULL) { res->node.resType.type = TSDB_DATA_TYPE_NULL; } else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) { @@ -318,14 +306,14 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) { +int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified, SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; SMetaReader mr = {0}; - metaReaderInit(&mr, metaHandle, 0); - code = metaGetTableEntryByUidCache(&mr, info->uid); + pAPI->metaReaderFn.initReader(&mr, metaHandle, 0, &pAPI->metaFn); + code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, info->uid); if (TSDB_CODE_SUCCESS != code) { - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); *pQualified = false; return TSDB_CODE_SUCCESS; @@ -334,7 +322,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, SNode* pTagCondTmp = nodesCloneNode(pTagCond); nodesRewriteExprPostOrder(&pTagCondTmp, doTranslateTagExpr, &mr); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); SNode* pNew = NULL; code = scalarCalculateConstants(pTagCondTmp, &pNew); @@ -436,16 +424,52 @@ void freeItem(void* p) { } } -int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo) { +static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) { + if (pTagCond == NULL) { + return; + } + + char* payload = NULL; + int32_t len = 0; + nodesNodeToMsg(pTagCond, &payload, &len); + + tMD5Init(pContext); + tMD5Update(pContext, (uint8_t*)payload, (uint32_t)len); + tMD5Final(pContext); + + taosMemoryFree(payload); +} + +static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_CTX* pContext) { + char* payload = NULL; + int32_t len = 0; + nodesNodeToMsg(pGroup, &payload, &len); + if (filterDigest[0]) { + payload = taosMemoryRealloc(payload, len + tListLen(pContext->digest)); + memcpy(payload + len, filterDigest + 1, tListLen(pContext->digest)); + len += tListLen(pContext->digest); + } + + tMD5Init(pContext); + tMD5Update(pContext, (uint8_t*)payload, (uint32_t)len); + tMD5Final(pContext); + + taosMemoryFree(payload); +} + +int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInfo* pTableListInfo, uint8_t* digest, + SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; void* keyBuf = NULL; SArray* groupData = NULL; + SArray* pUidTagList = NULL; + SArray* tableList = NULL; int32_t rows = taosArrayGetSize(pTableListInfo->pTableList); if (rows == 0) { - return TDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } tagFilterAssist ctx = {0}; @@ -468,21 +492,38 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis REPLACE_NODE(pNode); } - SArray* pUidTagList = taosArrayInit(8, sizeof(STUidTagInfo)); + T_MD5_CTX context = {0}; + if (tsTagFilterCache) { + SNodeListNode* listNode = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); + listNode->pNodeList = group; + genTbGroupDigest((SNode*)listNode, digest, &context); + nodesFree(listNode); + + pAPI->metaFn.metaGetCachedTbGroup(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), + &tableList); + if (tableList) { + taosArrayDestroy(pTableListInfo->pTableList); + pTableListInfo->pTableList = tableList; + qDebug("retrieve tb group list from cache, numOfTables:%d", + (int32_t)taosArrayGetSize(pTableListInfo->pTableList)); + goto end; + } + } + + pUidTagList = taosArrayInit(8, sizeof(STUidTagInfo)); for (int32_t i = 0; i < rows; ++i) { STableKeyInfo* pkeyInfo = taosArrayGet(pTableListInfo->pTableList, i); STUidTagInfo info = {.uid = pkeyInfo->uid}; taosArrayPush(pUidTagList, &info); } - // int64_t stt = taosGetTimestampUs(); - code = metaGetTableTags(metaHandle, pTableListInfo->idInfo.suid, pUidTagList); + code = pAPI->metaFn.getTableTags(pVnode, pTableListInfo->idInfo.suid, pUidTagList); if (code != TSDB_CODE_SUCCESS) { goto end; } int32_t numOfTables = taosArrayGetSize(pUidTagList); - pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, metaHandle); + pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, pVnode, pAPI); if (pResBlock == NULL) { code = terrno; goto end; @@ -594,6 +635,12 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis info->groupId = calcGroupId(keyBuf, len); } + if (tsTagFilterCache) { + tableList = taosArrayDup(pTableListInfo->pTableList, NULL); + pAPI->metaFn.metaPutTbGroupToCache(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), + tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); + } + // int64_t st2 = taosGetTimestampUs(); // qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1); @@ -693,12 +740,12 @@ static FilterCondType checkTagCond(SNode* cond) { return FILTER_OTHER; } -static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond) { +static int32_t optimizeTbnameInCond(void* pVnode, int64_t suid, SArray* list, SNode* cond, SStorageAPI* pAPI) { int32_t ret = -1; int32_t ntype = nodeType(cond); if (ntype == QUERY_NODE_OPERATOR) { - ret = optimizeTbnameInCondImpl(metaHandle, list, cond); + ret = optimizeTbnameInCondImpl(pVnode, list, cond, pAPI); } if (ntype != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { @@ -717,7 +764,7 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list SListCell* cell = pList->pHead; for (int i = 0; i < len; i++) { if (cell == NULL) break; - if (optimizeTbnameInCondImpl(metaHandle, list, cell->pNode) == 0) { + if (optimizeTbnameInCondImpl(pVnode, list, cell->pNode, pAPI) == 0) { hasTbnameCond = true; break; } @@ -728,14 +775,15 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list taosArrayRemoveDuplicate(list, filterTableInfoCompare, NULL); if (hasTbnameCond) { - ret = metaGetTableTagsByUids(metaHandle, suid, list); + ret = pAPI->metaFn.getTableTagsByUid(pVnode, suid, list); } return ret; } // only return uid that does not contained in pExistedUidList -static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond) { +static int32_t optimizeTbnameInCondImpl(void* pVnode, SArray* pExistedUidList, SNode* pTagCond, + SStorageAPI* pStoreAPI) { if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) { return -1; } @@ -772,9 +820,9 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis char* name = taosArrayGetP(pTbList, i); uint64_t uid = 0; - if (metaGetTableUidByName(metaHandle, name, &uid) == 0) { + if (pStoreAPI->metaFn.getTableUidByName(pVnode, name, &uid) == 0) { ETableType tbType = TSDB_TABLE_MAX; - if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) { + if (pStoreAPI->metaFn.getTableTypeByName(pVnode, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) { if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) { STUidTagInfo s = {.uid = uid, .name = name, .pTagVal = NULL}; taosArrayPush(pExistedUidList, &s); @@ -798,24 +846,8 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis return -1; } -static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) { - if (pTagCond == NULL) { - return; - } - - char* payload = NULL; - int32_t len = 0; - nodesNodeToMsg(pTagCond, &payload, &len); - - tMD5Init(pContext); - tMD5Update(pContext, (uint8_t*)payload, (uint32_t)len); - tMD5Final(pContext); - - taosMemoryFree(payload); -} - -static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, - void* metaHandle) { +static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode, + SStorageAPI* pStorageAPI) { SSDataBlock* pResBlock = createDataBlock(); if (pResBlock == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -850,7 +882,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa if (p1->name != NULL) { STR_TO_VARSTR(str, p1->name); } else { // name is not retrieved during filter - metaGetTableNameByUid(metaHandle, p1->uid, str); + pStorageAPI->metaFn.getTableNameByUid(pVnode, p1->uid, str); } colDataSetVal(pColInfo, i, str, false); @@ -863,7 +895,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa if (p1->pTagVal == NULL) { colDataSetNULL(pColInfo, i); } else { - const char* p = metaGetTableTagVal(p1->pTagVal, pColInfo->info.type, &tagVal); + const char* p = pStorageAPI->metaFn.extractTagVal(p1->pTagVal, pColInfo->info.type, &tagVal); if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) { colDataSetNULL(pColInfo, i); @@ -923,18 +955,19 @@ static void copyExistedUids(SArray* pUidTagList, const SArray* pUidList) { } } -static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* metaHandle, - SIdxFltStatus status) { +static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* pVnode, + SIdxFltStatus status, SStorageAPI* pAPI) { if (pTagCond == NULL) { return TSDB_CODE_SUCCESS; } - terrno = TDB_CODE_SUCCESS; + terrno = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; SScalarParam output = {0}; + SArray* pUidTagList = NULL; tagFilterAssist ctx = {0}; ctx.colHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); @@ -954,12 +987,12 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; // int64_t stt = taosGetTimestampUs(); - SArray* pUidTagList = taosArrayInit(10, sizeof(STUidTagInfo)); + pUidTagList = taosArrayInit(10, sizeof(STUidTagInfo)); copyExistedUids(pUidTagList, pUidList); FilterCondType condType = checkTagCond(pTagCond); - int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->idInfo.suid, pUidTagList, pTagCond); + int32_t filter = optimizeTbnameInCond(pVnode, pListInfo->idInfo.suid, pUidTagList, pTagCond, pAPI); if (filter == 0) { // tbname in filter is activated, do nothing and return taosArrayClear(pUidList); @@ -972,9 +1005,9 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN terrno = 0; } else { if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) { - code = metaGetTableTagsByUids(metaHandle, pListInfo->idInfo.suid, pUidTagList); + code = pAPI->metaFn.getTableTagsByUid(pVnode, pListInfo->idInfo.suid, pUidTagList); } else { - code = metaGetTableTags(metaHandle, pListInfo->idInfo.suid, pUidTagList); + code = pAPI->metaFn.getTableTags(pVnode, pListInfo->idInfo.suid, pUidTagList); } if (code != TSDB_CODE_SUCCESS) { qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->idInfo.suid); @@ -988,7 +1021,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN goto end; } - pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, metaHandle); + pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, pVnode, pAPI); if (pResBlock == NULL) { code = terrno; goto end; @@ -1026,8 +1059,8 @@ end: return code; } -int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, - STableListInfo* pListInfo, const char* idstr) { +int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, + STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI) { int32_t code = TSDB_CODE_SUCCESS; size_t numOfTables = 0; @@ -1039,10 +1072,10 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SIdxFltStatus status = SFLT_NOT_INDEX; if (pScanNode->tableType != TSDB_SUPER_TABLE) { pListInfo->idInfo.uid = pScanNode->uid; - if (metaIsTableExist(metaHandle, pScanNode->uid)) { + if (pStorageAPI->metaFn.isTableExisted(pVnode, pScanNode->uid)) { taosArrayPush(pUidList, &pScanNode->uid); } - code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); + code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI); if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -1054,35 +1087,39 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, genTagFilterDigest(pTagCond, &context); bool acquired = false; - metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pUidList, - &acquired); + pStorageAPI->metaFn.getCachedTableList(pVnode, pScanNode->suid, context.digest, tListLen(context.digest), + pUidList, &acquired); if (acquired) { + digest[0] = 1; + memcpy(digest + 1, context.digest, tListLen(context.digest)); qDebug("retrieve table uid list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(pUidList)); goto _end; } } if (!pTagCond) { // no tag filter condition exists, let's fetch all tables of this super table - vnodeGetCtbIdList(pVnode, pScanNode->suid, pUidList); + pStorageAPI->metaFn.getChildTableList(pVnode, pScanNode->suid, pUidList); } else { // failed to find the result in the cache, let try to calculate the results if (pTagIndexCond) { - void* pIndex = tsdbGetIvtIdx(metaHandle); - SIndexMetaArg metaArg = { - .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = pIndex, .suid = pScanNode->uid}; + void* pIndex = pStorageAPI->metaFn.getInvertIndex(pVnode); + + SIndexMetaArg metaArg = {.metaEx = pVnode, + .idx = pStorageAPI->metaFn.storeGetIndexInfo(pVnode), + .ivtIdx = pIndex, + .suid = pScanNode->uid}; status = SFLT_NOT_INDEX; - code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status); + code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status, &pStorageAPI->metaFilter); if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake - qWarn("failed to get tableIds from index, suid:%" PRIu64, pScanNode->uid); - code = TDB_CODE_SUCCESS; + qDebug("failed to get tableIds from index, suid:%" PRIu64, pScanNode->uid); } else { qInfo("succ to get filter result, table num: %d", (int)taosArrayGetSize(pUidList)); } } } - code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); + code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI); if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -1099,7 +1136,9 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, memcpy(pPayload + sizeof(int32_t), taosArrayGet(pUidList, 0), numOfTables * sizeof(uint64_t)); } - metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1); + pStorageAPI->metaFn.putCachedTableList(pVnode, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1); + digest[0] = 1; + memcpy(digest + 1, context.digest, tListLen(context.digest)); } } @@ -1121,6 +1160,23 @@ _end: return code; } +int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray** tableList, void* pTaskInfo) { + SSubplan* pSubplan = (SSubplan*)node; + SScanPhysiNode pNode = {0}; + pNode.suid = suid; + pNode.uid = suid; + pNode.tableType = TSDB_SUPER_TABLE; + STableListInfo* pTableListInfo = tableListCreate(); + uint8_t digest[17] = {0}; + int code = + getTableList(pVnode, &pNode, pSubplan ? pSubplan->pTagCond : NULL, pSubplan ? pSubplan->pTagIndexCond : NULL, + pTableListInfo, digest, "qGetTableList", &((SExecTaskInfo*)pTaskInfo)->storageAPI); + *tableList = pTableListInfo->pTableList; + pTableListInfo->pTableList = NULL; + tableListDestroy(pTableListInfo); + return code; +} + size_t getTableTagsBufLen(const SNodeList* pGroups) { size_t keyLen = 0; @@ -1134,11 +1190,13 @@ size_t getTableTagsBufLen(const SNodeList* pGroups) { return keyLen; } -int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) { +int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId, + SStorageAPI* pAPI) { SMetaReader mr = {0}; - metaReaderInit(&mr, pMeta, 0); - if (metaGetTableEntryByUidCache(&mr, uid) != 0) { // table not exist - metaReaderClear(&mr); + + pAPI->metaReaderFn.initReader(&mr, pVnode, 0, &pAPI->metaFn); + if (pAPI->metaReaderFn.getEntryGetUidCache(&mr, uid) != 0) { // table not exist + pAPI->metaReaderFn.clearReader(&mr); return TSDB_CODE_PAR_TABLE_NOT_EXIST; } @@ -1157,7 +1215,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, REPLACE_NODE(pNew); } else { nodesDestroyList(groupNew); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return code; } @@ -1174,7 +1232,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, if (tTagIsJson(data)) { terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; nodesDestroyList(groupNew); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return terrno; } int32_t len = getJsonValueLen(data); @@ -1194,7 +1252,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, *pGroupId = calcGroupId(keyBuf, len); nodesDestroyList(groupNew); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return TSDB_CODE_SUCCESS; } @@ -1250,6 +1308,7 @@ int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod c.colId = pColNode->colId; c.srcSlotId = pColNode->slotId; c.dstSlotId = pNode->slotId; + c.dataType = pColNode->node.resType; taosArrayPush(pList, &c); } } @@ -1484,7 +1543,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu return TSDB_CODE_OUT_OF_MEMORY; } - SHashObj *pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + SHashObj* pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); for (int32_t i = 0; i < numOfOutput; ++i) { const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0)) { @@ -1512,7 +1571,8 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu return TSDB_CODE_SUCCESS; } -SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset) { +SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, + SFunctionStateStore* pStore) { SqlFunctionCtx* pFuncCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; @@ -1575,6 +1635,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, pCtx->numOfParams = pExpr->base.numOfParams; pCtx->param = pFunct->pParam; pCtx->saveHandle.currentPage = -1; + pCtx->pStore = pStore; } for (int32_t i = 1; i < numOfOutput; ++i) { @@ -1712,12 +1773,12 @@ int32_t convertFillType(int32_t mode) { return type; } -static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) { +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) { if (ascQuery) { - *w = getAlignQueryTimeWindow(pInterval, pInterval->precision, ts); + *w = getAlignQueryTimeWindow(pInterval, ts); } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - *w = getAlignQueryTimeWindow(pInterval, pInterval->precision, ts); + *w = getAlignQueryTimeWindow(pInterval, ts); int64_t key = w->skey; while (key < ts) { // moving towards end @@ -1734,7 +1795,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindo static STimeWindow doCalculateTimeWindow(int64_t ts, SInterval* pInterval) { STimeWindow w = {0}; - w.skey = taosTimeTruncate(ts, pInterval, pInterval->precision); + w.skey = taosTimeTruncate(ts, pInterval); w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; return w; } @@ -1768,6 +1829,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI if (pRow) { w = pRow->win; } + // in case of typical time window, we can calculate time window directly. if (w.skey > ts || w.ekey < ts) { w = doCalculateTimeWindow(ts, pInterval); @@ -1782,6 +1844,34 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI return w; } +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); + if (!IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { + tw->skey += pInterval->sliding * factor; + tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + return; + } + + // convert key to second + int64_t key = convertTimePrecision(tw->skey, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; + + int64_t duration = pInterval->sliding; + if (pInterval->slidingUnit == 'y') { + duration *= 12; + } + + struct tm tm; + time_t t = (time_t)key; + taosLocalTime(&t, &tm, NULL); + + int mon = (int)(tm.tm_year * 12 + tm.tm_mon + duration * factor); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); + + tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; +} + bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo) { return (pLimitInfo->limit.limit != -1 || pLimitInfo->limit.offset != -1 || pLimitInfo->slimit.limit != -1 || pLimitInfo->slimit.offset != -1); @@ -1811,9 +1901,7 @@ uint64_t tableListGetSize(const STableListInfo* pTableList) { return taosArrayGetSize(pTableList->pTableList); } -uint64_t tableListGetSuid(const STableListInfo* pTableList) { - return pTableList->idInfo.suid; -} +uint64_t tableListGetSuid(const STableListInfo* pTableList) { return pTableList->idInfo.suid; } STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) { if (taosArrayGetSize(pTableList->pTableList) == 0) { @@ -2000,11 +2088,11 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { memcpy(pTableListInfo->groupOffset, taosArrayGet(pList, 0), sizeof(int32_t) * pTableListInfo->numOfOuputGroups); taosArrayDestroy(pList); - return TDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } -int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, - bool groupSort) { +int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, + SNodeList* group, bool groupSort, uint8_t* digest, SStorageAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; bool groupByTbname = groupbyTbname(group); @@ -2020,16 +2108,19 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* if (groupSort && groupByTbname) { taosArraySort(pTableListInfo->pTableList, orderbyGroupIdComparFn); pTableListInfo->numOfOuputGroups = numOfTables; + } else if (groupByTbname && pScanNode->groupOrderScan){ + pTableListInfo->numOfOuputGroups = numOfTables; } else { pTableListInfo->numOfOuputGroups = 1; } } else { - code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo); + code = getColInfoResultForGroupby(pHandle->vnode, group, pTableListInfo, digest, pAPI); if (code != TSDB_CODE_SUCCESS) { return code; } + if (pScanNode->groupOrderScan) pTableListInfo->numOfOuputGroups = taosArrayGetSize(pTableListInfo->pTableList); - if (groupSort) { + if (groupSort || pScanNode->groupOrderScan) { code = sortTableGroup(pTableListInfo); } } @@ -2055,7 +2146,9 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags return TSDB_CODE_INVALID_PARA; } - int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, idStr); + uint8_t digest[17] = {0}; + int32_t code = getTableList(pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr, + &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { qError("failed to getTableList, code: %s", tstrerror(code)); return code; @@ -2073,7 +2166,8 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags return TSDB_CODE_SUCCESS; } - code = buildGroupIdMapForAllTables(pTableListInfo, pHandle, pGroupTags, groupSort); + code = buildGroupIdMapForAllTables(pTableListInfo, pHandle, pScanNode, pGroupTags, groupSort, digest, + &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 3f605f5f238d6fe9357e20d6ca4403f3455ee71c..a6059c7c4200551c654847cb00c3f738be9e36fc 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -14,6 +14,8 @@ */ #include "executor.h" +#include +#include #include "executorInt.h" #include "operator.h" #include "planner.h" @@ -21,7 +23,8 @@ #include "tdatablock.h" #include "tref.h" #include "tudf.h" -#include "vnode.h" + +#include "storageapi.h" static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT; int32_t exchangeObjRefPool = -1; @@ -89,6 +92,7 @@ static int32_t doSetStreamOpOpen(SOperatorInfo* pOperator, char* id) { qError("join not supported for stream block scan, %s" PRIx64, id); return TSDB_CODE_APP_ERROR; } + pOperator->status = OP_NOT_OPENED; return doSetStreamOpOpen(pOperator->pDownstream[0], id); } @@ -127,9 +131,9 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu return doSetStreamBlock(pOperator->pDownstream[0], input, numOfBlocks, type, id); } else { pOperator->status = OP_NOT_OPENED; - SStreamScanInfo* pInfo = pOperator->info; - qDebug("s-task:%s set source blocks:%d", id, (int32_t)numOfBlocks); + + qDebug("s-task:%s in this batch, %d blocks need to be processed", id, (int32_t)numOfBlocks); ASSERT(pInfo->validBlockIndex == 0 && taosArrayGetSize(pInfo->pBlockLists) == 0); if (type == STREAM_INPUT__MERGED_SUBMIT) { @@ -137,6 +141,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); taosArrayPush(pInfo->pBlockLists, pReq); } + pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_SUBMIT) { taosArrayPush(pInfo->pBlockLists, input); @@ -147,6 +152,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu SPackedData tmp = { .pDataBlock = pDataBlock }; taosArrayPush(pInfo->pBlockLists, &tmp); } + pInfo->blockType = STREAM_INPUT__DATA_BLOCK; } else { ASSERT(0); @@ -156,18 +162,18 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } } -void doSetTaskId(SOperatorInfo* pOperator) { +void doSetTaskId(SOperatorInfo* pOperator, SStorageAPI *pAPI) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { SStreamScanInfo* pStreamScanInfo = pOperator->info; if (pStreamScanInfo->pTableScanOp != NULL) { STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info; if (pScanInfo->base.dataReader != NULL) { - tsdbReaderSetId(pScanInfo->base.dataReader, pTaskInfo->id.str); + pAPI->tsdReader.tsdSetReaderTaskId(pScanInfo->base.dataReader, pTaskInfo->id.str); } } } else { - doSetTaskId(pOperator->pDownstream[0]); + doSetTaskId(pOperator->pDownstream[0], pAPI); } } @@ -177,7 +183,7 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { buildTaskId(taskId, queryId, pTaskInfo->id.str); // set the idstr for tsdbReader - doSetTaskId(pTaskInfo->pRoot); + doSetTaskId(pTaskInfo->pRoot, &pTaskInfo->storageAPI); } int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { @@ -249,11 +255,12 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id) { if (msg == NULL) { // create raw scan - SExecTaskInfo* pTaskInfo = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE); + SExecTaskInfo* pTaskInfo = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE, &pReaderHandle->api); if (NULL == pTaskInfo) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pTaskInfo->pRoot = createRawScanOperatorInfo(pReaderHandle, pTaskInfo); if (NULL == pTaskInfo->pRoot) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -261,13 +268,13 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 return NULL; } + pTaskInfo->storageAPI = pReaderHandle->api; qDebug("create raw scan task info completed, vgId:%d, %s", vgId, GET_TASKID(pTaskInfo)); return pTaskInfo; } - struct SSubplan* pPlan = NULL; - - int32_t code = qStringToSubplan(msg, &pPlan); + SSubplan* pPlan = NULL; + int32_t code = qStringToSubplan(msg, &pPlan); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; @@ -297,30 +304,33 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 return pTaskInfo; } -qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t vgId) { +qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t vgId, int32_t taskId) { if (msg == NULL) { return NULL; } - struct SSubplan* pPlan = NULL; - int32_t code = qStringToSubplan(msg, &pPlan); + SSubplan* pPlan = NULL; + int32_t code = qStringToSubplan(msg, &pPlan); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; } qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(readers, vgId, 0, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM); + code = qCreateExecTask(readers, vgId, taskId, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM); if (code != TSDB_CODE_SUCCESS) { + nodesDestroyNode((SNode*)pPlan); qDestroyTask(pTaskInfo); terrno = code; return NULL; } + qStreamInfoResetTimewindowFilter(pTaskInfo); return pTaskInfo; } -static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) { +static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr, + SStorageAPI* pAPI) { SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); int32_t numOfUids = taosArrayGetSize(tableIdList); if (numOfUids == 0) { @@ -336,11 +346,11 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S // let's discard the tables those are not created according to the queried super table. SMetaReader mr = {0}; - metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.vnode, 0, &pAPI->metaFn); for (int32_t i = 0; i < numOfUids; ++i) { uint64_t* id = (uint64_t*)taosArrayGet(tableIdList, i); - int32_t code = metaGetTableEntryByUid(&mr, *id); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, *id); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:%" PRIu64 " code:%s, %s", *id, tstrerror(terrno), idstr); continue; @@ -368,7 +378,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S if (pScanInfo->pTagCond != NULL) { bool qualified = false; STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid}; - code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified); + code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.vnode, &qualified, pAPI); if (code != TSDB_CODE_SUCCESS) { qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr); continue; @@ -383,7 +393,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S taosArrayPush(qa, id); } - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return qa; } @@ -393,7 +403,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI int32_t code = 0; if (isAdd) { - qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), id); + qDebug("try to add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), id); } // traverse to the stream scanner node to add this table id @@ -401,10 +411,10 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI SStreamScanInfo* pScanInfo = pInfo->info; if (isAdd) { // add new table id - SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id); + SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id, &pTaskInfo->storageAPI); int32_t numOfQualifiedTables = taosArrayGetSize(qa); qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id); - code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); + code = pTaskInfo->storageAPI.tqReaderFn.tqReaderAddTables(pScanInfo->tqReader, qa); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(qa); return code; @@ -433,8 +443,8 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI if (assignUid) { keyInfo.groupId = keyInfo.uid; } else { - code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, - &keyInfo.groupId); + code = getGroupIdFromTagsVal(pScanInfo->readHandle.vnode, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, + &keyInfo.groupId, &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(keyBuf); taosArrayDestroy(qa); @@ -456,7 +466,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI } else { // remove the table id in current list qDebug("%d remove child tables from the stream scanner, %s", (int32_t)taosArrayGetSize(tableIdList), id); taosWLockLatch(&pTaskInfo->lock); - code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList); + code = pTaskInfo->storageAPI.tqReaderFn.tqReaderRemoveTables(pScanInfo->tqReader, tableIdList); taosWUnLockLatch(&pTaskInfo->lock); } @@ -502,7 +512,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, } SDataSinkMgtCfg cfg = {.maxDataBlockNum = 500, .maxDataBlockNumPerQuery = 50}; - code = dsDataSinkMgtInit(&cfg); + code = dsDataSinkMgtInit(&cfg, &(*pTask)->storageAPI); if (code != TSDB_CODE_SUCCESS) { qError("failed to dsDataSinkMgtInit, code:%s, %s", tstrerror(code), (*pTask)->id.str); goto _error; @@ -637,23 +647,33 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { *pRes = NULL; int64_t curOwner = 0; - if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { + + // todo extract method + taosRLockLatch(&pTaskInfo->lock); + bool isKilled = isTaskKilled(pTaskInfo); + if (isKilled) { + clearStreamBlock(pTaskInfo->pRoot); + qDebug("%s already killed, abort", GET_TASKID(pTaskInfo)); + + taosRUnLockLatch(&pTaskInfo->lock); + return TSDB_CODE_SUCCESS; + } + + if (pTaskInfo->owner != 0) { qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner); pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; + + taosRUnLockLatch(&pTaskInfo->lock); return pTaskInfo->code; } + pTaskInfo->owner = threadId; + taosRUnLockLatch(&pTaskInfo->lock); + if (pTaskInfo->cost.start == 0) { pTaskInfo->cost.start = taosGetTimestampUs(); } - if (isTaskKilled(pTaskInfo)) { - clearStreamBlock(pTaskInfo->pRoot); - atomic_store_64(&pTaskInfo->owner, 0); - qDebug("%s already killed, abort", GET_TASKID(pTaskInfo)); - return TSDB_CODE_SUCCESS; - } - // error occurs, record the error code and return to client int32_t ret = setjmp(pTaskInfo->env); if (ret != TSDB_CODE_SUCCESS) { @@ -757,11 +777,13 @@ int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode) { qDebug("%s sync killed execTask", GET_TASKID(pTaskInfo)); setTaskKilled(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); + taosWLockLatch(&pTaskInfo->lock); while (qTaskIsExecuting(pTaskInfo)) { taosMsleep(10); } - pTaskInfo->code = rspCode; + taosWUnLockLatch(&pTaskInfo->lock); + return TSDB_CODE_SUCCESS; } @@ -856,19 +878,37 @@ int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) { } } -int32_t qStreamSourceRecoverStep1(qTaskInfo_t tinfo, int64_t ver) { +int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); - pTaskInfo->streamInfo.fillHistoryVer1 = ver; - pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE1; + + SStreamTaskInfo* pStreamInfo = &pTaskInfo->streamInfo; + + pStreamInfo->fillHistoryVer = *pVerRange; + pStreamInfo->fillHistoryWindow = *pWindow; + pStreamInfo->recoverStep = STREAM_RECOVER_STEP__PREPARE1; + + qDebug("%s step 1. set param for stream scanner for scan-history data, verRange:%" PRId64 " - %" PRId64 ", window:%" PRId64 + " - %" PRId64, + GET_TASKID(pTaskInfo), pStreamInfo->fillHistoryVer.minVer, pStreamInfo->fillHistoryVer.maxVer, pWindow->skey, + pWindow->ekey); return 0; } -int32_t qStreamSourceRecoverStep2(qTaskInfo_t tinfo, int64_t ver) { +int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); - pTaskInfo->streamInfo.fillHistoryVer2 = ver; - pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE2; + + SStreamTaskInfo* pStreamInfo = &pTaskInfo->streamInfo; + + pStreamInfo->fillHistoryVer = *pVerRange; + pStreamInfo->fillHistoryWindow = *pWindow; + pStreamInfo->recoverStep = STREAM_RECOVER_STEP__PREPARE2; + + qDebug("%s step 2. set param for stream scanner for scan-history data, verRange:%" PRId64 " - %" PRId64 + ", window:%" PRId64 " - %" PRId64, + GET_TASKID(pTaskInfo), pStreamInfo->fillHistoryVer.minVer, pStreamInfo->fillHistoryVer.maxVer, pWindow->skey, + pWindow->ekey); return 0; } @@ -879,55 +919,58 @@ int32_t qStreamRecoverFinish(qTaskInfo_t tinfo) { return 0; } -int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) { +int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SOperatorInfo* pOperator = pTaskInfo->pRoot; while (1) { - if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { + int32_t type = pOperator->operatorType; + if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; - ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || - pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); - ASSERT(pInfo->twAggSup.calTriggerSaved == 0 && pInfo->twAggSup.deleteMarkSaved == 0); + STimeWindowAggSupp* pSup = &pInfo->twAggSup; - qInfo("save stream param for interval: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0); - pInfo->twAggSup.calTriggerSaved = pInfo->twAggSup.calTrigger; - pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; - pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; - pInfo->twAggSup.deleteMark = INT64_MAX; + qInfo("save stream param for interval: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); + + pSup->calTriggerSaved = pSup->calTrigger; + pSup->deleteMarkSaved = pSup->deleteMark; + pSup->calTrigger = STREAM_TRIGGER_AT_ONCE; + pSup->deleteMark = INT64_MAX; pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; pInfo->ignoreExpiredData = false; - } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || - pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + STimeWindowAggSupp* pSup = &pInfo->twAggSup; + + ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0); - ASSERT(pInfo->twAggSup.calTriggerSaved == 0 && pInfo->twAggSup.deleteMarkSaved == 0); - qInfo("save stream param for session: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + qInfo("save stream param for session: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); - pInfo->twAggSup.calTriggerSaved = pInfo->twAggSup.calTrigger; - pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; - pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; - pInfo->twAggSup.deleteMark = INT64_MAX; + pSup->calTriggerSaved = pSup->calTrigger; + pSup->deleteMarkSaved = pSup->deleteMark; + pSup->calTrigger = STREAM_TRIGGER_AT_ONCE; + pSup->deleteMark = INT64_MAX; pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; pInfo->ignoreExpiredData = false; - } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; - ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || - pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); - ASSERT(pInfo->twAggSup.calTriggerSaved == 0 && pInfo->twAggSup.deleteMarkSaved == 0); + STimeWindowAggSupp* pSup = &pInfo->twAggSup; - qInfo("save stream param for state: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0); - pInfo->twAggSup.calTriggerSaved = pInfo->twAggSup.calTrigger; - pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; - pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; - pInfo->twAggSup.deleteMark = INT64_MAX; + qInfo("save stream param for state: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); + + pSup->calTriggerSaved = pSup->calTrigger; + pSup->deleteMarkSaved = pSup->deleteMark; + pSup->calTrigger = STREAM_TRIGGER_AT_ONCE; + pSup->deleteMark = INT64_MAX; pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; pInfo->ignoreExpiredData = false; } @@ -948,33 +991,37 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) { return 0; } -int32_t qStreamRestoreParam(qTaskInfo_t tinfo) { +int32_t qRestoreStreamOperatorOption(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + const char* id = GET_TASKID(pTaskInfo); SOperatorInfo* pOperator = pTaskInfo->pRoot; while (1) { - if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { + uint16_t type = pOperator->operatorType; + if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved; - qInfo("restore stream param for interval: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); - } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { + qInfo("%s restore stream agg executors param for interval: %d, %" PRId64, id, pInfo->twAggSup.calTrigger, + pInfo->twAggSup.deleteMark); + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved; - qInfo("restore stream param for session: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); - } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { + qInfo("%s restore stream agg executor param for session: %d, %" PRId64, id, pInfo->twAggSup.calTrigger, + pInfo->twAggSup.deleteMark); + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved; - qInfo("restore stream param for state: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + qInfo("%s restore stream agg executor param for state: %d, %" PRId64, id, pInfo->twAggSup.calTrigger, + pInfo->twAggSup.deleteMark); } // iterate operator tree @@ -988,7 +1035,6 @@ int32_t qStreamRestoreParam(qTaskInfo_t tinfo) { pOperator = pOperator->pDownstream[0]; } } - return 0; } bool qStreamRecoverScanFinished(qTaskInfo_t tinfo) { @@ -996,6 +1042,18 @@ bool qStreamRecoverScanFinished(qTaskInfo_t tinfo) { return pTaskInfo->streamInfo.recoverScanFinished; } +int32_t qStreamInfoResetTimewindowFilter(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + STimeWindow* pWindow = &pTaskInfo->streamInfo.fillHistoryWindow; + + qDebug("%s remove scan-history filter window:%" PRId64 "-%" PRId64 ", set new window:%" PRId64 "-%" PRId64, + GET_TASKID(pTaskInfo), pWindow->skey, pWindow->ekey, INT64_MIN, INT64_MAX); + + pWindow->skey = INT64_MIN; + pWindow->ekey = INT64_MAX; + return 0; +} + void* qExtractReaderFromStreamScanner(void* scanner) { SStreamScanInfo* pInfo = scanner; return (void*)pInfo->tqReader; @@ -1058,22 +1116,23 @@ void qStreamSetOpen(qTaskInfo_t tinfo) { pOperator->status = OP_NOT_OPENED; } -void verifyOffset(void *pWalReader, STqOffsetVal* pOffset){ - // if offset version is small than first version , let's seek to first version - taosThreadMutexLock(&((SWalReader*)pWalReader)->pWal->mutex); - int64_t firstVer = walGetFirstVer(((SWalReader*)pWalReader)->pWal); - taosThreadMutexUnlock(&((SWalReader*)pWalReader)->pWal->mutex); - - if (pOffset->version + 1 < firstVer){ - pOffset->version = firstVer - 1; - } -} - int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SOperatorInfo* pOperator = pTaskInfo->pRoot; const char* id = GET_TASKID(pTaskInfo); + if(subType == TOPIC_SUB_TYPE__COLUMN && pOffset->type == TMQ_OFFSET__LOG){ + pOperator = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, id); + if (pOperator == NULL) { + return -1; + } + SStreamScanInfo* pInfo = pOperator->info; + SStoreTqReader* pReaderAPI = &pTaskInfo->storageAPI.tqReaderFn; + SWalReader* pWalReader = pReaderAPI->tqReaderGetWalReader(pInfo->tqReader); + walReaderVerifyOffset(pWalReader, pOffset); + } // if pOffset equal to current offset, means continue consume if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { return 0; @@ -1092,12 +1151,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (pOffset->type == TMQ_OFFSET__LOG) { // todo refactor: move away - tsdbReaderClose(pScanBaseInfo->dataReader); + pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pScanBaseInfo->dataReader); pScanBaseInfo->dataReader = NULL; - verifyOffset(pInfo->tqReader->pWalReader, pOffset); - if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { - qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id); + SStoreTqReader* pReaderAPI = &pTaskInfo->storageAPI.tqReaderFn; + SWalReader* pWalReader = pReaderAPI->tqReaderGetWalReader(pInfo->tqReader); + walReaderVerifyOffset(pWalReader, pOffset); + if (pReaderAPI->tqReaderSeek(pInfo->tqReader, pOffset->version, id) < 0) { + qError("tqReaderSeek failed ver:%" PRId64 ", %s", pOffset->version, id); return -1; } } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { @@ -1152,8 +1213,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pScanInfo->scanTimes = 0; if (pScanBaseInfo->dataReader == NULL) { - int32_t code = tsdbReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, - pScanInfo->pResBlock, &pScanBaseInfo->dataReader, id, false); + int32_t code = pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, + pScanInfo->pResBlock, (void**) &pScanBaseInfo->dataReader, id, false, NULL); if (code != TSDB_CODE_SUCCESS) { qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id); terrno = code; @@ -1163,8 +1224,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s", uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } else { - tsdbSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); - tsdbReaderReset(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); + pTaskInfo->storageAPI.tsdReader.tsdSetQueryTableList(pScanBaseInfo->dataReader, &keyInfo, 1); + pTaskInfo->storageAPI.tsdReader.tsdReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } @@ -1186,14 +1247,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SOperatorInfo* p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id); STableListInfo* pTableListInfo = ((SStreamRawScanInfo*)(p->info))->pTableListInfo; - if (setForSnapShot(sContext, pOffset->uid) != 0) { + if (pAPI->snapshotFn.setForSnapShot(sContext, pOffset->uid) != 0) { qError("setDataForSnapShot error. uid:%" PRId64 " , %s", pOffset->uid, id); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } - SMetaTableInfo mtInfo = getUidfromSnapShot(sContext); - tsdbReaderClose(pInfo->dataReader); + SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.getMetaTableInfoFromSnapshot(sContext); + pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); @@ -1211,8 +1272,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); int32_t size = tableListGetSize(pTableListInfo); - tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL, - false); + pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**) &pInfo->dataReader, NULL, + false, NULL); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); @@ -1223,7 +1284,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; - if (setForSnapShot(sContext, pOffset->uid) != 0) { + if (pTaskInfo->storageAPI.snapshotFn.setForSnapShot(sContext, pOffset->uid) != 0) { qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; @@ -1232,7 +1293,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT id); } else if (pOffset->type == TMQ_OFFSET__LOG) { SStreamRawScanInfo* pInfo = pOperator->info; - tsdbReaderClose(pInfo->dataReader); + pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id); } @@ -1307,4 +1368,16 @@ SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo) { SOperatorInfo* pOperator = pTaskInfo->pRoot; extractTableList(pArray, pOperator); return pArray; -} \ No newline at end of file +} + +int32_t qStreamOperatorReleaseState(qTaskInfo_t tInfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tInfo; + pTaskInfo->pRoot->fpSet.releaseStreamStateFn(pTaskInfo->pRoot); + return 0; +} + +int32_t qStreamOperatorReloadState(qTaskInfo_t tInfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tInfo; + pTaskInfo->pRoot->fpSet.reloadStreamStateFn(pTaskInfo->pRoot); + return 0; +} diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 1ab677291ce96a9bb2c787e45eb1aeef3e4d907d..ebec9aa94e5e8c71b6bc0e6c198cd7234550b0b0 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -33,7 +33,7 @@ #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "vnode.h" +#include "storageapi.h" #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) @@ -77,12 +77,11 @@ static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); -static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep, - int32_t status); +static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, int32_t status); static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol); static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, - SGroupResInfo* pGroupResInfo, int32_t threshold); + SGroupResInfo* pGroupResInfo, int32_t threshold, bool ignoreGroup); SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize) { SFilePage* pData = NULL; @@ -275,7 +274,7 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc for (int32_t i = 0; i < numOfRows; ++i) { colDataSetDouble(pColInfo, i, &v); } - } else if (type == TSDB_DATA_TYPE_VARCHAR) { + } else if (type == TSDB_DATA_TYPE_VARCHAR || type == TSDB_DATA_TYPE_GEOMETRY) { char* tmp = taosMemoryMalloc(pFuncParam->param.nLen + VARSTR_HEADER_SIZE); STR_WITH_SIZE_TO_VARSTR(tmp, pFuncParam->param.pz, pFuncParam->param.nLen); for (int32_t i = 0; i < numOfRows; ++i) { @@ -442,15 +441,15 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB } ///////////////////////////////////////////////////////////////////////////////////////////// -STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key) { +STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key) { STimeWindow win = {0}; - win.skey = taosTimeTruncate(key, pInterval, precision); + win.skey = taosTimeTruncate(key, pInterval); /* * if the realSkey > INT64_MAX - pInterval->interval, the query duration between * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. */ - win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; if (win.ekey < win.skey) { win.ekey = INT64_MAX; } @@ -501,20 +500,26 @@ void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } } -void doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo) { +int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo) { if (pFilterInfo == NULL || pBlock->info.rows == 0) { - return; + return TSDB_CODE_SUCCESS; } SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock}; - int32_t code = filterSetDataFromSlotId(pFilterInfo, ¶m1); + SColumnInfoData* p = NULL; - SColumnInfoData* p = NULL; - int32_t status = 0; + int32_t code = filterSetDataFromSlotId(pFilterInfo, ¶m1); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } - // todo the keep seems never to be True?? - bool keep = filterExecute(pFilterInfo, pBlock, &p, NULL, param1.numOfCols, &status); - extractQualifiedTupleByFilterResult(pBlock, p, keep, status); + int32_t status = 0; + code = filterExecute(pFilterInfo, pBlock, &p, NULL, param1.numOfCols, &status); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + extractQualifiedTupleByFilterResult(pBlock, p, status); if (pColMatchInfo != NULL) { size_t size = taosArrayGetSize(pColMatchInfo->pList); @@ -529,152 +534,24 @@ void doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pCol } } } + code = TSDB_CODE_SUCCESS; +_err: colDataDestroy(p); taosMemoryFree(p); + return code; } -void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep, int32_t status) { - if (keep) { - return; - } - +void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, int32_t status) { int8_t* pIndicator = (int8_t*)p->pData; - int32_t totalRows = pBlock->info.rows; - if (status == FILTER_RESULT_ALL_QUALIFIED) { // here nothing needs to be done } else if (status == FILTER_RESULT_NONE_QUALIFIED) { pBlock->info.rows = 0; + } else if (status == FILTER_RESULT_PARTIAL_QUALIFIED) { + trimDataBlock(pBlock, pBlock->info.rows, (bool*)pIndicator); } else { - int32_t bmLen = BitmapLen(totalRows); - char* pBitmap = NULL; - int32_t maxRows = 0; - - size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); - // it is a reserved column for scalar function, and no data in this column yet. - if (pDst->pData == NULL) { - continue; - } - - int32_t numOfRows = 0; - if (IS_VAR_DATA_TYPE(pDst->info.type)) { - int32_t j = 0; - pDst->varmeta.length = 0; - - while (j < totalRows) { - if (pIndicator[j] == 0) { - j += 1; - continue; - } - - if (colDataIsNull_var(pDst, j)) { - colDataSetNull_var(pDst, numOfRows); - } else { - char* p1 = colDataGetVarData(pDst, j); - colDataSetVal(pDst, numOfRows, p1, false); - } - numOfRows += 1; - j += 1; - } - - if (maxRows < numOfRows) { - maxRows = numOfRows; - } - } else { - if (pBitmap == NULL) { - pBitmap = taosMemoryCalloc(1, bmLen); - } - - memcpy(pBitmap, pDst->nullbitmap, bmLen); - memset(pDst->nullbitmap, 0, bmLen); - - int32_t j = 0; - - switch (pDst->info.type) { - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - while (j < totalRows) { - if (pIndicator[j] == 0) { - j += 1; - continue; - } - - if (colDataIsNull_f(pBitmap, j)) { - colDataSetNull_f(pDst->nullbitmap, numOfRows); - } else { - ((int64_t*)pDst->pData)[numOfRows] = ((int64_t*)pDst->pData)[j]; - } - numOfRows += 1; - j += 1; - } - break; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - while (j < totalRows) { - if (pIndicator[j] == 0) { - j += 1; - continue; - } - if (colDataIsNull_f(pBitmap, j)) { - colDataSetNull_f(pDst->nullbitmap, numOfRows); - } else { - ((int32_t*)pDst->pData)[numOfRows] = ((int32_t*)pDst->pData)[j]; - } - numOfRows += 1; - j += 1; - } - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - while (j < totalRows) { - if (pIndicator[j] == 0) { - j += 1; - continue; - } - if (colDataIsNull_f(pBitmap, j)) { - colDataSetNull_f(pDst->nullbitmap, numOfRows); - } else { - ((int16_t*)pDst->pData)[numOfRows] = ((int16_t*)pDst->pData)[j]; - } - numOfRows += 1; - j += 1; - } - break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - while (j < totalRows) { - if (pIndicator[j] == 0) { - j += 1; - continue; - } - if (colDataIsNull_f(pBitmap, j)) { - colDataSetNull_f(pDst->nullbitmap, numOfRows); - } else { - ((int8_t*)pDst->pData)[numOfRows] = ((int8_t*)pDst->pData)[j]; - } - numOfRows += 1; - j += 1; - } - break; - } - } - - if (maxRows < numOfRows) { - maxRows = numOfRows; - } - } - - pBlock->info.rows = maxRows; - if (pBitmap != NULL) { - taosMemoryFree(pBitmap); - } + qError("unknown filter result type: %d", status); } } @@ -717,6 +594,7 @@ void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultR } } + blockDataEnsureCapacity(pBlock, pBlock->info.rows + pCtx[j].resultInfo->numOfRes); int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); if (TAOS_FAILED(code)) { qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); @@ -777,7 +655,7 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos } int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, - SGroupResInfo* pGroupResInfo, int32_t threshold) { + SGroupResInfo* pGroupResInfo, int32_t threshold, bool ignoreGroup) { SExprInfo* pExprInfo = pSup->pExprInfo; int32_t numOfExprs = pSup->numOfExprs; int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; @@ -804,20 +682,23 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS continue; } - if (pBlock->info.id.groupId == 0) { - pBlock->info.id.groupId = pPos->groupId; - } else { - // current value belongs to different group, it can't be packed into one datablock - if (pBlock->info.id.groupId != pPos->groupId) { - releaseBufPage(pBuf, page); - break; + if (!ignoreGroup) { + if (pBlock->info.id.groupId == 0) { + pBlock->info.id.groupId = pPos->groupId; + } else { + // current value belongs to different group, it can't be packed into one datablock + if (pBlock->info.id.groupId != pPos->groupId) { + releaseBufPage(pBuf, page); + break; + } } } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - blockDataEnsureCapacity(pBlock, pBlock->info.rows + pRow->numOfRows); - qDebug("datablock capacity not sufficient, expand to required:%" PRId64 ", current capacity:%d, %s", - (pRow->numOfRows + pBlock->info.rows), pBlock->info.capacity, GET_TASKID(pTaskInfo)); + uint32_t newSize = pBlock->info.rows + pRow->numOfRows + ((numOfRows - i) > 1 ? 1 : 0); + blockDataEnsureCapacity(pBlock, newSize); + qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", + newSize, pBlock->info.capacity, GET_TASKID(pTaskInfo)); // todo set the pOperator->resultInfo size } @@ -841,6 +722,8 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSDataBlock* pBlock = pbInfo->pRes; // set output datablock version @@ -854,15 +737,16 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr // clear the existed group id pBlock->info.id.groupId = 0; ASSERT(!pbInfo->mergeResultBlock); - doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold); + doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold, false); void* tbname = NULL; - if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { + if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); } - streamFreeVal(tbname); + + pAPI->stateStore.streamStateFreeVal(tbname); } void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, @@ -881,10 +765,10 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG // clear the existed group id pBlock->info.id.groupId = 0; if (!pbInfo->mergeResultBlock) { - doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold); + doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold, false); } else { while (hasRemainResults(pGroupResInfo)) { - doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold); + doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold, true); if (pBlock->info.rows >= pOperator->resultInfo.threshold) { break; } @@ -916,8 +800,13 @@ void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) { int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz) { *defaultPgsz = 4096; + uint32_t last = *defaultPgsz; while (*defaultPgsz < rowSize * 4) { *defaultPgsz <<= 1u; + if (*defaultPgsz < last) { + return TSDB_CODE_INVALID_PARA; + } + last = *defaultPgsz; } // The default buffer for each operator in query is 10MB. @@ -926,6 +815,9 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul *defaultBufsz = 4096 * 2560; if ((*defaultBufsz) <= (*defaultPgsz)) { (*defaultBufsz) = (*defaultPgsz) * 4; + if (*defaultBufsz < ((int64_t)(*defaultPgsz)) * 4) { + return TSDB_CODE_INVALID_PARA; + } } return 0; @@ -973,11 +865,11 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { return NULL; } -int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr) { +int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr, SFunctionStateStore* pStore) { pSup->pExprInfo = pExprInfo; pSup->numOfExprs = numOfExpr; if (pSup->pExprInfo != NULL) { - pSup->pCtx = createSqlFunctionCtx(pExprInfo, numOfExpr, &pSup->rowEntryInfoOffset); + pSup->pCtx = createSqlFunctionCtx(pExprInfo, numOfExpr, &pSup->rowEntryInfoOffset, pStore); if (pSup->pCtx == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1063,122 +955,22 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* return TSDB_CODE_SUCCESS; } -int32_t resultRowEncode(void* k, int32_t* size, char* buf) { - // SResultRow* key = k; - // int len = 0; - // int struLen = *size; - // len += taosEncodeFixedI32((void**)&buf, key->pageId); - - // uint32_t offset = key->offset; - // len += taosEncodeFixedU32((void**)&buf, offset); - - // len += taosEncodeFixedI8((void**)&buf, key->startInterp); - // len += taosEncodeFixedI8((void**)&buf, key->endInterp); - // len += taosEncodeFixedI8((void**)&buf, key->closed); - // len += taosEncodeFixedU32((void**)&buf, key->numOfRows); - - // len += taosEncodeFixedI64((void**)&buf, key->win.skey); - // len += taosEncodeFixedI64((void**)&buf, key->win.ekey); - - // int32_t numOfEntryInfo = (struLen - sizeof(SResultRow)) / sizeof(struct SResultRowEntryInfo); - // len += taosEncodeFixedI32((void**)&buf, numOfEntryInfo); - // for (int i = 0; i < numOfEntryInfo; i++) { - // SResultRowEntryInfo* p = &key->pEntryInfo[i]; - - // uint8_t value = p->initialized ? 1 : 0; - // len += taosEncodeFixedU8((void**)&buf, value); - - // value = p->complete ? 1 : 0; - // len += taosEncodeFixedU8((void**)&buf, value); - - // value = p->isNullRes; - // len += taosEncodeFixedU8((void**)&buf, value); - - // len += taosEncodeFixedU16((void**)&buf, p->numOfRes); - // } - // { - // char* strBuf = taosMemoryCalloc(1, *size * 100); - // resultRowToString(key, *size, strBuf); - // qWarn("encode result row:%s", strBuf); - // } - - // return len; - return 0; -} - -int32_t resultRowDecode(void** k, size_t size, char* buf) { - // char* p1 = buf; - // int32_t numOfEntryInfo = 0; - // uint32_t entryOffset = sizeof(int32_t) + sizeof(uint32_t) + sizeof(int8_t) + sizeof(int8_t) + sizeof(int8_t) + - // sizeof(uint32_t) + sizeof(int64_t) + sizeof(int64_t); - // taosDecodeFixedI32(p1 + entryOffset, &numOfEntryInfo); - - // char* p = buf; - // size = sizeof(SResultRow) + numOfEntryInfo * sizeof(SResultRowEntryInfo); - // SResultRow* key = taosMemoryCalloc(1, size); - - // p = taosDecodeFixedI32(p, (int32_t*)&key->pageId); - // uint32_t offset = 0; - // p = taosDecodeFixedU32(p, &offset); - // key->offset = offset; - - // p = taosDecodeFixedI8(p, (int8_t*)(&key->startInterp)); - // p = taosDecodeFixedI8(p, (int8_t*)(&key->endInterp)); - // p = taosDecodeFixedI8(p, (int8_t*)&key->closed); - // p = taosDecodeFixedU32(p, &key->numOfRows); - - // p = taosDecodeFixedI64(p, &key->win.skey); - // p = taosDecodeFixedI64(p, &key->win.ekey); - // p = taosDecodeFixedI32(p, &numOfEntryInfo); - // for (int i = 0; i < numOfEntryInfo; i++) { - // SResultRowEntryInfo* pInfo = &key->pEntryInfo[i]; - // uint8_t value = 0; - // p = taosDecodeFixedU8(p, &value); - // pInfo->initialized = (value == 1) ? true : false; - - // p = taosDecodeFixedU8(p, &value); - // pInfo->complete = (value == 1) ? true : false; - - // p = taosDecodeFixedU8(p, &value); - // pInfo->isNullRes = value; - - // p = taosDecodeFixedU16(p, &pInfo->numOfRes); - // } - // *k = key; - - // { - // char* strBuf = taosMemoryCalloc(1, size * 100); - // resultRowToString(key, size, strBuf); - // qWarn("decode result row:%s", strBuf); - // } - // return size; - return 0; -} - -int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize) { - // char* buf = taosMemoryCalloc(1, resSize * 10); - // int len = resultRowEncode((void*)pResult, &resSize, buf); - // char* buf = taosMemoryCalloc(1, resSize); - // memcpy(buf, pResult, resSize); - streamStatePut(pState, pKey, (char*)pResult, resSize); - // taosMemoryFree(buf); - return TSDB_CODE_SUCCESS; -} - -int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult) { - streamStateReleaseBuf(pState, pKey, pResult); +int32_t releaseOutputBuf(void* pState, SWinKey* pKey, SResultRow* pResult, SStateStore* pAPI) { + pAPI->streamStateReleaseBuf(pState, pKey, pResult); return TSDB_CODE_SUCCESS; } -int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size) { - streamStateSessionPut(pState, key, (const void*)buf, size); - releaseOutputBuf(pState, NULL, (SResultRow*)buf); +int32_t saveSessionDiscBuf(void* pState, SSessionKey* key, void* buf, int32_t size, SStateStore* pAPI) { + pAPI->streamStateSessionPut(pState, key, (const void*)buf, size); + releaseOutputBuf(pState, NULL, (SResultRow*)buf, pAPI); return TSDB_CODE_SUCCESS; } -int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, +int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SExprInfo* pExprInfo = pSup->pExprInfo; int32_t numOfExprs = pSup->numOfExprs; int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; @@ -1190,10 +982,12 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta SSessionKey* pKey = taosArrayGet(pGroupResInfo->pRows, i); int32_t size = 0; void* pVal = NULL; - int32_t code = streamStateSessionGet(pState, pKey, &pVal, &size); - ASSERT(code == 0); + int32_t code = pAPI->stateStore.streamStateSessionGet(pState, pKey, &pVal, &size); + // ASSERT(code == 0); if (code == -1) { - // coverity scan + // for history + qWarn("===stream===not found session result key:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64, pKey->win.skey, + pKey->win.ekey, pKey->groupId); pGroupResInfo->index += 1; continue; } @@ -1202,7 +996,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta // no results, continue to check the next one if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; - releaseOutputBuf(pState, NULL, pRow); + releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore); continue; } @@ -1210,23 +1004,23 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta pBlock->info.id.groupId = pKey->groupId; void* tbname = NULL; - if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { + if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); } - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } else { // current value belongs to different group, it can't be packed into one datablock if (pBlock->info.id.groupId != pKey->groupId) { - releaseOutputBuf(pState, NULL, pRow); + releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore); break; } } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { ASSERT(pBlock->info.rows > 0); - releaseOutputBuf(pState, NULL, pRow); + releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore); break; } @@ -1257,46 +1051,22 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta pBlock->info.dataLoad = 1; pBlock->info.rows += pRow->numOfRows; - releaseOutputBuf(pState, NULL, pRow); + releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore); } blockDataUpdateTsWindow(pBlock, 0); return TSDB_CODE_SUCCESS; } -void qStreamCloseTsdbReader(void* task) { - if (task == NULL) { - return; +void streamOpReleaseState(SOperatorInfo* pOperator) { + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.releaseStreamStateFn) { + downstream->fpSet.releaseStreamStateFn(downstream); } +} - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)task; - SOperatorInfo* pOp = pTaskInfo->pRoot; - - qDebug("stream close tsdb reader, reset status uid:%" PRId64 " ts:%" PRId64, pTaskInfo->streamInfo.currentOffset.uid, - pTaskInfo->streamInfo.currentOffset.ts); - - // todo refactor, other thread may already use this read to extract data. - pTaskInfo->streamInfo.currentOffset = (STqOffsetVal){0}; - while (pOp->numOfDownstream == 1 && pOp->pDownstream[0]) { - SOperatorInfo* pDownstreamOp = pOp->pDownstream[0]; - if (pDownstreamOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - SStreamScanInfo* pInfo = pDownstreamOp->info; - if (pInfo->pTableScanOp) { - STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - - setOperatorCompleted(pInfo->pTableScanOp); - while (pTaskInfo->owner != 0) { - taosMsleep(100); - qDebug("wait for the reader stopping"); - } - - tsdbReaderClose(pTSInfo->base.dataReader); - pTSInfo->base.dataReader = NULL; - - // restore the status, todo refactor. - pInfo->pTableScanOp->status = OP_OPENED; - pTaskInfo->status = TASK_NOT_COMPLETED; - return; - } - } +void streamOpReloadState(SOperatorInfo* pOperator) { + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.reloadStreamStateFn) { + downstream->fpSet.reloadStreamStateFn(downstream); } } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 5101e369921e8a6aaa0f37cf195a66ec3c84fdcd..80c88a803effb72588097c4b86acd808dde78354 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -61,25 +61,28 @@ typedef struct SFillOperatorInfo { SExprSupp noFillExprSupp; } SFillOperatorInfo; +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order); static void destroyFillOperatorInfo(void* param); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, - SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { + SResultInfo* pResultInfo, int32_t order) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; SSDataBlock* pResBlock = pInfo->pFinalRes; - int32_t order = TSDB_ORDER_ASC; +// int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; - getTableScanInfo(pOperator, &order, &scanFlag, false); - - int64_t ekey = pInfo->existNewGroupBlock->info.window.ekey; +// getTableScanInfo(pOperator, &order, &scanFlag, false); taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); blockDataCleanup(pInfo->pRes); doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag); - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ekey); + revisedFillStartKey(pInfo, pInfo->existNewGroupBlock, order); + + int64_t ts = (order == TSDB_ORDER_ASC)? pInfo->existNewGroupBlock->info.window.ekey:pInfo->existNewGroupBlock->info.window.skey; + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts); + taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); int32_t numOfResultRows = pResultInfo->capacity - pResBlock->info.rows; @@ -90,7 +93,7 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp } static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, - SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { + SResultInfo* pResultInfo, int32_t order) { if (taosFillHasMoreResults(pInfo->pFillInfo)) { int32_t numOfResultRows = pResultInfo->capacity - pInfo->pFinalRes->info.rows; taosFillResultDataBlock(pInfo->pFillInfo, pInfo->pFinalRes, numOfResultRows); @@ -100,7 +103,7 @@ static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOpera // handle the cached new group data block if (pInfo->existNewGroupBlock) { - doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, order); } } @@ -119,6 +122,53 @@ void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int pInfo->pRes->info.id.groupId = pBlock->info.id.groupId; } +// todo refactor: decide the start key according to the query time range. +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order) { + if (order == TSDB_ORDER_ASC) { + int64_t skey = pBlock->info.window.skey; + if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey); + } else if (pInfo->pFillInfo->start < skey) { + int64_t t = skey; + SInterval* pInterval = &pInfo->pFillInfo->interval; + + while(1) { + int64_t prev = taosTimeAdd(t, -pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + if (prev <= pInfo->pFillInfo->start) { + t = prev; + break; + } + t = prev; + } + + // todo time window chosen problem: t or prev value? + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t); + } + } else { + int64_t ekey = pBlock->info.window.ekey; + if (ekey > pInfo->pFillInfo->start) { + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey); + } else if (ekey < pInfo->pFillInfo->start) { + int64_t t = ekey; + SInterval* pInterval = &pInfo->pFillInfo->interval; + + while(1) { + int64_t prev = taosTimeAdd(t, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + if (prev >= pInfo->pFillInfo->start) { + t = prev; + break; + } + t = prev; + } + + // todo time window chosen problem: t or prev value? + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t); + } + } +} + static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { SFillOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -128,17 +178,24 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { blockDataCleanup(pResBlock); - int32_t order = TSDB_ORDER_ASC; - int32_t scanFlag = MAIN_SCAN; - getTableScanInfo(pOperator, &order, &scanFlag, false); + int32_t order = pInfo->pFillInfo->order; + + SOperatorInfo* pDownstream = pOperator->pDownstream[0]; +#if 0 + // the scan order may be different from the output result order for agg interval operator. + if (pDownstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL) { + order = ((SIntervalAggOperatorInfo*) pDownstream->info)->resultTsOrder; + } else { + order = pInfo->pFillInfo->order; + } +#endif - doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, order); if (pResBlock->info.rows > 0) { pResBlock->info.id.groupId = pInfo->curGroupId; return pResBlock; } - SOperatorInfo* pDownstream = pOperator->pDownstream[0]; while (1) { SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); if (pBlock == NULL) { @@ -150,23 +207,25 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { taosFillSetStartInfo(pInfo->pFillInfo, 0, pInfo->win.ekey); } else { + pResBlock->info.scanFlag = pBlock->info.scanFlag; pBlock->info.dataLoad = 1; blockDataUpdateTsWindow(pBlock, pInfo->primarySrcSlotId); blockDataCleanup(pInfo->pRes); blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows); blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows); - doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); + doApplyScalarCalculation(pOperator, pBlock, order, pBlock->info.scanFlag); + + if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) { + if (pInfo->curGroupId == 0 && taosFillNotStarted(pInfo->pFillInfo)) { + revisedFillStartKey(pInfo, pBlock, order); + } - if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.id.groupId) { pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block pInfo->totalInputRows += pInfo->pRes->info.rows; - if (order == pInfo->pFillInfo->order) { - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey); - } else { - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey); - } + int64_t ts = (order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey; + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts); taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); } else if (pInfo->curGroupId != pBlock->info.id.groupId) { // the new group data block pInfo->existNewGroupBlock = pBlock; @@ -190,7 +249,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { return pResBlock; } - doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, order); if (pResBlock->info.rows >= pOperator->resultInfo.threshold || pBlock == NULL) { pResBlock->info.id.groupId = pInfo->curGroupId; return pResBlock; @@ -198,7 +257,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { } else if (pInfo->existNewGroupBlock) { // try next group blockDataCleanup(pResBlock); - doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, order); if (pResBlock->info.rows > pResultInfo->threshold) { pResBlock->info.id.groupId = pInfo->curGroupId; return pResBlock; @@ -256,11 +315,11 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); - int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, startKey); - w = getFirstQualifiedTimeWindow(startKey, &w, pInterval, order); + int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, +// STimeWindow w = {0}; +// getInitialStartTimeWindow(pInterval, startKey, &w, order == TSDB_ORDER_ASC); + pInfo->pFillInfo = taosCreateFillInfo(startKey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); if (order == TSDB_ORDER_ASC) { @@ -338,7 +397,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* goto _error; } - code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs); + code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -348,14 +407,14 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval : &((SIntervalAggOperatorInfo*)downstream->info)->interval; - int32_t order = (pPhyFillNode->inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; + int32_t order = (pPhyFillNode->node.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; int32_t type = convertFillType(pPhyFillNode->mode); SResultInfo* pResultInfo = &pOperator->resultInfo; initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr); + code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -402,13 +461,13 @@ _error: TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) { STimeWindow win = {.skey = ts, .ekey = ts}; - getNextIntervalWindow(pInterval, &win, TSDB_ORDER_ASC); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); return win.skey; } TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { STimeWindow win = {.skey = ts, .ekey = ts}; - getNextIntervalWindow(pInterval, &win, TSDB_ORDER_DESC); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_DESC); return win.skey; } @@ -443,9 +502,13 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); tSimpleHashCleanup(pFillSup->pResMap); pFillSup->pResMap = NULL; - releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); - pFillSup->cur.pRowVal = NULL; cleanupExprSupp(&pFillSup->notFillExprSup); + if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) { + taosMemoryFree(pFillSup->cur.pRowVal); + } + taosMemoryFree(pFillSup->prev.pRowVal); + taosMemoryFree(pFillSup->next.pRowVal); + taosMemoryFree(pFillSup->nextNext.pRowVal); taosMemoryFree(pFillSup); return NULL; @@ -487,77 +550,85 @@ static void destroyStreamFillOperatorInfo(void* param) { static void resetFillWindow(SResultRowData* pRowData) { pRowData->key = INT64_MIN; - pRowData->pRowVal = NULL; + taosMemoryFreeClear(pRowData->pRowVal); } -void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, SStreamState* pState) { +void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) { + if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) { + resetFillWindow(&pFillSup->cur); + } else { + pFillSup->cur.key = INT64_MIN; + pFillSup->cur.pRowVal = NULL; + } resetFillWindow(&pFillSup->prev); - releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); - resetFillWindow(&pFillSup->cur); resetFillWindow(&pFillSup->next); resetFillWindow(&pFillSup->nextNext); } void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { - SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState; - resetPrevAndNextWindow(pFillSup, pState); + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + + void* pState = pOperator->pTaskInfo->streamInfo.pState; + resetPrevAndNextWindow(pFillSup, pState, pAPI); SWinKey key = {.ts = ts, .groupId = groupId}; int32_t curVLen = 0; - int32_t code = streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); + + int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); ASSERT(code == TSDB_CODE_SUCCESS); pFillSup->cur.key = key.ts; } void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { - SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState; - resetPrevAndNextWindow(pFillSup, pState); + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + void* pState = pOperator->pTaskInfo->streamInfo.pState; + resetPrevAndNextWindow(pFillSup, pState, pAPI); SWinKey key = {.ts = ts, .groupId = groupId}; void* curVal = NULL; int32_t curVLen = 0; - int32_t code = streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); + int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); ASSERT(code == TSDB_CODE_SUCCESS); pFillSup->cur.key = key.ts; pFillSup->cur.pRowVal = curVal; - SStreamStateCur* pCur = streamStateFillSeekKeyPrev(pState, &key); + SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key); SWinKey preKey = {.ts = INT64_MIN, .groupId = groupId}; void* preVal = NULL; int32_t preVLen = 0; - code = streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); if (code == TSDB_CODE_SUCCESS) { pFillSup->prev.key = preKey.ts; pFillSup->prev.pRowVal = preVal; - code = streamStateCurNext(pState, pCur); + code = pAPI->stateStore.streamStateCurNext(pState, pCur); ASSERT(code == TSDB_CODE_SUCCESS); - code = streamStateCurNext(pState, pCur); + code = pAPI->stateStore.streamStateCurNext(pState, pCur); if (code != TSDB_CODE_SUCCESS) { - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); pCur = NULL; } } else { - streamStateFreeCur(pCur); - pCur = streamStateFillSeekKeyNext(pState, &key); + pAPI->stateStore.streamStateFreeCur(pCur); + pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key); } SWinKey nextKey = {.ts = INT64_MIN, .groupId = groupId}; void* nextVal = NULL; int32_t nextVLen = 0; - code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); if (code == TSDB_CODE_SUCCESS) { pFillSup->next.key = nextKey.ts; pFillSup->next.pRowVal = nextVal; if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { - code = streamStateCurNext(pState, pCur); + code = pAPI->stateStore.streamStateCurNext(pState, pCur); if (code == TSDB_CODE_SUCCESS) { SWinKey nextNextKey = {.groupId = groupId}; void* nextNextVal = NULL; int32_t nextNextVLen = 0; - code = streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); if (code == TSDB_CODE_SUCCESS) { pFillSup->nextNext.key = nextNextKey.ts; pFillSup->nextNext.pRowVal = nextNextVal; @@ -565,7 +636,7 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, } } } - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); } static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; } @@ -922,8 +993,10 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* } static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SWinKey key = {.groupId = groupId, .ts = pRow->key}; - int32_t code = streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); + int32_t code = pAPI->stateStore.streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 " code:%d", key.ts, key.groupId, code); ASSERT(code == TSDB_CODE_SUCCESS); } @@ -996,9 +1069,10 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) { SSDataBlock* pBlock = pInfo->pSrcBlock; uint64_t groupId = pBlock->info.id.groupId; SSDataBlock* pRes = pInfo->pRes; + SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol); + TSKEY* tsCol = (TSKEY*)pTsCol->pData; pRes->info.id.groupId = groupId; - SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol); - TSKEY* tsCol = (TSKEY*)pTsCol->pData; + pInfo->srcRowIndex++; if (pInfo->srcRowIndex == 0) { keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); @@ -1021,7 +1095,8 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) { } static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { - SStreamState* pState = pOp->pTaskInfo->streamInfo.pState; + SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; + void* pState = pOp->pTaskInfo->streamInfo.pState; SSDataBlock* pBlock = delRes; SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); @@ -1041,14 +1116,14 @@ static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_ SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); void* tbname = NULL; - streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname); + pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname); if (tbname == NULL) { colDataSetNULL(pTableCol, pBlock->info.rows); } else { char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } pBlock->info.rows++; @@ -1070,12 +1145,13 @@ static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY end } static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStreamFillOperatorInfo* pInfo = pOperator->info; getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup); setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo); SWinKey key = {.ts = startTs, .groupId = groupId}; if (!pInfo->pFillInfo->needFill) { - streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); } else { STimeRange tw = { @@ -1093,6 +1169,8 @@ static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKE } static void doDeleteFillFinalize(SOperatorInfo* pOperator) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillInfo* pFillInfo = pInfo->pFillInfo; int32_t size = taosArrayGetSize(pFillInfo->delRanges); @@ -1109,17 +1187,16 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) { pInfo->pRes->info.id.groupId = range->groupId; } SWinKey key = {.ts = range->skey, .groupId = range->groupId}; - streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); } } static void doDeleteFillResult(SOperatorInfo* pOperator) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamFillOperatorInfo* pInfo = pOperator->info; - SStreamFillSupporter* pFillSup = pInfo->pFillSup; SStreamFillInfo* pFillInfo = pInfo->pFillInfo; SSDataBlock* pBlock = pInfo->pSrcDelBlock; - SSDataBlock* pRes = pInfo->pRes; - SSDataBlock* pDelRes = pInfo->pDelRes; SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* tsStarts = (TSKEY*)pStartCol->pData; @@ -1130,7 +1207,8 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { TSKEY endTs = ts; uint64_t groupId = groupIds[pInfo->srcDelRowIndex]; SWinKey key = {.ts = ts, .groupId = groupId}; - SStreamStateCur* pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); + SStreamStateCur* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); + if (!pCur) { pInfo->srcDelRowIndex++; continue; @@ -1152,15 +1230,15 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; if (delTs == nextKey.ts) { - code = streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); + code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); if (code == TSDB_CODE_SUCCESS) { - code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen); + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen); } // ts will be deleted later if (delTs != ts) { - streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); - streamStateFreeCur(pCur); - pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); + pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); + pAPI->stateStore.streamStateFreeCur(pCur); + pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); } endTs = TMAX(delTs, nextKey.ts - 1); if (code != TSDB_CODE_SUCCESS) { @@ -1169,9 +1247,11 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { } pInfo->srcDelRowIndex++; } - streamStateFreeCur(pCur); + + pAPI->stateStore.streamStateFreeCur(pCur); doDeleteFillResultImpl(pOperator, ts, endTs, groupId); } + pFillInfo->current = pFillInfo->end + 1; } @@ -1230,7 +1310,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { SSDataBlock* fillResult = NULL; SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { - if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows) { + if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows || pInfo->pSrcBlock->info.rows == 0) { // If there are delete datablocks, we receive them first. SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -1269,7 +1349,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { case STREAM_PULL_DATA: { doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); - pInfo->srcRowIndex = 0; + pInfo->srcRowIndex = -1; } break; case STREAM_CREATE_CHILD_TABLE: { return pBlock; @@ -1322,7 +1402,7 @@ static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { } static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, - SExprInfo* pFillExprInfo, int32_t numOfFillCols) { + SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) { SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); if (!pFillSup) { return NULL; @@ -1335,6 +1415,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod pFillSup->type = convertFillType(pPhyFillNode->mode); pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols; pFillSup->interval = *pInterval; + pFillSup->pAPI = pAPI; int32_t code = initResultBuf(pFillSup); if (code != TSDB_CODE_SUCCESS) { @@ -1343,7 +1424,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod } SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); - code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols); + code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore); if (code != TSDB_CODE_SUCCESS) { destroyStreamFillSupporter(pFillSup); return NULL; @@ -1414,7 +1495,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; int32_t numOfFillCols = 0; SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); - pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols); + pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI); if (!pInfo->pFillSup) { goto _error; } @@ -1440,11 +1521,11 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi float v = 0; GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); SET_TYPED_DATA(pCell->pData, pCell->type, v); - } else if (pCell->type == TSDB_DATA_TYPE_DOUBLE) { + } else if (IS_FLOAT_TYPE(pCell->type)) { double v = 0; GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); SET_TYPED_DATA(pCell->pData, pCell->type, v); - } else if (IS_SIGNED_NUMERIC_TYPE(pCell->type)) { + } else if (IS_INTEGER_TYPE(pCell->type)) { int64_t v = 0; GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); SET_TYPED_DATA(pCell->pData, pCell->type, v); @@ -1479,16 +1560,17 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi goto _error; } - code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } - pInfo->srcRowIndex = 0; + pInfo->srcRowIndex = -1; setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 7ad8821ff950666a071d94158a3905f391d1615b..9228c923a6893ccacbbf51cf3ab90958f7b02bb4 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -378,9 +378,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { return buildGroupResultDataBlock(pOperator); } - int32_t order = TSDB_ORDER_ASC; - int32_t scanFlag = MAIN_SCAN; - + int32_t order = pInfo->binfo.inputTsOrder; int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; @@ -390,13 +388,10 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { break; } - int32_t code = getTableScanInfo(pOperator, &order, &scanFlag, false); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); - } + pInfo->binfo.pRes->info.scanFlag = pBlock->info.scanFlag; // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(&pOperator->exprSupp, pBlock, order, scanFlag, true); + setInputDataBlock(&pOperator->exprSupp, pBlock, order, pBlock->info.scanFlag, true); // there is an scalar expression that needs to be calculated right before apply the group aggregation. if (pInfo->scalarSup.pExprInfo != NULL) { @@ -451,7 +446,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* } pInfo->pGroupCols = extractColumnInfo(pAggNode->pGroupKeys); - code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -467,7 +462,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -481,6 +476,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* setOperatorInfo(pOperator, "GroupbyAggOperator", 0, true, OP_NOT_OPENED, pInfo, pTaskInfo); pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; + pInfo->binfo.inputTsOrder = pAggNode->node.inputTsOrder; + pInfo->binfo.outputTsOrder = pAggNode->node.outputTsOrder; pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, optrDefaultBufFn, NULL); @@ -647,6 +644,8 @@ uint64_t calcGroupId(char* pData, int32_t len) { // NOTE: only extract the initial 8 bytes of the final MD5 digest uint64_t id = 0; memcpy(&id, context.digest, sizeof(uint64_t)); + if (0 == id) + memcpy(&id, context.digest + 8, sizeof(uint64_t)); return id; } @@ -760,6 +759,7 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { break; } + pInfo->binfo.pRes->info.scanFlag = pBlock->info.scanFlag; // there is an scalar expression that needs to be calculated right before apply the group aggregation. if (pInfo->scalarSup.pExprInfo != NULL) { pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, @@ -849,7 +849,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition if (pPartNode->pExprs != NULL) { int32_t num = 0; SExprInfo* pExprInfo1 = createExprInfo(pPartNode->pExprs, NULL, &num); - int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num); + int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { terrno = code; pTaskInfo->code = terrno; @@ -869,7 +869,12 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition uint32_t defaultBufsz = 0; pInfo->binfo.pRes = createDataBlockFromDescNode(pPartNode->node.pOutputDataBlockDesc); - getBufferPgSize(pInfo->binfo.pRes->info.rowSize, &defaultPgsz, &defaultBufsz); + int32_t code = getBufferPgSize(pInfo->binfo.pRes->info.rowSize, &defaultPgsz, &defaultBufsz); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + pTaskInfo->code = code; + goto _error; + } if (!osTempSpaceAvailable()) { terrno = TSDB_CODE_NO_DISKSPACE; @@ -878,14 +883,15 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - int32_t code = createDiskbasedBuf(&pInfo->pBuf, defaultPgsz, defaultBufsz, pTaskInfo->id.str, tsTempDir); + code = createDiskbasedBuf(&pInfo->pBuf, defaultPgsz, defaultBufsz, pTaskInfo->id.str, tsTempDir); if (code != TSDB_CODE_SUCCESS) { terrno = code; pTaskInfo->code = code; goto _error; } - pInfo->rowCapacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, getBufPageSize(pInfo->pBuf)); + pInfo->rowCapacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, getBufPageSize(pInfo->pBuf), + blockDataGetSerialMetaSize(taosArrayGetSize(pInfo->binfo.pRes->pDataBlock))); pInfo->columnOffset = setupColumnOffset(pInfo->binfo.pRes, pInfo->rowCapacity); code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pInfo->pGroupCols); if (code != TSDB_CODE_SUCCESS) { @@ -950,6 +956,8 @@ static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo static bool hasRemainTbName(SStreamPartitionOperatorInfo* pInfo) { return pInfo->pTbNameIte != NULL; } static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamPartitionOperatorInfo* pInfo = pOperator->info; SSDataBlock* pDest = pInfo->binfo.pRes; ASSERT(hasRemainPartion(pInfo)); @@ -972,9 +980,9 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { pDest->info.parTbName[0] = 0; if (pInfo->tbnameCalSup.numOfExprs > 0) { void* tbname = NULL; - if (streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) { + if (pAPI->stateStore.streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) { memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } } taosArrayDestroy(pParInfo->rowIds); @@ -990,10 +998,10 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { return pDest; } -void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, - SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock) { +void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI) { void* pValue = NULL; - if (streamStateGetParName(pState, groupId, &pValue) != 0) { + if (pAPI->streamStateGetParName(pState, groupId, &pValue) != 0) { SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId); memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); pTmpBlock->info.id.groupId = groupId; @@ -1010,7 +1018,7 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1); len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); memcpy(tbName, varDataVal(pData), len); - streamStatePutParName(pState, groupId, tbName); + pAPI->streamStatePutParName(pState, groupId, tbName); } memcpy(pTmpBlock->info.parTbName, tbName, len); pDestBlock->info.rows--; @@ -1028,16 +1036,18 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* } void* pGpIdCol = taosArrayGet(pDestBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); - colDataAppend(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); + colDataSetVal(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); pDestBlock->info.rows++; blockDataDestroy(pTmpBlock); } else { memcpy(pSrcBlock->info.parTbName, pValue, TSDB_TABLE_NAME_LEN); } - streamStateReleaseBuf(pState, NULL, pValue); + pAPI->streamStateReleaseBuf(pState, NULL, pValue); } static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) { + SExecTaskInfo* pTask = pOperator->pTaskInfo; + SStreamPartitionOperatorInfo* pInfo = pOperator->info; if ((pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) || taosHashGetSize(pInfo->pPartitions) == 0) { @@ -1050,8 +1060,8 @@ static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) { if (pInfo->pTbNameIte != NULL) { SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte; int32_t rowId = *(int32_t*)taosArrayGet(pParInfo->rowIds, 0); - appendCreateTableRow(pOperator->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, - pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes); + appendCreateTableRow(pTask->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, + pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes, &pTask->storageAPI.stateStore); pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte); } return pInfo->pCreateTbRes->info.rows > 0 ? pInfo->pCreateTbRes : NULL; @@ -1164,14 +1174,17 @@ static void destroyStreamPartitionOperatorInfo(void* param) { } void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr) { + SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI; + if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { return; } + SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->partitionSup = *pParSup; pScanInfo->pPartScalarSup = pExpr; if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0); + pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0); } } @@ -1215,6 +1228,11 @@ SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) { return pBlock; } +void freePartItem(void* ptr) { + SPartitionDataInfo* pPart = (SPartitionDataInfo*)ptr; + taosArrayDestroy(pPart->rowIds); +} + SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; @@ -1230,7 +1248,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr if (pPartNode->part.pExprs != NULL) { int32_t num = 0; SExprInfo* pCalExprInfo = createExprInfo(pPartNode->part.pExprs, NULL, &num); - code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num); + code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1245,7 +1263,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } pInfo->tbnameCalSup.pExprInfo = pSubTableExpr; createExprFromOneNode(pSubTableExpr, pPartNode->pSubtable, 0); - code = initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1); + code = initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1259,7 +1277,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } - if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { + if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore) != 0) { code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } @@ -1293,6 +1311,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pPartitions = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + taosHashSetFreeFp(pInfo->pPartitions, freePartItem); pInfo->tsColIndex = 0; pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); @@ -1305,6 +1324,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 754b5f47373e8444c55c9eb64c6ab42694c5142d..f334ae02f6c7694e84f3e85c722316bcdfb0fd61 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -30,11 +30,13 @@ typedef struct SJoinRowCtx { bool rowRemains; int64_t ts; SArray* leftRowLocations; - SArray* rightRowLocations; SArray* leftCreatedBlocks; SArray* rightCreatedBlocks; int32_t leftRowIdx; int32_t rightRowIdx; + + bool rightUseBuildTable; + SArray* rightRowLocations; } SJoinRowCtx; typedef struct SJoinOperatorInfo { @@ -50,7 +52,17 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode* pCondAfterMerge; + SNode* pColEqualOnConditions; + + SArray* leftEqOnCondCols; + char* leftEqOnCondKeyBuf; + int32_t leftEqOnCondKeyLen; + SArray* rightEqOnCondCols; + char* rightEqOnCondKeyBuf; + int32_t rightEqOnCondKeyLen; + + SSHashObj* rightBuildTable; SJoinRowCtx rowCtx; } SJoinOperatorInfo; @@ -92,6 +104,103 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown setJoinColumnInfo(&pInfo->rightCol, rightTsCol); } +static void extractEqualOnCondColsFromOper(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstreams, SOperatorNode* pOperNode, + SColumn* pLeft, SColumn* pRight) { + SColumnNode* pLeftNode = (SColumnNode*)pOperNode->pLeft; + SColumnNode* pRightNode = (SColumnNode*)pOperNode->pRight; + if (pLeftNode->dataBlockId == pRightNode->dataBlockId || pLeftNode->dataBlockId == pDownstreams[0]->resultDataBlockId) { + *pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft); + *pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight); + } else { + *pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight); + *pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft); + } +} + +static void extractEqualOnCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownStream, SNode* pEqualOnCondNode, + SArray* leftTagEqCols, SArray* rightTagEqCols) { + SColumn left = {0}; + SColumn right = {0}; + if (nodeType(pEqualOnCondNode) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pEqualOnCondNode)->condType == LOGIC_COND_TYPE_AND) { + SNode* pNode = NULL; + FOREACH(pNode, ((SLogicConditionNode*)pEqualOnCondNode)->pParameterList) { + SOperatorNode* pOperNode = (SOperatorNode*)pNode; + extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + taosArrayPush(leftTagEqCols, &left); + taosArrayPush(rightTagEqCols, &right); + } + return; + } + + if (nodeType(pEqualOnCondNode) == QUERY_NODE_OPERATOR) { + SOperatorNode* pOperNode = (SOperatorNode*)pEqualOnCondNode; + extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + taosArrayPush(leftTagEqCols, &left); + taosArrayPush(rightTagEqCols, &right); + } +} + +static int32_t initTagColskeyBuf(int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { + int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = (SColumn*)taosArrayGet(pGroupColList, i); + (*keyLen) += pCol->bytes; // actual data + null_flag + } + + int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; + (*keyLen) += nullFlagSize; + + if (*keyLen >= 0) { + + (*keyBuf) = taosMemoryCalloc(1, (*keyLen)); + if ((*keyBuf) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t fillKeyBufFromTagCols(SArray* pCols, SSDataBlock* pBlock, int32_t rowIndex, void* pKey) { + SColumnDataAgg* pColAgg = NULL; + size_t numOfGroupCols = taosArrayGetSize(pCols); + char* isNull = (char*)pKey; + char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols; + + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = (SColumn*) taosArrayGet(pCols, i); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); + + // valid range check. todo: return error code. + if (pCol->slotId > taosArrayGetSize(pBlock->pDataBlock)) { + continue; + } + + if (pBlock->pBlockAgg != NULL) { + pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + } + + if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { + isNull[i] = 1; + } else { + isNull[i] = 0; + char* val = colDataGetData(pColInfoData, rowIndex); + if (pCol->type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(val); + memcpy(pStart, val, dataLen); + pStart += dataLen; + } else if (IS_VAR_DATA_TYPE(pCol->type)) { + varDataCopy(pStart, val); + pStart += varDataTLen(val); + } else { + memcpy(pStart, val, pCol->bytes); + pStart += pCol->bytes; + } + } + } + return (int32_t)(pStart - (char*)pKey); +} + SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) { SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo)); @@ -147,12 +256,22 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t } pInfo->inputOrder = TSDB_ORDER_ASC; - if (pJoinNode->inputTsOrder == ORDER_ASC) { + if (pJoinNode->node.inputTsOrder == ORDER_ASC) { pInfo->inputOrder = TSDB_ORDER_ASC; - } else if (pJoinNode->inputTsOrder == ORDER_DESC) { + } else if (pJoinNode->node.inputTsOrder == ORDER_DESC) { pInfo->inputOrder = TSDB_ORDER_DESC; } + pInfo->pColEqualOnConditions = pJoinNode->pColEqualOnConditions; + if (pInfo->pColEqualOnConditions != NULL) { + pInfo->leftEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->rightEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); + extractEqualOnCondCols(pInfo, pDownstream, pInfo->pColEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols); + initTagColskeyBuf(&pInfo->leftEqOnCondKeyLen, &pInfo->leftEqOnCondKeyBuf, pInfo->leftEqOnCondCols); + initTagColskeyBuf(&pInfo->rightEqOnCondKeyLen, &pInfo->rightEqOnCondKeyBuf, pInfo->rightEqOnCondCols); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->rightBuildTable = tSimpleHashInit(256, hashFn); + } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { @@ -179,10 +298,35 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { pColumn->scale = pColumnNode->node.resType.scale; } +static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { + void* p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } + + tSimpleHashCleanup(pBuildTable); +} + void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; + if (pJoinOperator->pColEqualOnConditions != NULL) { + mergeJoinDestoryBuildTable(pJoinOperator->rightBuildTable); + taosMemoryFreeClear(pJoinOperator->rightEqOnCondKeyBuf); + taosArrayDestroy(pJoinOperator->rightEqOnCondCols); + + taosMemoryFreeClear(pJoinOperator->leftEqOnCondKeyBuf); + taosArrayDestroy(pJoinOperator->leftEqOnCondCols); + } nodesDestroyNode(pJoinOperator->pCondAfterMerge); + taosArrayDestroy(pJoinOperator->rowCtx.leftCreatedBlocks); + taosArrayDestroy(pJoinOperator->rowCtx.rightCreatedBlocks); + taosArrayDestroy(pJoinOperator->rowCtx.leftRowLocations); + taosArrayDestroy(pJoinOperator->rowCtx.rightRowLocations); + pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes); taosMemoryFreeClear(param); } @@ -300,21 +444,122 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator return 0; } +static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations) { + for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { + SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); + int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightEqOnCondCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightEqOnCondKeyBuf); + SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightEqOnCondKeyBuf, keyLen); + if (!ppRows) { + SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); + taosArrayPush(rows, rightRow); + tSimpleHashPut(pInfo->rightBuildTable, pInfo->rightEqOnCondKeyBuf, keyLen, &rows, POINTER_BYTES); + } else { + taosArrayPush(*ppRows, rightRow); + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* pRes, int32_t* nRows, + const SArray* leftRowLocations, int32_t leftRowIdx, + int32_t rightRowIdx, bool useBuildTableTSRange, SArray* rightRowLocations, bool* pReachThreshold) { + *pReachThreshold = false; + uint32_t limitRowNum = pOperator->resultInfo.threshold; + SJoinOperatorInfo* pJoinInfo = pOperator->info; + size_t leftNumJoin = taosArrayGetSize(leftRowLocations); + + int32_t i,j; + + for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { + SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); + SArray* pRightRows = NULL; + if (useBuildTableTSRange) { + int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftEqOnCondCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftEqOnCondKeyBuf); + SArray** ppRightRows = tSimpleHashGet(pJoinInfo->rightBuildTable, pJoinInfo->leftEqOnCondKeyBuf, keyLen); + if (!ppRightRows) { + continue; + } + pRightRows = *ppRightRows; + } else { + pRightRows = rightRowLocations; + } + size_t rightRowsSize = taosArrayGetSize(pRightRows); + for (j = rightRowIdx; j < rightRowsSize; ++j) { + if (*nRows >= limitRowNum) { + *pReachThreshold = true; + break; + } + + SRowLocation* rightRow = taosArrayGet(pRightRows, j); + mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock, + rightRow->pos); + ++*nRows; + } + if (*pReachThreshold) { + break; + } + } + + if (*pReachThreshold) { + pJoinInfo->rowCtx.rowRemains = true; + pJoinInfo->rowCtx.leftRowIdx = i; + pJoinInfo->rowCtx.rightRowIdx = j; + } + return TSDB_CODE_SUCCESS; +} + +static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks, + SArray* rightCreatedBlocks, bool rightUseBuildTable, SArray* rightRowLocations) { + for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { + SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); + blockDataDestroy(pBlock); + } + taosArrayDestroy(rightCreatedBlocks); + for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) { + SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); + blockDataDestroy(pBlock); + } + if (rightRowLocations != NULL) { + taosArrayDestroy(rightRowLocations); + } + if (rightUseBuildTable) { + void* p = NULL; + int32_t iter = 0; + while ((p = tSimpleHashIterate(pJoinInfo->rightBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } + tSimpleHashClear(pJoinInfo->rightBuildTable); + } + + taosArrayDestroy(leftCreatedBlocks); + taosArrayDestroy(leftRowLocations); + + pJoinInfo->rowCtx.rowRemains = false; + pJoinInfo->rowCtx.leftRowLocations = NULL; + pJoinInfo->rowCtx.leftCreatedBlocks = NULL; + pJoinInfo->rowCtx.rightCreatedBlocks = NULL; + pJoinInfo->rowCtx.rightUseBuildTable = false; + pJoinInfo->rowCtx.rightRowLocations = NULL; +} + static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t timestamp, SSDataBlock* pRes, int32_t* nRows) { int32_t code = TSDB_CODE_SUCCESS; SJoinOperatorInfo* pJoinInfo = pOperator->info; SArray* leftRowLocations = NULL; - SArray* leftCreatedBlocks = NULL; SArray* rightRowLocations = NULL; + SArray* leftCreatedBlocks = NULL; SArray* rightCreatedBlocks = NULL; int32_t leftRowIdx = 0; int32_t rightRowIdx = 0; - int32_t i, j; - + SSHashObj* rightTableHash = NULL; + bool rightUseBuildTable = false; + if (pJoinInfo->rowCtx.rowRemains) { leftRowLocations = pJoinInfo->rowCtx.leftRowLocations; leftCreatedBlocks = pJoinInfo->rowCtx.leftCreatedBlocks; + rightUseBuildTable = pJoinInfo->rowCtx.rightUseBuildTable; rightRowLocations = pJoinInfo->rowCtx.rightRowLocations; rightCreatedBlocks = pJoinInfo->rowCtx.rightCreatedBlocks; leftRowIdx = pJoinInfo->rowCtx.leftRowIdx; @@ -330,78 +575,40 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); + if (pJoinInfo->pColEqualOnConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { + mergeJoinFillBuildTable(pJoinInfo, rightRowLocations); + rightUseBuildTable = true; + taosArrayDestroy(rightRowLocations); + rightRowLocations = NULL; + } } size_t leftNumJoin = taosArrayGetSize(leftRowLocations); - size_t rightNumJoin = taosArrayGetSize(rightRowLocations); - uint32_t maxRowNum = *nRows + (leftNumJoin - leftRowIdx - 1) * rightNumJoin + rightNumJoin - rightRowIdx; - uint32_t limitRowNum = maxRowNum; - if (maxRowNum > pOperator->resultInfo.threshold) { - limitRowNum = pOperator->resultInfo.threshold; - if (!pJoinInfo->rowCtx.rowRemains) { - pJoinInfo->rowCtx.rowRemains = true; - pJoinInfo->rowCtx.ts = timestamp; - pJoinInfo->rowCtx.leftRowLocations = leftRowLocations; - pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; - pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; - pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; - } - } - - code = blockDataEnsureCapacity(pRes, limitRowNum); + code = blockDataEnsureCapacity(pRes, pOperator->resultInfo.threshold); if (code != TSDB_CODE_SUCCESS) { - qError("%s can not ensure block capacity for join. left: %zu, right: %zu", GET_TASKID(pOperator->pTaskInfo), - leftNumJoin, rightNumJoin); + qError("%s can not ensure block capacity for join. left: %zu", GET_TASKID(pOperator->pTaskInfo), + leftNumJoin); } - - if (code == TSDB_CODE_SUCCESS) { - bool done = false; - for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { - for (j = rightRowIdx; j < rightNumJoin; ++j) { - if (*nRows >= limitRowNum) { - done = true; - break; - } - - SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); - SRowLocation* rightRow = taosArrayGet(rightRowLocations, j); - mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock, - rightRow->pos); - ++*nRows; - } - if (done) { - break; - } - } + bool reachThreshold = false; - if (maxRowNum > pOperator->resultInfo.threshold) { - pJoinInfo->rowCtx.leftRowIdx = i; - pJoinInfo->rowCtx.rightRowIdx = j; - } + if (code == TSDB_CODE_SUCCESS) { + mergeJoinLeftRowsRightRows(pOperator, pRes, nRows, leftRowLocations, leftRowIdx, + rightRowIdx, rightUseBuildTable, rightRowLocations, &reachThreshold); } - if (maxRowNum <= pOperator->resultInfo.threshold) { - for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { - SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); - blockDataDestroy(pBlock); - } - taosArrayDestroy(rightCreatedBlocks); - taosArrayDestroy(rightRowLocations); - for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) { - SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); - blockDataDestroy(pBlock); - } - taosArrayDestroy(leftCreatedBlocks); - taosArrayDestroy(leftRowLocations); + if (!reachThreshold) { + mergeJoinDestroyTSRangeCtx(pJoinInfo, leftRowLocations, leftCreatedBlocks, rightCreatedBlocks, + rightUseBuildTable, rightRowLocations); - if (pJoinInfo->rowCtx.rowRemains) { - pJoinInfo->rowCtx.rowRemains = false; - pJoinInfo->rowCtx.leftRowLocations = NULL; - pJoinInfo->rowCtx.rightRowLocations = NULL; - pJoinInfo->rowCtx.leftCreatedBlocks = NULL; - pJoinInfo->rowCtx.rightCreatedBlocks = NULL; - } + } else { + pJoinInfo->rowCtx.rowRemains = true; + pJoinInfo->rowCtx.ts = timestamp; + pJoinInfo->rowCtx.leftRowLocations = leftRowLocations; + pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; + pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; + pJoinInfo->rowCtx.rightUseBuildTable = rightUseBuildTable; + pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; } return TSDB_CODE_SUCCESS; } @@ -480,6 +687,7 @@ static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes) // the pDataBlock are always the same one, no need to call this again pRes->info.rows = nrows; pRes->info.dataLoad = 1; + pRes->info.scanFlag = MAIN_SCAN; if (pRes->info.rows >= pOperator->resultInfo.threshold) { break; } diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 729178dc60b482fc82b0fddba5b78ce5358c216d..8ddcc8fd150b7cb6ef17252e62fc1fff9d5fecd2 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -25,7 +25,8 @@ #include "operator.h" #include "query.h" #include "querytask.h" -#include "vnode.h" + +#include "storageapi.h" SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, @@ -37,11 +38,18 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, .closeFn = closeFn, .reqBufFn = reqBufFn, .getExplainFn = explain, + .releaseStreamStateFn = NULL, + .reloadStreamStateFn = NULL, }; return fpSet; } +void setOperatorStreamStateFn(SOperatorInfo* pOperator, __optr_state_fn_t relaseFn, __optr_state_fn_t reloadFn) { + pOperator->fpSet.releaseStreamStateFn = relaseFn; + pOperator->fpSet.reloadStreamStateFn = reloadFn; +} + int32_t optrDummyOpenFn(SOperatorInfo* pOperator) { OPTR_SET_OPENED(pOperator); pOperator->cost.openCost = 0; @@ -233,11 +241,12 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan } static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, const char* pIdStr) { + SStorageAPI* pAPI = pParam->pParam; if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pInfo = pOperator->info; if (pInfo->base.dataReader != NULL) { - tsdbReaderSetCloseFlag(pInfo->base.dataReader); + pAPI->tsdReader.tsdReaderNotifyClosing(pInfo->base.dataReader); } return OPTR_FN_RET_ABORT; } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -246,7 +255,7 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, if (pInfo->pTableScanOp != NULL) { STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; if (pTableScanInfo != NULL && pTableScanInfo->base.dataReader != NULL) { - tsdbReaderSetCloseFlag(pTableScanInfo->base.dataReader); + pAPI->tsdReader.tsdReaderNotifyClosing(pTableScanInfo->base.dataReader); } } @@ -256,8 +265,8 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, return OPTR_FN_RET_CONTINUE; } -int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr) { - STraverParam p = {0}; +int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr, SStorageAPI* pAPI) { + STraverParam p = {.pParam = pAPI}; traverseOperatorTree(pOperator, doStopDataReader, &p, pIdStr); return p.code; } @@ -266,7 +275,6 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR SNode* pTagIndexCond, const char* pUser, const char* dbname) { int32_t type = nodeType(pPhyNode); const char* idstr = GET_TASKID(pTaskInfo); - if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { SOperatorInfo* pOperator = NULL; if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) { @@ -378,17 +386,18 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR STableListInfo* pTableListInfo = tableListCreate(); if (pBlockNode->tableType == TSDB_SUPER_TABLE) { - SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo)); - int32_t code = vnodeGetAllTableList(pHandle->vnode, pBlockNode->uid, pList); + SArray* pList = taosArrayInit(4, sizeof(uint64_t)); + int32_t code = pTaskInfo->storageAPI.metaFn.getChildTableList(pHandle->vnode, pBlockNode->uid, pList); if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = terrno; + pTaskInfo->code = code; + taosArrayDestroy(pList); return NULL; } size_t num = taosArrayGetSize(pList); for (int32_t i = 0; i < num; ++i) { - STableKeyInfo* p = taosArrayGet(pList, i); - tableListAddTableInfo(pTableListInfo, p->uid, 0); + uint64_t* id = taosArrayGet(pList, i); + tableListAddTableInfo(pTableListInfo, *id, 0); } taosArrayDestroy(pList); @@ -482,13 +491,13 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; pOptr = createSessionAggOperatorInfo(ops[0], pSessionNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION == type) { - pOptr = createStreamSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); + pOptr = createStreamSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION == type) { int32_t children = 0; - pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); + pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children, pHandle); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION == type) { int32_t children = pHandle->numOfVgroups; - pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); + pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children, pHandle); } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { pOptr = createPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION == type) { @@ -497,7 +506,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*)pPhyNode; pOptr = createStatewindowOperatorInfo(ops[0], pStateNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE == type) { - pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); + pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN == type) { pOptr = createMergeJoinOperatorInfo(ops, size, (SSortMergeJoinPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) { diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 02f504bef03aaf4cff5d1193a2c23fd66b0b6146..1cc377b3ee9320c79e30c34a45d0289936da5918 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -18,6 +18,7 @@ #include "functionMgt.h" #include "operator.h" #include "querytask.h" +#include "tdatablock.h" typedef struct SProjectOperatorInfo { SOptrBasicInfo binfo; @@ -37,7 +38,7 @@ typedef struct SIndefOperatorInfo { SSDataBlock* pNextGroupRes; } SIndefOperatorInfo; -static SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator); +static int32_t doGenerateSourceData(SOperatorInfo* pOperator); static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator); static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator); static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols); @@ -72,6 +73,20 @@ static void destroyIndefinitOperatorInfo(void* param) { taosMemoryFreeClear(param); } +void streamOperatorReleaseState(SOperatorInfo* pOperator) { + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.releaseStreamStateFn) { + downstream->fpSet.releaseStreamStateFn(downstream); + } +} + +void streamOperatorReloadState(SOperatorInfo* pOperator) { + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.reloadStreamStateFn) { + downstream->fpSet.reloadStreamStateFn(downstream); + } +} + SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; @@ -92,6 +107,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys pInfo->binfo.pRes = pResBlock; pInfo->pFinalRes = createOneDataBlock(pResBlock, false); + pInfo->binfo.inputTsOrder = pProjPhyNode->node.inputTsOrder; + pInfo->binfo.outputTsOrder = pProjPhyNode->node.outputTsOrder; if (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) { pInfo->mergeDataBlocks = false; @@ -114,7 +131,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys initResultSizeInfo(&pOperator->resultInfo, numOfRows); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -133,6 +150,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doProjectOperation, NULL, destroyProjectOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamOperatorReleaseState, streamOperatorReloadState); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -197,7 +215,7 @@ static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, S if (newGroup) { resetLimitInfoForNextGroup(pLimitInfo); } - + return PROJECT_RETRIEVE_CONTINUE; } @@ -212,6 +230,8 @@ static int32_t doIngroupLimitOffset(SLimitInfo* pLimitInfo, uint64_t groupId, SS } else { if (limitReached && (pLimitInfo->slimit.limit >= 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { setOperatorCompleted(pOperator); + } else if (limitReached && groupId == 0) { + setOperatorCompleted(pOperator); } } @@ -235,8 +255,9 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } int64_t st = 0; - int32_t order = 0; + int32_t order = pInfo->inputTsOrder; int32_t scanFlag = 0; + int32_t code = TSDB_CODE_SUCCESS; if (pOperator->cost.openCost == 0) { st = taosGetTimestampUs(); @@ -246,7 +267,12 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { SLimitInfo* pLimitInfo = &pProjectInfo->limitInfo; if (downstream == NULL) { - return doGenerateSourceData(pOperator); + code = doGenerateSourceData(pOperator); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, code); + } + + return (pRes->info.rows > 0) ? pRes : NULL; } while (1) { @@ -281,10 +307,10 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { break; } - // the pDataBlock are always the same one, no need to call this again - int32_t code = getTableScanInfo(downstream, &order, &scanFlag, false); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); + if (pProjectInfo->mergeDataBlocks) { + pFinalRes->info.scanFlag = scanFlag = pBlock->info.scanFlag; + } else { + pRes->info.scanFlag = scanFlag = pBlock->info.scanFlag; } setInputDataBlock(pSup, pBlock, order, scanFlag, false); @@ -369,7 +395,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy if (pPhyNode->pExprs != NULL) { int32_t num = 0; SExprInfo* pSExpr = createExprInfo(pPhyNode->pExprs, NULL, &num); - int32_t code = initExprSupp(&pInfo->scalarSup, pSExpr, num); + int32_t code = initExprSupp(&pInfo->scalarSup, pSExpr, num, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -391,7 +417,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy blockDataEnsureCapacity(pResBlock, numOfRows); int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -403,6 +429,8 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy } pInfo->binfo.pRes = pResBlock; + pInfo->binfo.inputTsOrder = pNode->inputTsOrder; + pInfo->binfo.outputTsOrder = pNode->outputTsOrder; pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr); setOperatorInfo(pOperator, "IndefinitOperator", QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, false, OP_NOT_OPENED, pInfo, @@ -426,18 +454,13 @@ _error: static void doHandleDataBlock(SOperatorInfo* pOperator, SSDataBlock* pBlock, SOperatorInfo* downstream, SExecTaskInfo* pTaskInfo) { - int32_t order = 0; - int32_t scanFlag = 0; - SIndefOperatorInfo* pIndefInfo = pOperator->info; SOptrBasicInfo* pInfo = &pIndefInfo->binfo; SExprSupp* pSup = &pOperator->exprSupp; - // the pDataBlock are always the same one, no need to call this again - int32_t code = getTableScanInfo(downstream, &order, &scanFlag, false); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); - } + int32_t order = pInfo->inputTsOrder; + int32_t scanFlag = pBlock->info.scanFlag; + int32_t code = TSDB_CODE_SUCCESS; // there is an scalar expression that needs to be calculated before apply the group aggregation. SExprSupp* pScalarSup = &pIndefInfo->scalarSup; @@ -503,6 +526,7 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { setOperatorCompleted(pOperator); break; } + pInfo->pRes->info.scanFlag = pBlock->info.scanFlag; if (pIndefInfo->groupId == 0 && pBlock->info.id.groupId != 0) { pIndefInfo->groupId = pBlock->info.id.groupId; // this is the initial group result @@ -597,7 +621,7 @@ SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) { return pList; } -SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { +int32_t doGenerateSourceData(SOperatorInfo* pOperator) { SProjectOperatorInfo* pProjectInfo = pOperator->info; SExprSupp* pSup = &pOperator->exprSupp; @@ -611,14 +635,45 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { for (int32_t k = 0; k < pSup->numOfExprs; ++k) { int32_t outputSlotId = pExpr[k].base.resSchema.slotId; - ASSERT(pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE); - SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, outputSlotId); + if (pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE) { + SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, outputSlotId); + + int32_t type = pExpr[k].base.pParam[0].param.nType; + if (TSDB_DATA_TYPE_NULL == type) { + colDataSetNNULL(pColInfoData, 0, 1); + } else { + colDataSetVal(pColInfoData, 0, taosVariantGet(&pExpr[k].base.pParam[0].param, type), false); + } + } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_FUNCTION) { + SqlFunctionCtx* pfCtx = &pSup->pCtx[k]; + + // UDF scalar functions will be calculated here, for example, select foo(n) from (select 1 n). + // UDF aggregate functions will be handled in agg operator. + if (fmIsScalarFunc(pfCtx->functionId)) { + SArray* pBlockList = taosArrayInit(4, POINTER_BYTES); + taosArrayPush(pBlockList, &pRes); + + SColumnInfoData* pResColData = taosArrayGet(pRes->pDataBlock, outputSlotId); + SColumnInfoData idata = {.info = pResColData->info, .hasNull = true}; + + SScalarParam dest = {.columnData = &idata}; + int32_t code = scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pBlockList); + return code; + } + + int32_t startOffset = pRes->info.rows; + ASSERT(pRes->info.capacity > 0); + colDataAssign(pResColData, &idata, dest.numOfRows, &pRes->info); + colDataDestroy(&idata); - int32_t type = pExpr[k].base.pParam[0].param.nType; - if (TSDB_DATA_TYPE_NULL == type) { - colDataSetNNULL(pColInfoData, 0, 1); + taosArrayDestroy(pBlockList); + } else { + return TSDB_CODE_OPS_NOT_SUPPORT; + } } else { - colDataSetVal(pColInfoData, 0, taosVariantGet(&pExpr[k].base.pParam[0].param, type), false); + return TSDB_CODE_OPS_NOT_SUPPORT; } } @@ -634,7 +689,7 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; } - return (pRes->info.rows > 0) ? pRes : NULL; + return TSDB_CODE_SUCCESS; } static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, SArray* pPseudoList) { diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index 7716b5976b2fd37fd3486abc2fd9fc125e5a87c7..22d171e74a5117f13ee648701211d2fff2d9da26 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -29,13 +29,13 @@ #include "operator.h" #include "query.h" #include "querytask.h" +#include "storageapi.h" #include "thash.h" #include "ttypes.h" -#include "vnode.h" #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) -SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model) { +SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI) { SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); if (pTaskInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -48,6 +48,7 @@ SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOP pTaskInfo->execModel = model; pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo)); pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES); + pTaskInfo->storageAPI = *pAPI; taosInitRWLatch(&pTaskInfo->lock); @@ -55,7 +56,6 @@ SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOP pTaskInfo->id.queryId = queryId; pTaskInfo->id.str = taosMemoryMalloc(64); buildTaskId(taskId, queryId, pTaskInfo->id.str); - return pTaskInfo; } @@ -63,7 +63,7 @@ bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code); } void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; - stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str); + stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str, &pTaskInfo->storageAPI); } void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { @@ -78,7 +78,7 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, int32_t vgId, char* sql, EOPTR_EXEC_MODEL model) { - *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model); + *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model, &pHandle->api); if (*pTaskInfo == NULL) { taosMemoryFree(sql); return terrno; @@ -120,13 +120,15 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo return terrno; } - metaReaderInit(&mr, pHandle->meta, 0); - int32_t code = metaGetTableEntryByUidCache(&mr, pScanNode->uid); + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + + pAPI->metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pAPI->metaFn); + int32_t code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, pScanNode->uid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid, GET_TASKID(pTaskInfo)); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return terrno; } @@ -142,9 +144,9 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo tDecoderClear(&mr.coder); tb_uid_t suid = mr.me.ctbEntry.suid; - code = metaGetTableEntryByUidCache(&mr, suid); + code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, suid); if (code != TSDB_CODE_SUCCESS) { - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); return terrno; } @@ -154,7 +156,7 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.ntbEntry.schemaRow); } - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); pSchemaInfo->qsw = extractQueriedColumnSchema(pScanNode); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8909d83d31284e406010021d77e23f6580f79ccb..71b0747be8803210fdee885ba7c9434639e204d4 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -33,12 +33,16 @@ #include "operator.h" #include "querytask.h" +#include "storageapi.h" +#include "wal.h" + int32_t scanDebug = 0; #define MULTI_READER_MAX_TABLE_NUM 5000 #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) #define STREAM_SCAN_OP_NAME "StreamScanOperator" +#define STREAM_SCAN_OP_STATE_NAME "StreamScanFillHistoryState" typedef struct STableMergeScanExecInfo { SFileBlockLoadRecorder blockRecorder; @@ -49,9 +53,7 @@ typedef struct STableMergeScanSortSourceParam { SOperatorInfo* pOperator; int32_t readerIdx; uint64_t uid; - SSDataBlock* inputBlock; - bool multiReader; - STsdbReader* dataReader; + STsdbReader* reader; } STableMergeScanSortSourceParam; typedef struct STableCountScanOperatorInfo { @@ -85,39 +87,6 @@ static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } } -static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm, NULL); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); - - tw->ekey -= 1; -} - static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo, int32_t order) { STimeWindow w = {0}; @@ -127,7 +96,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } if (order == TSDB_ORDER_ASC) { - w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey); + w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.skey); ASSERT(w.ekey >= pBlockInfo->window.skey); if (w.ekey < pBlockInfo->window.ekey) { @@ -146,7 +115,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } } } else { - w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.ekey); + w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.ekey); ASSERT(w.skey <= pBlockInfo->window.ekey); if (w.skey > pBlockInfo->window.skey) { @@ -195,8 +164,23 @@ static SResultRow* getTableGroupOutputBuf(SOperatorInfo* pOperator, uint64_t gro return (SResultRow*)((char*)(*pPage) + p1->offset); } +static int32_t insertTableToScanIgnoreList(STableScanInfo* pTableScanInfo, uint64_t uid) { + if (NULL == pTableScanInfo->pIgnoreTables) { + int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList); + pTableScanInfo->pIgnoreTables = taosHashInit(tableNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (NULL == pTableScanInfo->pIgnoreTables) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + taosHashPut(pTableScanInfo->pIgnoreTables, &uid, sizeof(uid), &pTableScanInfo->scanTimes, sizeof(pTableScanInfo->scanTimes)); + + return TSDB_CODE_SUCCESS; +} + static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* pBlockInfo, uint32_t* status) { STableScanInfo* pTableScanInfo = pOperator->info; + int32_t code = TSDB_CODE_SUCCESS; if (pTableScanInfo->base.pdInfo.pExprSup == NULL) { return TSDB_CODE_SUCCESS; @@ -229,9 +213,10 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* if (notLoadBlock) { *status = FUNC_DATA_REQUIRED_NOT_LOAD; + code = insertTableToScanIgnoreList(pTableScanInfo, pBlockInfo->id.uid); } - return TSDB_CODE_SUCCESS; + return code; } static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsAgg, int32_t numOfCols, @@ -245,13 +230,16 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA } static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + bool allColumnsHaveAgg = true; - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); + bool hasNullSMA = false; + int32_t code = pAPI->tsdReader.tsdReaderRetrieveBlockSMAInfo(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg, &hasNullSMA); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } - if (!allColumnsHaveAgg) { + if (!allColumnsHaveAgg || hasNullSMA) { return false; } return true; @@ -307,6 +295,8 @@ bool applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder; pCost->totalBlocks += 1; @@ -327,7 +317,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; pCost->totalRows += pBlock->info.rows; - tsdbReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, @@ -335,7 +325,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->id.uid); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); pCost->skipBlocks += 1; - tsdbReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) { pCost->loadBlockStatis += 1; @@ -345,7 +335,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); - tsdbReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -367,7 +357,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->filterOutBlocks += 1; (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - tsdbReleaseDataBlock(pTableScanInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } } @@ -382,15 +372,21 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; - tsdbReleaseDataBlock(pTableScanInfo->dataReader); - *status = FUNC_DATA_REQUIRED_FILTEROUT; + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader); + + STableScanInfo* p1 = pOperator->info; + if (taosHashGetSize(p1->pIgnoreTables) == taosArrayGetSize(p1->base.pTableListInfo->pTableList)) { + *status = FUNC_DATA_REQUIRED_ALL_FILTEROUT; + } else { + *status = FUNC_DATA_REQUIRED_FILTEROUT; + } return TSDB_CODE_SUCCESS; } pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; - SSDataBlock* p = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL); + SSDataBlock* p = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); if (p == NULL) { return terrno; } @@ -402,9 +398,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->totalRows -= pBlock->info.rows; if (pOperator->exprSupp.pFilterInfo != NULL) { - int64_t st = taosGetTimestampUs(); - doFilter(pBlock, pOperator->exprSupp.pFilterInfo, &pTableScanInfo->matchInfo); + int32_t code = doFilter(pBlock, pOperator->exprSupp.pFilterInfo, &pTableScanInfo->matchInfo); + if (code != TSDB_CODE_SUCCESS) return code; + int64_t st = taosGetTimestampUs(); double el = (taosGetTimestampUs() - st) / 1000.0; pTableScanInfo->readRecorder.filterTime += el; @@ -467,7 +464,12 @@ static STableCachedVal* createTableCacheVal(const SMetaReader* pMetaReader) { } // const void *key, size_t keyLen, void *value -static void freeCachedMetaItem(const void* key, size_t keyLen, void* value) { freeTableCachedVal(value); } +static void freeCachedMetaItem(const void* key, size_t keyLen, void* value, void* ud) { + (void)key; + (void)keyLen; + (void)ud; + freeTableCachedVal(value); +} static void doSetNullValue(SSDataBlock* pBlock, const SExprInfo* pExpr, int32_t numOfExpr) { for (int32_t j = 0; j < numOfExpr; ++j) { @@ -486,12 +488,12 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int } int32_t code = 0; + bool freeReader = false; // backup the rows int32_t backupRows = pBlock->info.rows; pBlock->info.rows = rows; - bool freeReader = false; STableCachedVal val = {0}; SMetaReader mr = {0}; @@ -502,8 +504,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // 1. check if it is existed in meta cache if (pCache == NULL) { - metaReaderInit(&mr, pHandle->meta, 0); - code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, META_READER_NOLOCK, &pHandle->api.metaFn); + code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { // when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed. if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { @@ -516,11 +518,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), idStr); } - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return terrno; } - metaReaderReleaseLock(&mr); + pHandle->api.metaReaderFn.readerReleaseLock(&mr); val.pName = mr.me.name; val.pTags = (STag*)mr.me.ctbEntry.pTags; @@ -531,8 +533,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid)); if (h == NULL) { - metaReaderInit(&mr, pHandle->meta, 0); - code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pHandle->api.metaFn); + code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", @@ -543,11 +545,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), idStr); } - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return terrno; } - metaReaderReleaseLock(&mr); + pHandle->api.metaReaderFn.readerReleaseLock(&mr); STableCachedVal* pVal = createTableCacheVal(&mr); @@ -555,7 +557,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int freeReader = true; int32_t ret = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(uint64_t), pVal, - sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW); + sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW, NULL); if (ret != TAOS_LRU_STATUS_OK) { qError("failed to put meta into lru cache, code:%d, %s", ret, idStr); freeTableCachedVal(pVal); @@ -587,7 +589,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int } else { // these are tags STagVal tagVal = {0}; tagVal.cid = pExpr1->base.pParam[0].pCol->colId; - const char* p = metaGetTableTagVal(val.pTags, pColInfoData->info.type, &tagVal); + const char* p = pHandle->api.metaFn.extractTagVal(val.pTags, pColInfoData->info.type, &tagVal); char* data = NULL; if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { @@ -606,7 +608,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int } if (code) { if (freeReader) { - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); } return code; } @@ -621,7 +623,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int // restore the rows pBlock->info.rows = backupRows; if (freeReader) { - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); } return TSDB_CODE_SUCCESS; @@ -655,6 +657,8 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSDataBlock* pBlock = pTableScanInfo->pResBlock; bool hasNext = false; int32_t code = TSDB_CODE_SUCCESS; @@ -662,9 +666,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { int64_t st = taosGetTimestampUs(); while (true) { - code = tsdbNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); + code = pAPI->tsdReader.tsdNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); if (code) { - tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, code); } @@ -673,12 +677,12 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { } if (isTaskKilled(pTaskInfo)) { - tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } if (pOperator->status == OP_EXEC_DONE) { - tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader); break; } @@ -698,6 +702,10 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } + if (status == FUNC_DATA_REQUIRED_ALL_FILTEROUT) { + break; + } + // current block is filter out according to filter condition, continue load the next block if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) { continue; @@ -707,14 +715,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; pOperator->cost.totalCost = pTableScanInfo->base.readRecorder.elapsedTime; - - // todo refactor - /*pTableScanInfo->lastStatus.uid = pBlock->info.id.uid;*/ - /*pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;*/ - // pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; - // pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; - // pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; - + pBlock->info.scanFlag = pTableScanInfo->base.scanFlag; return pBlock; } return NULL; @@ -723,6 +724,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { @@ -737,6 +739,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { } pTableScanInfo->scanTimes += 1; + taosHashClear(pTableScanInfo->pIgnoreTables); if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); @@ -745,7 +748,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo)); // do prepare for the next round table scan operation - tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } @@ -753,7 +756,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->base.cond.order == TSDB_ORDER_ASC) { prepareForDescendingScan(&pTableScanInfo->base, pOperator->exprSupp.pCtx, 0); - tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); } @@ -764,13 +767,14 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { } pTableScanInfo->scanTimes += 1; + taosHashClear(pTableScanInfo->pIgnoreTables); if (pTableScanInfo->scanTimes < total) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->base.scanFlag = MAIN_SCAN; qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo)); - tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); + pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); } } } @@ -781,6 +785,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; // scan table one by one sequentially if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) { @@ -808,11 +813,11 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); taosRUnLockLatch(&pTaskInfo->lock); - tsdbSetTableList(pInfo->base.dataReader, &tInfo, 1); + pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, &tInfo, 1); qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables, pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); - tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); + pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; } } else { // scan table group by group sequentially @@ -827,8 +832,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); - int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, - (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly); + int32_t code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, + (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -843,30 +848,29 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return result; } - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { - setOperatorCompleted(pOperator); - return NULL; - } + while (1) { + if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { + setOperatorCompleted(pOperator); + return NULL; + } - // reset value for the next group data output - pOperator->status = OP_OPENED; - resetLimitInfoForNextGroup(&pInfo->base.limitInfo); + // reset value for the next group data output + pOperator->status = OP_OPENED; + resetLimitInfoForNextGroup(&pInfo->base.limitInfo); - int32_t num = 0; - STableKeyInfo* pList = NULL; - tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); + int32_t num = 0; + STableKeyInfo* pList = NULL; + tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); - tsdbSetTableList(pInfo->base.dataReader, pList, num); - tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); - pInfo->scanTimes = 0; + pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, pList, num); + pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); + pInfo->scanTimes = 0; - result = doGroupedTableScan(pOperator); - if (result != NULL) { - return result; + result = doGroupedTableScan(pOperator); + if (result != NULL) { + return result; + } } - - setOperatorCompleted(pOperator); - return NULL; } } @@ -879,10 +883,10 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr return 0; } -static void destroyTableScanBase(STableScanBase* pBase) { +static void destroyTableScanBase(STableScanBase* pBase, TsdReader* pAPI) { cleanupQueryTableDataCond(&pBase->cond); - tsdbReaderClose(pBase->dataReader); + pAPI->tsdReaderClose(pBase->dataReader); pBase->dataReader = NULL; if (pBase->matchInfo.pList != NULL) { @@ -897,7 +901,8 @@ static void destroyTableScanBase(STableScanBase* pBase) { static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); - destroyTableScanBase(&pTableScanInfo->base); + taosHashCleanup(pTableScanInfo->pIgnoreTables); + destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); taosMemoryFreeClear(param); } @@ -930,7 +935,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, if (pScanNode->pScanPseudoCols != NULL) { SExprSupp* pSup = &pInfo->base.pseudoSup; pSup->pExprInfo = createExprInfo(pScanNode->pScanPseudoCols, NULL, &pSup->numOfExprs); - pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset); + pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; @@ -943,6 +948,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); + pInfo->base.readerAPI = pTaskInfo->storageAPI.tsdReader; initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); // blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); @@ -1037,13 +1043,13 @@ static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t grou pInfo->groupId = groupCol[rowIndex]; } -void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint64_t version) { +void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint64_t ver) { pTableScanInfo->base.cond.twindows = *pWin; - pTableScanInfo->base.cond.endVersion = version; + pTableScanInfo->base.cond.startVersion = 0; + pTableScanInfo->base.cond.endVersion = ver; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; - tsdbReaderClose(pTableScanInfo->base.dataReader); - qDebug("1"); + pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; } @@ -1059,11 +1065,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU cond.twindows = (STimeWindow){.skey = startTs, .ekey = endTs}; SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SSDataBlock* pBlock = pTableScanInfo->pResBlock; STsdbReader* pReader = NULL; - int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, - (STsdbReader**)&pReader, GET_TASKID(pTaskInfo), false); + int32_t code = pAPI->tsdReader.tsdReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, + (void**)&pReader, GET_TASKID(pTaskInfo), false, NULL); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -1071,7 +1078,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } bool hasNext = false; - code = tsdbNextDataBlock(pReader, &hasNext); + code = pAPI->tsdReader.tsdNextDataBlock(pReader, &hasNext); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -1079,12 +1086,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } if (hasNext) { - /*SSDataBlock* p = */ tsdbRetrieveDataBlock(pReader, NULL); + /*SSDataBlock* p = */ pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, NULL); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); } - tsdbReaderClose(pReader); + pAPI->tsdReader.tsdReaderClose(pReader); qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 ", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid); @@ -1149,15 +1156,19 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ win.ekey = TMAX(win.ekey, endData[*pRowIndex]); continue; } + if (win.skey == endData[*pRowIndex] && groupId == gpData[*pRowIndex]) { win.skey = TMIN(win.skey, startData[*pRowIndex]); continue; } + ASSERT(!(win.skey > startData[*pRowIndex] && win.ekey < endData[*pRowIndex]) || !(isInTimeWindow(&win, startData[*pRowIndex], 0) || isInTimeWindow(&win, endData[*pRowIndex], 0))); break; } + STableScanInfo* pTScanInfo = pInfo->pTableScanOp->info; + qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey, pInfo->pUpdateInfo->maxDataVersion); resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); pInfo->pTableScanOp->status = OP_OPENED; return true; @@ -1216,8 +1227,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 *pRowIndex = 0; pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTableScanInfo->base.dataReader); - qDebug("2"); + pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; return NULL; } @@ -1262,8 +1272,8 @@ static int32_t getPreSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, pKey->win.ekey = endTs; pKey->groupId = groupId; - SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, pKey); - int32_t code = streamStateSessionGetKVByCur(pCur, pKey, NULL, 0); + void* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, pKey); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, pKey, NULL, 0); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pKey); } @@ -1295,9 +1305,9 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - int64_t version = pSrcBlock->info.version - 1; + int64_t ver = pSrcBlock->info.version - 1; for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { - uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version); + uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], ver); // gap must be 0. SSessionKey startWin = {0}; getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], startData[i], groupId, &startWin); @@ -1343,13 +1353,13 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; - int64_t version = pSrcBlock->info.version - 1; + int64_t ver = pSrcBlock->info.version - 1; if (pInfo->partitionSup.needCalc && srcStartTsCol[0] != srcEndTsCol[0]) { uint64_t srcUid = srcUidData[0]; TSKEY startTs = srcStartTsCol[0]; TSKEY endTs = srcEndTsCol[0]; - SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, startTs, endTs, version); + SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, startTs, endTs, ver); printDataBlock(pPreRes, "pre res"); blockDataCleanup(pSrcBlock); int32_t code = blockDataEnsureCapacity(pSrcBlock, pPreRes->info.rows); @@ -1387,7 +1397,7 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS uint64_t srcUid = srcUidData[i]; uint64_t groupId = srcGp[i]; if (groupId == 0) { - groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); + groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], ver); } TSKEY calStartTs = srcStartTsCol[i]; colDataSetVal(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false); @@ -1424,21 +1434,21 @@ static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pS ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; - int64_t version = pSrcBlock->info.version - 1; + int64_t ver = pSrcBlock->info.version - 1; for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { uint64_t srcUid = srcUidData[i]; uint64_t groupId = srcGp[i]; char* tbname[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN] = {0}; if (groupId == 0) { - groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); + groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], ver); } if (pInfo->tbnameCalSup.pExprInfo) { void* parTbname = NULL; - streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname); + pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname); memcpy(varDataVal(tbname), parTbname, TSDB_TABLE_NAME_LEN); varDataSetLen(tbname, strlen(varDataVal(tbname))); - streamFreeVal(parTbname); + pInfo->stateStore.streamStateFreeVal(parTbname); } appendOneRowToStreamSpecialBlock(pDestBlock, srcStartTsCol + i, srcEndTsCol + i, srcUidData + i, &groupId, tbname[0] == 0 ? NULL : tbname); @@ -1469,7 +1479,7 @@ static void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { pBlock->info.parTbName[0] = 0; } else { appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, - pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes); + pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes, &pInfo->stateStore); } } @@ -1500,7 +1510,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* tsCol = (TSKEY*)pColDataInfo->pData; - bool tableInserted = updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.id.uid); + bool tableInserted = pInfo->stateStore.updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.id.uid); for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; @@ -1516,10 +1526,9 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock isClosed = isCloseWindow(&win, &pInfo->twAggSup); } // must check update info first. - bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.id.uid, tsCol[rowId]); + bool update = pInfo->stateStore.updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.id.uid, tsCol[rowId]); bool closedWin = isClosed && isSignleIntervalWindow(pInfo) && - isDeletedStreamWindow(&win, pBlock->info.id.groupId, - pInfo->pState, &pInfo->twAggSup); + isDeletedStreamWindow(&win, pBlock->info.id.groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->stateStore); if ((update || closedWin) && out) { qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin); uint64_t gpId = 0; @@ -1540,20 +1549,109 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock } } -static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock, bool filter) { +static void doBlockDataWindowFilter(SSDataBlock* pBlock, int32_t tsIndex, STimeWindow* pWindow, const char* id) { + if (pWindow->skey != INT64_MIN || pWindow->ekey != INT64_MAX) { + bool* p = taosMemoryCalloc(pBlock->info.rows, sizeof(bool)); + bool hasUnqualified = false; + + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, tsIndex); + + if (pWindow->skey != INT64_MIN) { + qDebug("%s filter for additional history window, skey:%" PRId64, id, pWindow->skey); + + ASSERT(pCol->pData != NULL); + for (int32_t i = 0; i < pBlock->info.rows; ++i) { + int64_t* ts = (int64_t*)colDataGetData(pCol, i); + p[i] = (*ts >= pWindow->skey); + + if (!p[i]) { + hasUnqualified = true; + } + } + } else if (pWindow->ekey != INT64_MAX) { + qDebug("%s filter for additional history window, ekey:%" PRId64, id, pWindow->ekey); + for (int32_t i = 0; i < pBlock->info.rows; ++i) { + int64_t* ts = (int64_t*)colDataGetData(pCol, i); + p[i] = (*ts <= pWindow->ekey); + + if (!p[i]) { + hasUnqualified = true; + } + } + } + + if (hasUnqualified) { + trimDataBlock(pBlock, pBlock->info.rows, p); + } + + taosMemoryFree(p); + } +} + +// re-build the delete block, ONLY according to the split timestamp +static void rebuildDeleteBlockData(SSDataBlock* pBlock, STimeWindow* pWindow, const char* id) { + int32_t numOfRows = pBlock->info.rows; + bool* p = taosMemoryCalloc(numOfRows, sizeof(bool)); + bool hasUnqualified = false; + int64_t skey = pWindow->skey; + int64_t ekey = pWindow->ekey; + + SColumnInfoData* pSrcStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + uint64_t* tsStartCol = (uint64_t*)pSrcStartCol->pData; + SColumnInfoData* pSrcEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + uint64_t* tsEndCol = (uint64_t*)pSrcEndCol->pData; + + if (pWindow->skey != INT64_MIN) { + for (int32_t i = 0; i < numOfRows; i++) { + if (tsStartCol[i] < skey) { + tsStartCol[i] = skey; + } + + if (tsEndCol[i] >= skey) { + p[i] = true; + } else { // this row should be removed, since it is not in this query time window, which is [skey, INT64_MAX] + hasUnqualified = true; + } + } + } else if (pWindow->ekey != INT64_MAX) { + for(int32_t i = 0; i < numOfRows; ++i) { + if (tsEndCol[i] > ekey) { + tsEndCol[i] = ekey; + } + + if (tsStartCol[i] <= ekey) { + p[i] = true; + } else { + hasUnqualified = true; + } + } + } + + if (hasUnqualified) { + trimDataBlock(pBlock, pBlock->info.rows, p); + qDebug("%s re-build delete datablock, start key revised to:%"PRId64", rows:%"PRId64, id, skey, pBlock->info.rows); + } else { + qDebug("%s not update the delete block", id); + } + + taosMemoryFree(p); +} + +static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock, STimeWindow* pTimeWindow, bool filter) { SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; SOperatorInfo* pOperator = pInfo->pStreamScanOp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + const char* id = GET_TASKID(pTaskInfo); blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows); - pInfo->pRes->info.rows = pBlock->info.rows; - pInfo->pRes->info.id.uid = pBlock->info.id.uid; - pInfo->pRes->info.type = STREAM_NORMAL; - pInfo->pRes->info.version = pBlock->info.version; + pBlockInfo->rows = pBlock->info.rows; + pBlockInfo->id.uid = pBlock->info.id.uid; + pBlockInfo->type = STREAM_NORMAL; + pBlockInfo->version = pBlock->info.version; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - pInfo->pRes->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); + pBlockInfo->id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); // todo extract method for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) { @@ -1583,7 +1681,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, - pInfo->pRes->info.rows, GET_TASKID(pTaskInfo), &pTableScanInfo->base.metaCache); + pBlockInfo->rows, id, &pTableScanInfo->base.metaCache); // ignore the table not exists error, since this table may have been dropped during the scan procedure. if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_PAR_TABLE_NOT_EXIST) { blockDataFreeRes((SSDataBlock*)pBlock); @@ -1598,91 +1696,76 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); } + // filter the block extracted from WAL files, according to the time window apply additional time window filter + doBlockDataWindowFilter(pInfo->pRes, pInfo->primaryTsIndex, pTimeWindow, id); pInfo->pRes->info.dataLoad = 1; + blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); -// blockDataFreeRes((SSDataBlock*)pBlock); + if (pInfo->pRes->info.rows == 0) { + return 0; + } calBlockTbName(pInfo, pInfo->pRes); return 0; } static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SStreamScanInfo* pInfo = pOperator->info; const char* id = GET_TASKID(pTaskInfo); qDebug("start to exec queue scan, %s", id); -#if 0 - if (pTaskInfo->streamInfo.submit.msgStr != NULL) { - if (pInfo->tqReader->msg.msgStr == NULL) { - SPackedData submit = pTaskInfo->streamInfo.submit; - if (tqReaderSetSubmitMsg(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { - qError("submit msg messed up when initing stream submit block %p", submit.msgStr); - return NULL; - } - } - - blockDataCleanup(pInfo->pRes); - SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - - while (tqNextBlockImpl(pInfo->tqReader, NULL)) { - int32_t code = tqRetrieveDataBlock(pInfo->tqReader, NULL); - if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) { - continue; - } - - setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, true); - - if (pBlockInfo->rows > 0) { - return pInfo->pRes; - } - } - - pInfo->tqReader->msg = (SPackedData){0}; - pTaskInfo->streamInfo.submit = (SPackedData){0}; + if (isTaskKilled(pTaskInfo)) { return NULL; } -#endif if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, - pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, - pInfo->tqReader->pWalReader->curVersion); +// qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, +// pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, +// pInfo->tqReader->pWalReader->curVersion); tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pResult->info.id.uid, pResult->info.window.ekey); return pResult; } + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; - qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); - if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { + int64_t validVer = pTaskInfo->streamInfo.snapshotVer + 1; + qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", validVer); + if (pAPI->tqReaderFn.tqReaderSeek(pInfo->tqReader, validVer, pTaskInfo->id.str) < 0) { return NULL; } - tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pTaskInfo->streamInfo.snapshotVer); + tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, validVer); } if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) { while (1) { - int32_t type = tqNextBlockInWal(pInfo->tqReader); - SSDataBlock* pRes = pInfo->tqReader->pResBlock; + bool hasResult = pAPI->tqReaderFn.tqReaderNextBlockInWal(pInfo->tqReader, id); - // curVersion move to next, so currentOffset = curVersion - 1 - tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion - 1); + SSDataBlock* pRes = pAPI->tqReaderFn.tqGetResultBlock(pInfo->tqReader); + struct SWalReader* pWalReader = pAPI->tqReaderFn.tqReaderGetWalReader(pInfo->tqReader); - if (type == FETCH_TYPE__DATA) { + // curVersion move to next + tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pWalReader->curVersion); + + if (hasResult) { qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, pRes->info.rows, pTaskInfo->streamInfo.currentOffset.version); blockDataCleanup(pInfo->pRes); - setBlockIntoRes(pInfo, pRes, true); + STimeWindow defaultWindow = {.skey = INT64_MIN, .ekey = INT64_MAX}; + setBlockIntoRes(pInfo, pRes, &defaultWindow, true); if (pInfo->pRes->info.rows > 0) { return pInfo->pRes; } - } else if (type == FETCH_TYPE__NONE) { + } else { qDebug("doQueueScan get none from log, return, version:%" PRId64, pTaskInfo->streamInfo.currentOffset.version); return NULL; } @@ -1708,9 +1791,10 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S SColumnInfoData* pDstStartCol = taosArrayGet(pDst->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pDstEndCol = taosArrayGet(pDst->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pDstUidCol = taosArrayGet(pDst->pDataBlock, UID_COLUMN_INDEX); - int32_t j = 0; + + int32_t j = 0; for (int32_t i = 0; i < rows; i++) { - if (taosHashGet(pReader->tbIdHash, &uidCol[i], sizeof(uint64_t))) { + if (pInfo->readerFn.tqReaderIsQueriedTable(pReader, uidCol[i])) { colDataSetVal(pDstStartCol, j, (const char*)&startCol[i], false); colDataSetVal(pDstEndCol, j, (const char*)&endCol[i], false); colDataSetVal(pDstUidCol, j, (const char*)&uidCol[i], false); @@ -1721,6 +1805,7 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S j++; } } + uint32_t cap = pDst->info.capacity; pDst->info = pSrc->info; pDst->info.rows = j; @@ -1765,21 +1850,21 @@ static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBl } } -int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) { - int32_t len = updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); - *pBuff = taosMemoryCalloc(1, len); - updateInfoSerialize(*pBuff, len, pInfo->pUpdateInfo); - return len; -} +//int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) { +// int32_t len = updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); +// *pBuff = taosMemoryCalloc(1, len); +// updateInfoSerialize(*pBuff, len, pInfo->pUpdateInfo); +// return len; +//} // other properties are recovered from the execution plan -void streamScanOperatorDeocde(void* pBuff, int32_t len, SStreamScanInfo* pInfo) { +void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo) { if (!pBuff || len == 0) { return; } - SUpdateInfo* pUpInfo = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); - int32_t code = updateInfoDeserialize(pBuff, len, pUpInfo); + void* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); + int32_t code = pInfo->stateStore.updateInfoDeserialize(pBuff, len, pUpInfo); if (code == TSDB_CODE_SUCCESS) { pInfo->pUpdateInfo = pUpInfo; } @@ -1787,44 +1872,49 @@ void streamScanOperatorDeocde(void* pBuff, int32_t len, SStreamScanInfo* pInfo) static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + const char* id = GET_TASKID(pTaskInfo); + + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SStreamScanInfo* pInfo = pOperator->info; + SStreamTaskInfo* pStreamInfo = &pTaskInfo->streamInfo; - qDebug("stream scan called"); + qDebug("stream scan started, %s", id); - if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1 || - pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE2) { + if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__PREPARE1 || pStreamInfo->recoverStep == STREAM_RECOVER_STEP__PREPARE2) { STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - memcpy(&pTSInfo->base.cond, &pTaskInfo->streamInfo.tableCond, sizeof(SQueryTableDataCond)); - if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1) { - pTSInfo->base.cond.startVersion = 0; - pTSInfo->base.cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer1; - qDebug("stream recover step 1, from %" PRId64 " to %" PRId64, pTSInfo->base.cond.startVersion, - pTSInfo->base.cond.endVersion); - pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN1; + memcpy(&pTSInfo->base.cond, &pStreamInfo->tableCond, sizeof(SQueryTableDataCond)); + + if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__PREPARE1) { + pTSInfo->base.cond.startVersion = pStreamInfo->fillHistoryVer.minVer; + pTSInfo->base.cond.endVersion = pStreamInfo->fillHistoryVer.maxVer; + + pTSInfo->base.cond.twindows = pStreamInfo->fillHistoryWindow; + qDebug("stream recover step1, verRange:%" PRId64 "-%" PRId64 " window:%"PRId64"-%"PRId64", %s", pTSInfo->base.cond.startVersion, + pTSInfo->base.cond.endVersion, pTSInfo->base.cond.twindows.skey, pTSInfo->base.cond.twindows.ekey, id); + pStreamInfo->recoverStep = STREAM_RECOVER_STEP__SCAN1; + pStreamInfo->recoverScanFinished = false; } else { - pTSInfo->base.cond.startVersion = pTaskInfo->streamInfo.fillHistoryVer1 + 1; - pTSInfo->base.cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer2; - qDebug("stream recover step 2, from %" PRId64 " to %" PRId64, pTSInfo->base.cond.startVersion, - pTSInfo->base.cond.endVersion); - pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN2; + pTSInfo->base.cond.startVersion = pStreamInfo->fillHistoryVer.minVer; + pTSInfo->base.cond.endVersion = pStreamInfo->fillHistoryVer.maxVer; + pTSInfo->base.cond.twindows = pStreamInfo->fillHistoryWindow; + qDebug("stream recover step2, verRange:%" PRId64 " - %" PRId64 ", window:%" PRId64 "-%" PRId64 ", %s", + pTSInfo->base.cond.startVersion, pTSInfo->base.cond.endVersion, pTSInfo->base.cond.twindows.skey, + pTSInfo->base.cond.twindows.ekey, id); + pStreamInfo->recoverStep = STREAM_RECOVER_STEP__NONE; } - tsdbReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; pInfo->pTableScanOp->status = OP_OPENED; pTSInfo->scanTimes = 0; pTSInfo->currentGroupId = -1; - pTaskInfo->streamInfo.recoverScanFinished = false; } - if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN1 || - pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN2) { - if (pInfo->blockRecoverContiCnt > 100) { - pInfo->blockRecoverTotCnt += pInfo->blockRecoverContiCnt; - pInfo->blockRecoverContiCnt = 0; + if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__SCAN1) { + if (isTaskKilled(pTaskInfo)) { return NULL; } @@ -1834,70 +1924,71 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { printDataBlock(pInfo->pRecoverRes, "scan recover"); return pInfo->pRecoverRes; } break; - case STREAM_SCAN_FROM_UPDATERES: { - generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); - prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; - printDataBlock(pInfo->pUpdateRes, "recover update"); - return pInfo->pUpdateRes; - } break; - case STREAM_SCAN_FROM_DELETE_DATA: { - generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); - prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; - copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); - pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; - printDataBlock(pInfo->pDeleteDataRes, "recover delete"); - return pInfo->pDeleteDataRes; - } break; - case STREAM_SCAN_FROM_DATAREADER_RANGE: { - SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); - if (pSDB) { - STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA; - checkUpdateData(pInfo, true, pSDB, false); - printDataBlock(pSDB, "scan recover update"); - calBlockTbName(pInfo, pSDB); - return pSDB; - } - blockDataCleanup(pInfo->pUpdateDataRes); - pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - } break; + // case STREAM_SCAN_FROM_UPDATERES: { + // generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); + // prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + // pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + // printDataBlock(pInfo->pUpdateRes, "recover update"); + // return pInfo->pUpdateRes; + // } break; + // case STREAM_SCAN_FROM_DELETE_DATA: { + // generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); + // prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + // pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + // copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); + // pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; + // printDataBlock(pInfo->pDeleteDataRes, "recover delete"); + // return pInfo->pDeleteDataRes; + // } break; + // case STREAM_SCAN_FROM_DATAREADER_RANGE: { + // SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); + // if (pSDB) { + // STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; + // pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA; + // checkUpdateData(pInfo, true, pSDB, false); + // printDataBlock(pSDB, "scan recover update"); + // calBlockTbName(pInfo, pSDB); + // return pSDB; + // } + // blockDataCleanup(pInfo->pUpdateDataRes); + // pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + // } break; default: break; } pInfo->pRecoverRes = doTableScan(pInfo->pTableScanOp); if (pInfo->pRecoverRes != NULL) { - pInfo->blockRecoverContiCnt++; calBlockTbName(pInfo, pInfo->pRecoverRes); - if (pInfo->pUpdateInfo) { - if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN1) { - TSKEY maxTs = updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); - } else { - doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes); - } + if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { + // if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__SCAN1) { + TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); + // } else { + // pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pStreamInfo->fillHistoryVer.maxVer); + // doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes); + // } } if (pInfo->pCreateTbRes->info.rows > 0) { pInfo->scanMode = STREAM_SCAN_FROM_RES; printDataBlock(pInfo->pCreateTbRes, "recover createTbl"); return pInfo->pCreateTbRes; } + qDebug("stream recover scan get block, rows %" PRId64, pInfo->pRecoverRes->info.rows); printDataBlock(pInfo->pRecoverRes, "scan recover"); return pInfo->pRecoverRes; } - pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; + pStreamInfo->recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->base.dataReader); + pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; pTSInfo->base.cond.startVersion = -1; pTSInfo->base.cond.endVersion = -1; - pTaskInfo->streamInfo.recoverScanFinished = true; + pStreamInfo->recoverScanFinished = true; return NULL; } @@ -1907,15 +1998,16 @@ FETCH_NEXT_BLOCK: if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { if (pInfo->validBlockIndex >= total) { doClearBufferedBlocks(pInfo); - /*pOperator->status = OP_EXEC_DONE;*/ return NULL; } - int32_t current = pInfo->validBlockIndex++; + int32_t current = pInfo->validBlockIndex++; + qDebug("process %d/%d input data blocks, %s", current, (int32_t) total, id); + SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); SSDataBlock* pBlock = pPacked->pDataBlock; if (pBlock->info.parTbName[0]) { - streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); + pAPI->stateStore.streamStatePutParName(pStreamInfo->pState, pBlock->info.id.groupId, pBlock->info.parTbName); } // TODO move into scan @@ -1925,6 +2017,7 @@ FETCH_NEXT_BLOCK: if (pInfo->pUpdateInfo) { pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); } + blockDataUpdateTsWindow(pBlock, 0); switch (pBlock->info.type) { case STREAM_NORMAL: @@ -1936,7 +2029,7 @@ FETCH_NEXT_BLOCK: copyDataBlock(pInfo->pUpdateRes, pBlock); pInfo->updateResIndex = 0; prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); - updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); + pAPI->stateStore.updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); } break; case STREAM_DELETE_DATA: { printDataBlock(pBlock, "stream scan delete recv"); @@ -1947,7 +2040,9 @@ FETCH_NEXT_BLOCK: } else { pDelBlock = pBlock; } + setBlockGroupIdByUid(pInfo, pDelBlock); + rebuildDeleteBlockData(pDelBlock, &pStreamInfo->fillHistoryWindow, id); printDataBlock(pDelBlock, "stream scan delete recv filtered"); if (pDelBlock->info.rows == 0) { if (pInfo->tqReader) { @@ -1955,6 +2050,7 @@ FETCH_NEXT_BLOCK: } goto FETCH_NEXT_BLOCK; } + if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) { generateDeleteResultBlock(pInfo, pDelBlock, pInfo->pDeleteDataRes); pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT; @@ -1991,7 +2087,7 @@ FETCH_NEXT_BLOCK: // printDataBlock(pBlock, "stream scan recv"); return pBlock; } else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) { - qDebug("scan mode %d", pInfo->scanMode); + qDebug("stream scan mode:%d, %s", pInfo->scanMode, id); switch (pInfo->scanMode) { case STREAM_SCAN_FROM_RES: { pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; @@ -2046,24 +2142,17 @@ FETCH_NEXT_BLOCK: return pInfo->pUpdateRes; } - const char* id = GET_TASKID(pTaskInfo); SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; int32_t totalBlocks = taosArrayGetSize(pInfo->pBlockLists); NEXT_SUBMIT_BLK: while (1) { - if (pInfo->tqReader->msg.msgStr == NULL) { + if (pInfo->readerFn.tqReaderCurrentBlockConsumed(pInfo->tqReader)) { if (pInfo->validBlockIndex >= totalBlocks) { - updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); + pAPI->stateStore.updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); doClearBufferedBlocks(pInfo); qDebug("stream scan return empty, all %d submit blocks consumed, %s", totalBlocks, id); - void* buff = NULL; - // int32_t len = streamScanOperatorEncode(pInfo, &buff); - // if (len > 0) { - // streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_NAME, strlen(STREAM_SCAN_OP_NAME), buff, len); - // } - taosMemoryFreeClear(buff); return NULL; } @@ -2071,7 +2160,7 @@ FETCH_NEXT_BLOCK: SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); qDebug("set %d/%d as the input submit block, %s", current, totalBlocks, id); - if (tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { + if (pAPI->tqReaderFn.tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { qError("submit msg messed up when initializing stream submit block %p, current %d/%d, %s", pSubmit, current, totalBlocks, id); continue; } @@ -2079,25 +2168,34 @@ FETCH_NEXT_BLOCK: blockDataCleanup(pInfo->pRes); - while (tqNextBlockImpl(pInfo->tqReader, id)) { - int32_t code = tqRetrieveDataBlock(pInfo->tqReader, id); - if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) { + while (pAPI->tqReaderFn.tqNextBlockImpl(pInfo->tqReader, id)) { + SSDataBlock* pRes = NULL; + + int32_t code = pAPI->tqReaderFn.tqRetrieveBlock(pInfo->tqReader, &pRes, id); + qDebug("retrieve data from submit completed code:%s rows:%" PRId64 " %s", tstrerror(code), pRes->info.rows, id); + + if (code != TSDB_CODE_SUCCESS || pRes->info.rows == 0) { + qDebug("retrieve data failed, try next block in submit block, %s", id); continue; } - setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, false); + setBlockIntoRes(pInfo, pRes, &pStreamInfo->fillHistoryWindow, false); + if (pInfo->pRes->info.rows == 0) { + continue; + } if (pInfo->pCreateTbRes->info.rows > 0) { pInfo->scanMode = STREAM_SCAN_FROM_RES; + qDebug("create table res exists, rows:%"PRId64" return from stream scan, %s", pInfo->pCreateTbRes->info.rows, id); return pInfo->pCreateTbRes; } doCheckUpdate(pInfo, pBlockInfo->window.ekey, pInfo->pRes); doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); - pInfo->pRes->info.dataLoad = 1; - blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); - if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { + int64_t numOfUpdateRes = pInfo->pUpdateDataRes->info.rows; + qDebug("%s %" PRId64 " rows in datablock, update res:%" PRId64, id, pBlockInfo->rows, numOfUpdateRes); + if (pBlockInfo->rows > 0 || numOfUpdateRes > 0) { break; } } @@ -2113,7 +2211,7 @@ FETCH_NEXT_BLOCK: pInfo->numOfExec++; pOperator->resultInfo.totalRows += pBlockInfo->rows; - qDebug("stream scan get source rows:%" PRId64", %s", pBlockInfo->rows, id); + qDebug("stream scan completed, and return source rows:%" PRId64", %s", pBlockInfo->rows, id); if (pBlockInfo->rows > 0) { return pInfo->pRes; } @@ -2123,10 +2221,9 @@ FETCH_NEXT_BLOCK: } goto NEXT_SUBMIT_BLK; - } else { - ASSERT(0); - return NULL; } + + return NULL; } static SArray* extractTableIdList(const STableListInfo* pTableListInfo) { @@ -2144,7 +2241,9 @@ static SArray* extractTableIdList(const STableListInfo* pTableListInfo) { static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SStreamRawScanInfo* pInfo = pOperator->info; int32_t code = TSDB_CODE_SUCCESS; pTaskInfo->streamInfo.metaRsp.metaRspLen = 0; // use metaRspLen !=0 to judge if data is meta @@ -2153,21 +2252,21 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { qDebug("tmqsnap doRawScan called"); if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { bool hasNext = false; - if (pInfo->dataReader) { - code = tsdbNextDataBlock(pInfo->dataReader, &hasNext); + if (pInfo->dataReader && pInfo->sContext->withMeta != ONLY_META) { + code = pAPI->tsdReader.tsdNextDataBlock(pInfo->dataReader, &hasNext); if (code) { - tsdbReleaseDataBlock(pInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, code); } } if (pInfo->dataReader && hasNext) { if (isTaskKilled(pTaskInfo)) { - tsdbReleaseDataBlock(pInfo->dataReader); + pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } - SSDataBlock* pBlock = tsdbRetrieveDataBlock(pInfo->dataReader, NULL); + SSDataBlock* pBlock = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, NULL); if (pBlock == NULL) { T_LONG_JMP(pTaskInfo->env, terrno); } @@ -2177,11 +2276,11 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { return pBlock; } - SMetaTableInfo mtInfo = getUidfromSnapShot(pInfo->sContext); + SMetaTableInfo mtInfo = pAPI->snapshotFn.getMetaTableInfoFromSnapshot(pInfo->sContext); STqOffsetVal offset = {0}; - if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal + if (mtInfo.uid == 0 || pInfo->sContext->withMeta == ONLY_META) { // read snapshot done, change to get data from wal qDebug("tmqsnap read snapshot done, change to get data from wal"); - tqOffsetResetToLog(&offset, pInfo->sContext->snapVersion); + tqOffsetResetToLog(&offset, pInfo->sContext->snapVersion + 1); } else { tqOffsetResetToData(&offset, mtInfo.uid, INT64_MIN); qDebug("tmqsnap change get data uid:%" PRId64 "", mtInfo.uid); @@ -2195,8 +2294,8 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { int32_t dataLen = 0; int16_t type = 0; int64_t uid = 0; - if (getMetafromSnapShot(sContext, &data, &dataLen, &type, &uid) < 0) { - qError("tmqsnap getMetafromSnapShot error"); + if (pAPI->snapshotFn.getTableInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) { + qError("tmqsnap getTableInfoFromSnapshot error"); taosMemoryFreeClear(data); return NULL; } @@ -2214,51 +2313,13 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { return NULL; } - // else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__LOG) { - // int64_t fetchVer = pTaskInfo->streamInfo.prepareStatus.version + 1; - // - // while(1){ - // if (tqFetchLog(pInfo->tqReader->pWalReader, pInfo->sContext->withMeta, &fetchVer, &pInfo->pCkHead) < 0) { - // qDebug("tmqsnap tmq poll: consumer log end. offset %" PRId64, fetchVer); - // pTaskInfo->streamInfo.lastStatus.version = fetchVer; - // pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__LOG; - // return NULL; - // } - // SWalCont* pHead = &pInfo->pCkHead->head; - // qDebug("tmqsnap tmq poll: consumer log offset %" PRId64 " msgType %d", fetchVer, pHead->msgType); - // - // if (pHead->msgType == TDMT_VND_SUBMIT) { - // SSubmitReq* pCont = (SSubmitReq*)&pHead->body; - // tqReaderSetDataMsg(pInfo->tqReader, pCont, 0); - // SSDataBlock* block = tqLogScanExec(pInfo->sContext->subType, pInfo->tqReader, pInfo->pFilterOutTbUid, - // &pInfo->pRes); if(block){ - // pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__LOG; - // pTaskInfo->streamInfo.lastStatus.version = fetchVer; - // qDebug("tmqsnap fetch data msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType); - // return block; - // }else{ - // fetchVer++; - // } - // } else{ - // ASSERT(pInfo->sContext->withMeta); - // ASSERT(IS_META_MSG(pHead->msgType)); - // qDebug("tmqsnap fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType); - // pTaskInfo->streamInfo.metaRsp.rspOffset.version = fetchVer; - // pTaskInfo->streamInfo.metaRsp.rspOffset.type = TMQ_OFFSET__LOG; - // pTaskInfo->streamInfo.metaRsp.resMsgType = pHead->msgType; - // pTaskInfo->streamInfo.metaRsp.metaRspLen = pHead->bodyLen; - // pTaskInfo->streamInfo.metaRsp.metaRsp = taosMemoryMalloc(pHead->bodyLen); - // memcpy(pTaskInfo->streamInfo.metaRsp.metaRsp, pHead->body, pHead->bodyLen); - // return NULL; - // } - // } return NULL; } static void destroyRawScanOperatorInfo(void* param) { SStreamRawScanInfo* pRawScan = (SStreamRawScanInfo*)param; - tsdbReaderClose(pRawScan->dataReader); - destroySnapContext(pRawScan->sContext); + pRawScan->pAPI->tsdReader.tsdReaderClose(pRawScan->dataReader); + pRawScan->pAPI->snapshotFn.destroySnapshot(pRawScan->sContext); tableListDestroy(pRawScan->pTableListInfo); taosMemoryFree(pRawScan); } @@ -2283,6 +2344,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT pInfo->pTableListInfo = tableListCreate(); pInfo->vnode = pHandle->vnode; + pInfo->pAPI = &pTaskInfo->storageAPI; pInfo->sContext = pHandle->sContext; setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, @@ -2306,7 +2368,7 @@ static void destroyStreamScanOperatorInfo(void* param) { } if (pStreamScan->tqReader) { - tqCloseReader(pStreamScan->tqReader); + pStreamScan->readerFn.tqReaderClose(pStreamScan->tqReader); } if (pStreamScan->matchInfo.pList) { taosArrayDestroy(pStreamScan->matchInfo.pList); @@ -2319,7 +2381,7 @@ static void destroyStreamScanOperatorInfo(void* param) { cleanupExprSupp(&pStreamScan->tbnameCalSup); cleanupExprSupp(&pStreamScan->tagCalSup); - updateInfoDestroy(pStreamScan->pUpdateInfo); + pStreamScan->stateStore.updateInfoDestroy(pStreamScan->pUpdateInfo); blockDataDestroy(pStreamScan->pRes); blockDataDestroy(pStreamScan->pUpdateRes); blockDataDestroy(pStreamScan->pPullDataRes); @@ -2330,11 +2392,66 @@ static void destroyStreamScanOperatorInfo(void* param) { taosMemoryFree(pStreamScan); } +void streamScanReleaseState(SOperatorInfo* pOperator) { + SStreamScanInfo* pInfo = pOperator->info; + if (!pInfo->pState) { + return; + } + if (!pInfo->pUpdateInfo) { + return; + } + int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); + void* pBuff = taosMemoryCalloc(1, len); + pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo); + pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), pBuff, len); + taosMemoryFree(pBuff); +} + +void streamScanReloadState(SOperatorInfo* pOperator) { + SStreamScanInfo* pInfo = pOperator->info; + if (!pInfo->pState) { + return; + } + void* pBuff = NULL; + int32_t len = 0; + pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), &pBuff, &len); + SUpdateInfo* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); + int32_t code = pInfo->stateStore.updateInfoDeserialize(pBuff, len, pUpInfo); + taosMemoryFree(pBuff); + if (code == TSDB_CODE_SUCCESS && pInfo->pUpdateInfo) { + if (pInfo->pUpdateInfo->minTS < 0) { + pInfo->stateStore.updateInfoDestroy(pInfo->pUpdateInfo); + pInfo->pUpdateInfo = pUpInfo; + } else { + pInfo->stateStore.windowSBfDelete(pInfo->pUpdateInfo, 1); + pInfo->stateStore.windowSBfAdd(pInfo->pUpdateInfo, 1); + ASSERT(pInfo->pUpdateInfo->minTS > pUpInfo->minTS); + pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pUpInfo->maxDataVersion); + SHashObj* curMap = pInfo->pUpdateInfo->pMap; + void *pIte = taosHashIterate(curMap, NULL); + while (pIte != NULL) { + size_t keySize = 0; + int64_t* pUid = taosHashGetKey(pIte, &keySize); + taosHashPut(pUpInfo->pMap, pUid, sizeof(int64_t), pIte, sizeof(TSKEY)); + pIte = taosHashIterate(curMap, pIte); + } + taosHashCleanup(curMap); + pInfo->pUpdateInfo->pMap = pUpInfo->pMap; + pUpInfo->pMap = NULL; + pInfo->stateStore.updateInfoDestroy(pUpInfo); + } + } else { + pInfo->stateStore.updateInfoDestroy(pUpInfo); + } +} + SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { SArray* pColIds = NULL; SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + const char* idstr = pTaskInfo->id.str; if (pInfo == NULL || pOperator == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -2378,7 +2495,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->tbnameCalSup.pExprInfo = pSubTableExpr; createExprFromOneNode(pSubTableExpr, pTableScanNode->pSubtable, 0); - if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1) != 0) { + if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1, &pTaskInfo->storageAPI.functionStore) != 0) { tableListDestroy(pTableListInfo); goto _error; } @@ -2392,7 +2509,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys tableListDestroy(pTableListInfo); goto _error; } - if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { + if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tableListDestroy(pTableListInfo); goto _error; @@ -2408,6 +2525,10 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pHandle->vnode) { SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo); + if (pTableScanOp == NULL) { + qError("createTableScanOperatorInfo error, errorcode: %d", pTaskInfo->code); + goto _error; + } STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info; if (pHandle->version > 0) { pTSInfo->base.cond.endVersion = pHandle->version; @@ -2424,7 +2545,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pHandle->initTqReader) { ASSERT(pHandle->tqReader == NULL); - pInfo->tqReader = tqReaderOpen(pHandle->vnode); + pInfo->tqReader = pAPI->tqReaderFn.tqReaderOpen(pHandle->vnode); ASSERT(pInfo->tqReader); } else { ASSERT(pHandle->tqReader); @@ -2434,7 +2555,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pUpdateInfo = NULL; pInfo->pTableScanOp = pTableScanOp; if (pInfo->pTableScanOp->pTaskInfo->streamInfo.pState) { - streamStateSetNumber(pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, -1); + pAPI->stateStore.streamStateSetNumber(pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, -1); } pInfo->readHandle = *pHandle; @@ -2443,9 +2564,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys blockDataEnsureCapacity(pInfo->pCreateTbRes, 8); // set the extract column id to streamHandle - tqReaderSetColIdList(pInfo->tqReader, pColIds); + pAPI->tqReaderFn.tqReaderSetColIdList(pInfo->tqReader, pColIds); SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo); - code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList); + code = pAPI->tqReaderFn.tqReaderSetQueryTableList(pInfo->tqReader, tableIdList, idstr); if (code != 0) { taosArrayDestroy(tableIdList); goto _error; @@ -2485,14 +2606,17 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->igCheckUpdate = pTableScanNode->igCheckUpdate; pInfo->igExpired = pTableScanNode->igExpired; pInfo->twAggSup.maxTs = INT64_MIN; - pInfo->pState = NULL; + pInfo->pState = pTaskInfo->streamInfo.pState; + pInfo->stateStore = pTaskInfo->storageAPI.stateStore; + pInfo->readerFn = pTaskInfo->storageAPI.tqReaderFn; // for stream if (pTaskInfo->streamInfo.pState) { void* buff = NULL; int32_t len = 0; - streamStateGetInfo(pTaskInfo->streamInfo.pState, STREAM_SCAN_OP_NAME, strlen(STREAM_SCAN_OP_NAME), &buff, &len); - streamScanOperatorDeocde(buff, len, pInfo); + pAPI->stateStore.streamStateGetInfo(pTaskInfo->streamInfo.pState, STREAM_SCAN_OP_NAME, strlen(STREAM_SCAN_OP_NAME), &buff, &len); + streamScanOperatorDecode(buff, len, pInfo); + taosMemoryFree(buff); } setOperatorInfo(pOperator, STREAM_SCAN_OP_NAME, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo, @@ -2502,6 +2626,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys __optr_fn_t nextFn = (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) ? doStreamScan : doQueueScan; pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamScanReleaseState, streamScanReloadState); return pOperator; @@ -2518,18 +2643,18 @@ _error: return NULL; } -static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr) { +static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr, SStorageAPI* pAPI) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STagScanInfo* pInfo = pOperator->info; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos); - int32_t code = metaGetTableEntryByUid(mr, item->uid); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(mr, item->uid); tDecoderClear(&(*mr).coder); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(mr); + pAPI->metaReaderFn.clearReader(mr); T_LONG_JMP(pTaskInfo->env, terrno); } @@ -2544,7 +2669,7 @@ static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, } else { // it is a tag value STagVal val = {0}; val.cid = pExprInfo[j].base.pParam[0].pCol->colId; - const char* p = metaGetTableTagVal((*mr).me.ctbEntry.pTags, pDst->info.type, &val); + const char* p = pAPI->metaFn.extractTagVal((*mr).me.ctbEntry.pTags, pDst->info.type, &val); char* data = NULL; if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { @@ -2569,6 +2694,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; STagScanInfo* pInfo = pOperator->info; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; @@ -2584,10 +2710,10 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { char str[512] = {0}; int32_t count = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0, &pAPI->metaFn); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { - doTagScanOneTable(pOperator, pRes, count, &mr); + doTagScanOneTable(pOperator, pRes, count, &mr, &pTaskInfo->storageAPI); ++count; if (++pInfo->curPos >= size) { setOperatorCompleted(pOperator); @@ -2605,7 +2731,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } } - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); // qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); if (pOperator->status == OP_EXEC_DONE) { @@ -2638,7 +2764,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi int32_t numOfExprs = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs); - int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2672,38 +2798,25 @@ _error: return NULL; } -static SSDataBlock* getTableDataBlockImpl(void* param) { +static SSDataBlock* getBlockForTableMergeScan(void* param) { STableMergeScanSortSourceParam* source = param; SOperatorInfo* pOperator = source->pOperator; STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - int32_t readIdx = source->readerIdx; - SSDataBlock* pBlock = source->inputBlock; - int32_t code = 0; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; - SQueryTableDataCond* pQueryCond = taosArrayGet(pInfo->queryConds, readIdx); + SSDataBlock* pBlock = pInfo->pReaderBlock; + int32_t code = 0; int64_t st = taosGetTimestampUs(); - void* p = tableListGetInfo(pInfo->base.pTableListInfo, readIdx + pInfo->tableStartIndex); - SReadHandle* pHandle = &pInfo->base.readHandle; - - if (NULL == source->dataReader || !source->multiReader) { - code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo), false); - if (code != 0) { - T_LONG_JMP(pTaskInfo->env, code); - } - } - - pInfo->base.dataReader = source->dataReader; - STsdbReader* reader = pInfo->base.dataReader; bool hasNext = false; - qTrace("tsdb/read-table-data: %p, enter next reader", reader); + STsdbReader* reader = pInfo->base.dataReader; while (true) { - code = tsdbNextDataBlock(reader, &hasNext); + code = pAPI->tsdReader.tsdNextDataBlock(reader, &hasNext); if (code != 0) { - tsdbReleaseDataBlock(reader); - pInfo->base.dataReader = NULL; + pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); + qError("table merge scan fetch next data block error code: %d, %s", code, GET_TASKID(pTaskInfo)); T_LONG_JMP(pTaskInfo->env, code); } @@ -2712,9 +2825,9 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } if (isTaskKilled(pTaskInfo)) { - tsdbReleaseDataBlock(reader); - pInfo->base.dataReader = NULL; - T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); + qInfo("table merge scan fetch next data block found task killed. %s", GET_TASKID(pTaskInfo)); + pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); + break; } // process this data block based on the probabilities @@ -2723,19 +2836,18 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { continue; } - if (pQueryCond->order == TSDB_ORDER_ASC) { - pQueryCond->twindows.skey = pBlock->info.window.ekey + 1; - } else { - pQueryCond->twindows.ekey = pBlock->info.window.skey - 1; - } - uint32_t status = 0; code = loadDataBlock(pOperator, &pInfo->base, pBlock, &status); // code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { + qInfo("table merge scan load datablock code %d, %s", code, GET_TASKID(pTaskInfo)); T_LONG_JMP(pTaskInfo->env, code); } + if (status == FUNC_DATA_REQUIRED_ALL_FILTEROUT) { + break; + } + // current block is filter out according to filter condition, continue load the next block if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) { continue; @@ -2746,20 +2858,9 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; - qTrace("tsdb/read-table-data: %p, close reader", reader); - if (!source->multiReader) { - tsdbReaderClose(pInfo->base.dataReader); - source->dataReader = NULL; - } - pInfo->base.dataReader = NULL; return pBlock; } - if (!source->multiReader) { - tsdbReaderClose(pInfo->base.dataReader); - source->dataReader = NULL; - } - pInfo->base.dataReader = NULL; return NULL; } @@ -2795,6 +2896,8 @@ int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SReadHandle* pHandle = &pInfo->base.readHandle; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; { size_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); @@ -2811,46 +2914,46 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t tableStartIdx = pInfo->tableStartIndex; int32_t tableEndIdx = pInfo->tableEndIndex; - pInfo->base.dataReader = NULL; - - // todo the total available buffer should be determined by total capacity of buffer of this task. - // the additional one is reserved for merge result - pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1); - int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, - pInfo->pSortInputBlock, pTaskInfo->id.str); + bool hasLimit = pInfo->limitInfo.limit.limit != -1 || pInfo->limitInfo.limit.offset != -1; + int64_t mergeLimit = -1; + if (hasLimit) { + mergeLimit = pInfo->limitInfo.limit.limit + pInfo->limitInfo.limit.offset; + } + size_t szRow = blockDataGetRowSize(pInfo->pResBlock); + if (hasLimit) { + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, + NULL, pTaskInfo->id.str, mergeLimit, szRow+8, tsPQSortMemThreshold * 1024* 1024); + } else { + pInfo->sortBufSize = 2048 * pInfo->bufPageSize; + int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, + pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); + + tsortSetMergeLimit(pInfo->pSortHandle, mergeLimit); + } - tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlockImpl, NULL, NULL); + tsortSetFetchRawDataFp(pInfo->pSortHandle, getBlockForTableMergeScan, NULL, NULL); // one table has one data block int32_t numOfTable = tableEndIdx - tableStartIdx + 1; - pInfo->queryConds = taosArrayInit(numOfTable, sizeof(SQueryTableDataCond)); - for (int32_t i = 0; i < numOfTable; ++i) { - STableMergeScanSortSourceParam param = {0}; - param.readerIdx = i; - param.pOperator = pOperator; - param.multiReader = (numOfTable <= MULTI_READER_MAX_TABLE_NUM) ? true : false; - param.inputBlock = createOneDataBlock(pInfo->pResBlock, false); - blockDataEnsureCapacity(param.inputBlock, pOperator->resultInfo.capacity); + STableMergeScanSortSourceParam *param = taosMemoryCalloc(1, sizeof(STableMergeScanSortSourceParam)); + param->pOperator = pOperator; + STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx); + pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, NULL); - taosArrayPush(pInfo->sortSourceParams, ¶m); - - SQueryTableDataCond cond; - dumpQueryTableCond(&pInfo->base.cond, &cond); - taosArrayPush(pInfo->queryConds, &cond); - } + SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); + ps->param = param; + ps->onlyRef = false; + tsortAddSource(pInfo->pSortHandle, ps); - for (int32_t i = 0; i < numOfTable; ++i) { - SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); - STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); - ps->param = param; - ps->onlyRef = true; - tsortAddSource(pInfo->pSortHandle, ps); + int32_t code = TSDB_CODE_SUCCESS; + if (numOfTable == 1) { + setSingleTableMerge(pInfo->pSortHandle); + } else { + code = tsortOpen(pInfo->pSortHandle); } - int32_t code = tsortOpen(pInfo->pSortHandle); - if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, terrno); } @@ -2861,8 +2964,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - int32_t numOfTable = taosArrayGetSize(pInfo->queryConds); + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle); pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod; @@ -2871,24 +2973,14 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes; pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes; - for (int32_t i = 0; i < numOfTable; ++i) { - STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); - blockDataDestroy(param->inputBlock); - tsdbReaderClose(param->dataReader); - param->dataReader = NULL; + if (pInfo->base.dataReader != NULL) { + pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader); + pInfo->base.dataReader = NULL; } - taosArrayClear(pInfo->sortSourceParams); tsortDestroySortHandle(pInfo->pSortHandle); pInfo->pSortHandle = NULL; - for (int32_t i = 0; i < taosArrayGetSize(pInfo->queryConds); i++) { - SQueryTableDataCond* cond = taosArrayGet(pInfo->queryConds, i); - taosMemoryFree(cond->colList); - } - taosArrayDestroy(pInfo->queryConds); - pInfo->queryConds = NULL; - resetLimitInfoForNextGroup(&pInfo->limitInfo); return TSDB_CODE_SUCCESS; } @@ -2901,23 +2993,32 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; blockDataCleanup(pResBlock); - + STupleHandle* pTupleHandle = NULL; while (1) { - STupleHandle* pTupleHandle = tsortNextTuple(pHandle); - if (pTupleHandle == NULL) { - break; - } + while (1) { + pTupleHandle = tsortNextTuple(pHandle); + if (pTupleHandle == NULL) { + break; + } - appendOneRowToDataBlock(pResBlock, pTupleHandle); - if (pResBlock->info.rows >= capacity) { - break; + appendOneRowToDataBlock(pResBlock, pTupleHandle); + if (pResBlock->info.rows >= capacity) { + break; + } } - } - bool limitReached = applyLimitOffset(&pInfo->limitInfo, pResBlock, pTaskInfo); - qDebug("%s get sorted row block, rows:%" PRId64 ", limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows, - pInfo->limitInfo.numOfOutputRows); + if (tsortIsClosed(pHandle)) { + terrno = TSDB_CODE_TSC_QUERY_CANCELLED; + T_LONG_JMP(pOperator->pTaskInfo->env, terrno); + } + bool limitReached = applyLimitOffset(&pInfo->limitInfo, pResBlock, pTaskInfo); + qDebug("%s get sorted row block, rows:%" PRId64 ", limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows, + pInfo->limitInfo.numOfOutputRows); + if (pTupleHandle == NULL || limitReached || pResBlock->info.rows > 0) { + break; + } + } return (pResBlock->info.rows > 0) ? pResBlock : NULL; } @@ -2981,32 +3082,20 @@ void destroyTableMergeScanOperatorInfo(void* param) { STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param; cleanupQueryTableDataCond(&pTableScanInfo->base.cond); - int32_t numOfTable = taosArrayGetSize(pTableScanInfo->queryConds); + int32_t numOfTable = taosArrayGetSize(pTableScanInfo->sortSourceParams); - for (int32_t i = 0; i < numOfTable; i++) { - STableMergeScanSortSourceParam* p = taosArrayGet(pTableScanInfo->sortSourceParams, i); - blockDataDestroy(p->inputBlock); - tsdbReaderClose(p->dataReader); - p->dataReader = NULL; - } - - tsdbReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; taosArrayDestroy(pTableScanInfo->sortSourceParams); tsortDestroySortHandle(pTableScanInfo->pSortHandle); pTableScanInfo->pSortHandle = NULL; - for (int i = 0; i < taosArrayGetSize(pTableScanInfo->queryConds); i++) { - SQueryTableDataCond* pCond = taosArrayGet(pTableScanInfo->queryConds, i); - taosMemoryFree(pCond->colList); - } - - taosArrayDestroy(pTableScanInfo->queryConds); - destroyTableScanBase(&pTableScanInfo->base); + destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); + pTableScanInfo->pReaderBlock = blockDataDestroy(pTableScanInfo->pReaderBlock); taosArrayDestroy(pTableScanInfo->pSortInfo); taosMemoryFreeClear(param); @@ -3052,7 +3141,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN if (pTableScanNode->scan.pScanPseudoCols != NULL) { SExprSupp* pSup = &pInfo->base.pseudoSup; pSup->pExprInfo = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pSup->numOfExprs); - pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset); + pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; @@ -3063,10 +3152,13 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN goto _error; } + pInfo->base.readerAPI = pTaskInfo->storageAPI.tsdReader; pInfo->base.dataBlockLoadFlag = FUNC_DATA_REQUIRED_DATA_LOAD; pInfo->base.scanFlag = MAIN_SCAN; pInfo->base.readHandle = *readHandle; + pInfo->readIdx = -1; + pInfo->base.limitInfo.limit.limit = -1; pInfo->base.limitInfo.slimit.limit = -1; pInfo->base.pTableListInfo = pTableListInfo; @@ -3089,6 +3181,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false); initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo); + pInfo->pReaderBlock = createOneDataBlock(pInfo->pResBlock, false); + int32_t rowSize = pInfo->pResBlock->info.rowSize; uint32_t nCols = taosArrayGetSize(pInfo->pResBlock->pDataBlock); pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols); @@ -3114,9 +3208,9 @@ _error: static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator); static void destoryTableCountScanOperator(void* param); static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName, tb_uid_t stbUid); + SSDataBlock* pRes, char* dbName, tb_uid_t stbUid, SStorageAPI* pAPI); static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName); + SSDataBlock* pRes, char* dbName, SStorageAPI* pAPI); static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName); static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, @@ -3203,6 +3297,7 @@ int32_t getTableCountScanSupp(SNodeList* groupTags, SName* tableName, SNodeList* qError("%s get table count scan supp. get inputs error", GET_TASKID(taskInfo)); return code; } + supp->dbNameSlotId = -1; supp->stbNameSlotId = -1; supp->tbCountSlotId = -1; @@ -3212,6 +3307,7 @@ int32_t getTableCountScanSupp(SNodeList* groupTags, SName* tableName, SNodeList* qError("%s get table count scan supp. get group tags slot id error", GET_TASKID(taskInfo)); return code; } + code = tblCountScanGetCountSlotId(pseudoCols, supp); if (code != TSDB_CODE_SUCCESS) { qError("%s get table count scan supp. get count error", GET_TASKID(taskInfo)); @@ -3371,9 +3467,11 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount const char* db = NULL; int32_t vgId = 0; char dbName[TSDB_DB_NAME_LEN] = {0}; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; // get dbname - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); tNameGetDbName(&sn, dbName); @@ -3388,20 +3486,23 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + if (pSupp->groupByStbName) { if (pInfo->stbUidList == NULL) { pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); - if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) { + if (pAPI->metaFn.storeGetTableList(pInfo->readHandle.vnode, TSDB_SUPER_TABLE, pInfo->stbUidList) < 0) { qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr()); } } if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) { tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx); - buildVnodeGroupedStbTableCount(pInfo, pSupp, pRes, dbName, stbUid); + buildVnodeGroupedStbTableCount(pInfo, pSupp, pRes, dbName, stbUid, pAPI); pInfo->currGrpIdx++; } else if (pInfo->currGrpIdx == taosArrayGetSize(pInfo->stbUidList)) { - buildVnodeGroupedNtbTableCount(pInfo, pSupp, pRes, dbName); + buildVnodeGroupedNtbTableCount(pInfo, pSupp, pRes, dbName, pAPI); pInfo->currGrpIdx++; } else { @@ -3410,7 +3511,9 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca } else { uint64_t groupId = calcGroupId(dbName, strlen(dbName)); pRes->info.id.groupId = groupId; - int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta); + + int64_t dbTableCount = 0; + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &dbTableCount, NULL); fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); setOperatorCompleted(pOperator); } @@ -3418,26 +3521,34 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + if (strlen(pSupp->dbNameFilter) != 0) { if (strlen(pSupp->stbNameFilter) != 0) { - tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter); - SMetaStbStats stats = {0}; - metaGetStbStats(pInfo->readHandle.meta, uid, &stats); - int64_t ctbNum = stats.ctbNum; - fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes); + uint64_t uid = 0; + pAPI->metaFn.getTableUidByName(pInfo->readHandle.vnode, pSupp->stbNameFilter, &uid); + + int64_t numOfChildTables = 0; + pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, uid, &numOfChildTables); + + fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, numOfChildTables, pRes); } else { - int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); + int64_t tbNumVnode = 0; + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &tbNumVnode, NULL); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } } else { - int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); + int64_t tbNumVnode = 0; + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &tbNumVnode, NULL); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } + setOperatorCompleted(pOperator); } static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName) { + SSDataBlock* pRes, char* dbName, SStorageAPI* pAPI) { char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; if (pSupp->groupByDbName) { snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, ""); @@ -3445,32 +3556,33 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; - int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta); - if (ntbNum != 0) { - fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); + + int64_t numOfTables = 0; + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, NULL, &numOfTables); + + if (numOfTables != 0) { + fillTableCountScanDataBlock(pSupp, dbName, "", numOfTables, pRes); } } static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, - SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) { + SSDataBlock* pRes, char* dbName, tb_uid_t stbUid, SStorageAPI* pAPI) { char stbName[TSDB_TABLE_NAME_LEN] = {0}; - metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); + pAPI->metaFn.getTableNameByUid(pInfo->readHandle.vnode, stbUid, stbName); char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; if (pSupp->groupByDbName) { - snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName); + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, varDataVal(stbName)); } else { - snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s", stbName); + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s", varDataVal(stbName)); } uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; - SMetaStbStats stats = {0}; - metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); - int64_t ctbNum = stats.ctbNum; - - fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); + int64_t ctbNum = 0; + int32_t code = pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, stbUid, &ctbNum); + fillTableCountScanDataBlock(pSupp, dbName, varDataVal(stbName), ctbNum, pRes); } static void destoryTableCountScanOperator(void* param) { diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 718bb596c5ef9578b6ff740d4c5d3258a1984472..459474d06e79ea0c8407509a765d887c2cd3d469 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -29,6 +29,8 @@ typedef struct SSortOperatorInfo { int64_t startTs; // sort start time uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. SLimitInfo limitInfo; + uint64_t maxTupleLength; + int64_t maxRows; } SSortOperatorInfo; static SSDataBlock* doSort(SOperatorInfo* pOperator); @@ -36,6 +38,7 @@ static int32_t doOpenSortOperator(SOperatorInfo* pOperator); static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); static void destroySortOperatorInfo(void* param); +static int32_t calcSortOperMaxTupleLength(SSortOperatorInfo* pSortOperInfo, SNodeList* pSortKeys); // todo add limit/offset impl SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortNode, SExecTaskInfo* pTaskInfo) { @@ -51,16 +54,22 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* int32_t numOfCols = 0; pOperator->exprSupp.pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols); pOperator->exprSupp.numOfExprs = numOfCols; - int32_t numOfOutputCols = 0; int32_t code = extractColMatchInfo(pSortNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; } + + calcSortOperMaxTupleLength(pInfo, pSortNode->pSortKeys); + pInfo->maxRows = -1; + if (pSortNode->node.pLimit) { + SLimitNode* pLimit = (SLimitNode*)pSortNode->node.pLimit; + if (pLimit->limit > 0) pInfo->maxRows = pLimit->limit + pLimit->offset; + } pOperator->exprSupp.pCtx = - createSqlFunctionCtx(pOperator->exprSupp.pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); + createSqlFunctionCtx(pOperator->exprSupp.pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); initResultSizeInfo(&pOperator->resultInfo, 1024); code = filterInitFromNode((SNode*)pSortNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { @@ -69,6 +78,8 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode); pInfo->pSortInfo = createSortInfo(pSortNode->pSortKeys); + pInfo->binfo.inputTsOrder = pSortNode->node.inputTsOrder; + pInfo->binfo.outputTsOrder = pSortNode->node.outputTsOrder; initLimitInfo(pSortNode->node.pLimit, pSortNode->node.pSlimit, &pInfo->limitInfo); setOperatorInfo(pOperator, "SortOperator", QUERY_NODE_PHYSICAL_PLAN_SORT, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -114,6 +125,7 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { } pBlock->info.dataLoad = 1; + pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag; pBlock->info.rows += 1; } @@ -153,7 +165,9 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } + pDataBlock->info.dataLoad = 1; pDataBlock->info.rows = p->info.rows; + pDataBlock->info.scanFlag = p->info.scanFlag; } blockDataDestroy(p); @@ -188,9 +202,9 @@ int32_t doOpenSortOperator(SOperatorInfo* pOperator) { } pInfo->startTs = taosGetTimestampUs(); - // pInfo->binfo.pRes is not equalled to the input datablock. - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, NULL, pTaskInfo->id.str); + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, NULL, pTaskInfo->id.str, + pInfo->maxRows, pInfo->maxTupleLength, tsPQSortMemThreshold * 1024 * 1024); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator); @@ -228,6 +242,11 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { // multi-group case not handle here SSDataBlock* pBlock = NULL; while (1) { + if (tsortIsClosed(pInfo->pSortHandle)) { + terrno = TSDB_CODE_TSC_QUERY_CANCELLED; + T_LONG_JMP(pOperator->pTaskInfo->env, terrno); + } + pBlock = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->matchInfo.pList, pInfo); if (pBlock == NULL) { @@ -276,6 +295,20 @@ int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* return TSDB_CODE_SUCCESS; } +static int32_t calcSortOperMaxTupleLength(SSortOperatorInfo* pSortOperInfo, SNodeList* pSortKeys) { + SColMatchInfo* pColItem = &pSortOperInfo->matchInfo; + size_t size = taosArrayGetSize(pColItem->pList); + for (size_t i = 0; i < size; ++i) { + pSortOperInfo->maxTupleLength += ((SColMatchItem*)taosArrayGet(pColItem->pList, i))->dataType.bytes; + } + size = LIST_LENGTH(pSortKeys); + for (size_t i = 0; i < size; ++i) { + SOrderByExprNode* pOrderExprNode = (SOrderByExprNode*)nodesListGetNode(pSortKeys, i); + pSortOperInfo->maxTupleLength += ((SColumnNode*)pOrderExprNode->pExpr)->node.resType.bytes; + } + return TSDB_CODE_SUCCESS; +} + //===================================================================================== // Group Sort Operator typedef enum EChildOperatorStatus { CHILD_OP_NEW_GROUP, CHILD_OP_SAME_GROUP, CHILD_OP_FINISHED } EChildOperatorStatus; @@ -330,6 +363,7 @@ SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo pDataBlock->info.rows = p->info.rows; pDataBlock->info.capacity = p->info.rows; + pDataBlock->info.scanFlag = p->info.scanFlag; } blockDataDestroy(p); @@ -373,7 +407,7 @@ int32_t beginSortGroup(SOperatorInfo* pOperator) { // pInfo->binfo.pRes is not equalled to the input datablock. pInfo->pCurrSortHandle = - tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, NULL, pTaskInfo->id.str); + tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, NULL, pTaskInfo->id.str, 0, 0, 0); tsortSetFetchRawDataFp(pInfo->pCurrSortHandle, fetchNextGroupSortDataBlock, applyScalarFunction, pOperator); @@ -439,6 +473,11 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) { SSDataBlock* pBlock = NULL; while (pInfo->pCurrSortHandle != NULL) { + if (tsortIsClosed(pInfo->pCurrSortHandle)) { + terrno = TSDB_CODE_TSC_QUERY_CANCELLED; + T_LONG_JMP(pOperator->pTaskInfo->env, terrno); + } + // beginSortGroup would fetch all child blocks of pInfo->currGroupId; ASSERT(pInfo->childOpStatus != CHILD_OP_SAME_GROUP); pBlock = getGroupSortedBlockData(pInfo->pCurrSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, @@ -500,10 +539,12 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pSup->numOfExprs = numOfCols; initResultSizeInfo(&pOperator->resultInfo, 1024); - pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); + pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + pInfo->binfo.inputTsOrder = pSortPhyNode->node.inputTsOrder; + pInfo->binfo.outputTsOrder = pSortPhyNode->node.outputTsOrder; int32_t numOfOutputCols = 0; int32_t code = extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, @@ -564,7 +605,7 @@ int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) { int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, - pInfo->pInputBlock, pTaskInfo->id.str); + pInfo->pInputBlock, pTaskInfo->id.str, 0, 0, 0); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL); tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort); @@ -681,7 +722,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData resetLimitInfoForNextGroup(&pInfo->limitInfo); } - if (p->info.rows > 0) { + if (p->info.rows > 0 || limitReached) { break; } } @@ -697,6 +738,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData } pDataBlock->info.rows = p->info.rows; + pDataBlock->info.scanFlag = p->info.scanFlag; if (pInfo->ignoreGroupId) { pDataBlock->info.id.groupId = 0; } else { @@ -798,6 +840,8 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size size_t numOfCols = taosArrayGetSize(pInfo->binfo.pRes->pDataBlock); pInfo->bufPageSize = getProperSortPageSize(rowSize, numOfCols); pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); // one additional is reserved for merged result. + pInfo->binfo.inputTsOrder = pMergePhyNode->node.inputTsOrder; + pInfo->binfo.outputTsOrder = pMergePhyNode->node.outputTsOrder; setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(openMultiwayMergeOperator, doMultiwayMerge, NULL, diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index c75c49fe77cfdb7ee690adb980c0be1059c59e02..a1f83dda2f051f7ea09b4048b0fa5a2634c49d22 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -21,27 +21,27 @@ #include "querynodes.h" #include "systable.h" #include "tname.h" -#include "ttime.h" #include "tdatablock.h" #include "tmsg.h" +#include "index.h" +#include "operator.h" #include "query.h" +#include "querytask.h" +#include "storageapi.h" #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "vnode.h" -#include "operator.h" -#include "querytask.h" - typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); typedef int32_t (*__sys_check)(SNode* cond); typedef struct SSTabFltArg { - void* pMeta; - void* pVnode; + void* pMeta; + void* pVnode; + SStorageAPI* pAPI; } SSTabFltArg; typedef struct SSysTableIndex { @@ -72,6 +72,7 @@ typedef struct SSysTableScanInfo { SLoadRemoteDataInfo loadInfo; SLimitInfo limitInfo; int32_t tbnameSlotId; + SStorageAPI* pAPI; } SSysTableScanInfo; typedef struct { @@ -153,10 +154,11 @@ static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t SFilterInfo* pFilterInfo); int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { - void* pVnode = ((SSTabFltArg*)arg)->pVnode; + SSTabFltArg* pArg = arg; + void* pVnode = pArg->pVnode; const char* db = NULL; - vnodeGetInfo(pVnode, &db, NULL); + pArg->pAPI->metaFn.getBasicInfo(pVnode, &db, NULL, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -180,10 +182,11 @@ int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { } int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { - void* pVnode = ((SSTabFltArg*)arg)->pVnode; + SSTabFltArg* pArg = arg; + void* pVnode = ((SSTabFltArg*)arg)->pVnode; int64_t vgId = 0; - vnodeGetInfo(pVnode, NULL, (int32_t*)&vgId); + pArg->pAPI->metaFn.getBasicInfo(pVnode, NULL, (int32_t*)&vgId, NULL, NULL); SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; @@ -200,7 +203,7 @@ int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { } int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; + SSTabFltArg* pArg = arg; SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; @@ -220,7 +223,8 @@ int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { } int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; + SSTabFltArg* pArg = arg; + SStorageAPI* pAPI = pArg->pAPI; SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; @@ -237,7 +241,7 @@ int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { .equal = equal, .filterFunc = func}; - int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); + int32_t ret = pAPI->metaFilter.metaFilterCreateTime(pArg->pVnode, ¶m, result); return ret; } @@ -431,8 +435,9 @@ static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) { } static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { - qDebug("sysTableScanUserCols get cols start"); - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSysTableScanInfo* pInfo = pOperator->info; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -446,7 +451,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -461,18 +466,18 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, pInfo->req.filterTb); SMetaReader smrTable = {0}; - metaReaderInit(&smrTable, pInfo->readHandle.meta, 0); - int32_t code = metaGetTableEntryByName(&smrTable, pInfo->req.filterTb); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); + int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly - metaReaderClear(&smrTable); + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly + pAPI->metaReaderFn.clearReader(&smrTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } if (smrTable.me.type == TSDB_SUPER_TABLE) { - metaReaderClear(&smrTable); + pAPI->metaReaderFn.clearReader(&smrTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; @@ -480,12 +485,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if (smrTable.me.type == TSDB_CHILD_TABLE) { int64_t suid = smrTable.me.ctbEntry.suid; - metaReaderClear(&smrTable); - metaReaderInit(&smrTable, pInfo->readHandle.meta, 0); - code = metaGetTableEntryByUid(&smrTable, suid); + pAPI->metaReaderFn.clearReader(&smrTable); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); + code = pAPI->metaReaderFn.getTableEntryByUid(&smrTable, suid); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly - metaReaderClear(&smrTable); + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly + pAPI->metaReaderFn.clearReader(&smrTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; @@ -503,7 +508,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); - metaReaderClear(&smrTable); + pAPI->metaReaderFn.clearReader(&smrTable); if (numOfRows > 0) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); @@ -517,7 +522,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } if (pInfo->pSchema == NULL) { @@ -535,8 +540,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t restore = pInfo->restore; pInfo->restore = false; - while (restore || ((ret = metaTbCursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) { - if (restore) restore = false; + + while (restore || ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) { + if (restore) { + restore = false; + } + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -560,12 +569,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { schemaRow = *(SSchemaWrapper**)schema; } else { SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); - int code = metaGetTableEntryByUid(&smrSuperTable, suid); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); + int code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d", suid, code); - metaReaderClear(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrSuperTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; @@ -573,7 +582,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&smrSuperTable.me.stbEntry.schemaRow); taosHashPut(pInfo->pSchema, &suid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES); schemaRow = schemaWrapper; - metaReaderClear(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrSuperTable); } } else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) { qDebug("sysTableScanUserCols cursor get normal table"); @@ -605,7 +614,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { blockDataDestroy(dataBlock); if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -617,7 +626,9 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSysTableScanInfo* pInfo = pOperator->info; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -631,7 +642,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -647,37 +658,37 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, condTableName); SMetaReader smrChildTable = {0}; - metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); - int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); + pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); + int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrChildTable, condTableName); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly - metaReaderClear(&smrChildTable); + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly + pAPI->metaReaderFn.clearReader(&smrChildTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } if (smrChildTable.me.type != TSDB_CHILD_TABLE) { - metaReaderClear(&smrChildTable); + pAPI->metaReaderFn.clearReader(&smrChildTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); - code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); + code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByUid - metaReaderClear(&smrSuperTable); - metaReaderClear(&smrChildTable); + // terrno has been set by pAPI->metaReaderFn.getTableEntryByUid + pAPI->metaReaderFn.clearReader(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrChildTable); blockDataDestroy(dataBlock); return NULL; } sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); - metaReaderClear(&smrSuperTable); - metaReaderClear(&smrChildTable); + pAPI->metaReaderFn.clearReader(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrChildTable); if (numOfRows > 0) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); @@ -691,11 +702,11 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } bool blockFull = false; - while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { + while ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) { continue; } @@ -704,27 +715,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&smrSuperTable); - metaCloseTbCursor(pInfo->pCur); + pAPI->metaReaderFn.clearReader(&smrSuperTable); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; T_LONG_JMP(pTaskInfo->env, terrno); } if ((smrSuperTable.me.stbEntry.schemaTag.nCols + numOfRows) > pOperator->resultInfo.capacity) { - metaTbCursorPrev(pInfo->pCur, TSDB_TABLE_MAX); + pAPI->metaFn.cursorPrev(pInfo->pCur, TSDB_TABLE_MAX); blockFull = true; } else { sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock); } - metaReaderClear(&smrSuperTable); + pAPI->metaReaderFn.clearReader(&smrSuperTable); if (blockFull || numOfRows >= pOperator->resultInfo.capacity) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); @@ -745,7 +756,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { blockDataDestroy(dataBlock); if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -955,20 +966,20 @@ static int32_t sysTableUserColsFillOneTableCols(const SSysTableScanInfo* pInfo, // table name pColInfoData = taosArrayGet(dataBlock->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, tName, false); + colDataSetVal(pColInfoData, numOfRows, tName, false); // database name pColInfoData = taosArrayGet(dataBlock->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); + colDataSetVal(pColInfoData, numOfRows, dbname, false); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, tableType, false); + colDataSetVal(pColInfoData, numOfRows, tableType, false); // col name char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(colName, schemaRow->pSchema[i].name); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, colName, false); + colDataSetVal(pColInfoData, numOfRows, colName, false); // col type int8_t colType = schemaRow->pSchema[i].type; @@ -983,10 +994,10 @@ static int32_t sysTableUserColsFillOneTableCols(const SSysTableScanInfo* pInfo, (int32_t)((schemaRow->pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(colTypeStr, colTypeLen); - colDataAppend(pColInfoData, numOfRows, (char*)colTypeStr, false); + colDataSetVal(pColInfoData, numOfRows, (char*)colTypeStr, false); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (const char*)&schemaRow->pSchema[i].bytes, false); + colDataSetVal(pColInfoData, numOfRows, (const char*)&schemaRow->pSchema[i].bytes, false); for (int32_t j = 6; j <= 8; ++j) { pColInfoData = taosArrayGet(dataBlock->pDataBlock, j); @@ -1089,7 +1100,9 @@ int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { } static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSysTableScanInfo* pInfo = pOperator->info; SSysTableIndex* pIdx = pInfo->pIdx; @@ -1100,7 +1113,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -1118,10 +1131,10 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { tb_uid_t* uid = taosArrayGet(pIdx->uids, i); SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, 0); - ret = metaGetTableEntryByUid(&mr, *uid); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0, &pAPI->metaFn); + ret = pAPI->metaReaderFn.getTableEntryByUid(&mr, *uid); if (ret < 0) { - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); continue; } STR_TO_VARSTR(n, mr.me.name); @@ -1141,20 +1154,20 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { int32_t tableType = mr.me.type; if (tableType == TSDB_CHILD_TABLE) { // create time - int64_t ts = mr.me.ctbEntry.ctime; + int64_t ts = mr.me.ctbEntry.btime; pColInfoData = taosArrayGet(p->pDataBlock, 2); colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr1 = {0}; - metaReaderInit(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); int64_t suid = mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr1, suid); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr1, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&mr1); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr1); + pAPI->metaReaderFn.clearReader(&mr); T_LONG_JMP(pTaskInfo->env, terrno); } pColInfoData = taosArrayGet(p->pDataBlock, 3); @@ -1164,7 +1177,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { STR_TO_VARSTR(n, mr1.me.name); pColInfoData = taosArrayGet(p->pDataBlock, 4); colDataSetVal(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr1); + pAPI->metaReaderFn.clearReader(&mr1); // table comment pColInfoData = taosArrayGet(p->pDataBlock, 8); @@ -1193,7 +1206,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { } else if (tableType == TSDB_NORMAL_TABLE) { // create time pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.btime, false); // number of columns pColInfoData = taosArrayGet(p->pDataBlock, 3); @@ -1229,7 +1242,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { // impl later } - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); pColInfoData = taosArrayGet(p->pDataBlock, 9); colDataSetVal(pColInfoData, numOfRows, n, false); @@ -1275,10 +1288,16 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + int8_t firstMetaCursor = 0; SSysTableScanInfo* pInfo = pOperator->info; if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); + firstMetaCursor = 1; + } + if (!firstMetaCursor) { + pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0); } blockDataCleanup(pInfo->pRes); @@ -1286,7 +1305,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { const char* db = NULL; int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); SName sn = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -1301,7 +1320,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t ret = 0; - while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { + while ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); // table name @@ -1319,20 +1338,20 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { int32_t tableType = pInfo->pCur->mr.me.type; if (tableType == TSDB_CHILD_TABLE) { // create time - int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; + int64_t ts = pInfo->pCur->mr.me.ctbEntry.btime; pColInfoData = taosArrayGet(p->pDataBlock, 2); colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); + pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr, suid); + int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&mr); - metaCloseTbCursor(pInfo->pCur); + pAPI->metaReaderFn.clearReader(&mr); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; T_LONG_JMP(pTaskInfo->env, terrno); } @@ -1345,7 +1364,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { STR_TO_VARSTR(n, mr.me.name); pColInfoData = taosArrayGet(p->pDataBlock, 4); colDataSetVal(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr); + pAPI->metaReaderFn.clearReader(&mr); // table comment pColInfoData = taosArrayGet(p->pDataBlock, 8); @@ -1373,7 +1392,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { } else if (tableType == TSDB_NORMAL_TABLE) { // create time pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); + colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.btime, false); // number of columns pColInfoData = taosArrayGet(p->pDataBlock, 3); @@ -1422,12 +1441,14 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { numOfRows = 0; if (pInfo->pRes->info.rows > 0) { + pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); break; } } } if (numOfRows > 0) { + pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); p->info.rows = numOfRows; pInfo->pRes->info.rows = numOfRows; @@ -1442,7 +1463,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); + pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; setOperatorCompleted(pOperator); } @@ -1471,7 +1492,8 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { } else { if (pInfo->showRewrite == false) { if (pCondition != NULL && pInfo->pIdx == NULL) { - SSTabFltArg arg = {.pMeta = pInfo->readHandle.meta, .pVnode = pInfo->readHandle.vnode}; + SSTabFltArg arg = { + .pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode, .pAPI = &pTaskInfo->storageAPI}; SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); idx->init = 0; @@ -1579,6 +1601,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { SSysTableScanInfo* pInfo = pOperator->info; char dbName[TSDB_DB_NAME_LEN] = {0}; + blockDataCleanup(pInfo->pRes); + const char* name = tNameGetTableName(&pInfo->name); if (pInfo->showRewrite) { getDBNameFromCondition(pInfo->pCondition, dbName); @@ -1708,7 +1732,7 @@ static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableSca SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo) { - int32_t code = TDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1726,6 +1750,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan extractTbnameSlotId(pInfo, pScanNode); + pInfo->pAPI = &pTaskInfo->storageAPI; + pInfo->accountId = pScanPhyNode->accountId; pInfo->pUser = taosStrdup((void*)pUser); pInfo->sysInfo = pScanPhyNode->sysInfo; @@ -1798,16 +1824,20 @@ void destroySysScanOperator(void* param) { if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { - metaCloseTbCursor(pInfo->pCur); + if (pInfo->pAPI->metaFn.closeTableMetaCursor != NULL) { + pInfo->pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); + } + pInfo->pCur = NULL; } + if (pInfo->pIdx) { taosArrayDestroy(pInfo->pIdx->uids); taosMemoryFree(pInfo->pIdx); pInfo->pIdx = NULL; } - if(pInfo->pSchema) { + if (pInfo->pSchema) { taosHashCleanup(pInfo->pSchema); pInfo->pSchema = NULL; } @@ -2124,15 +2154,15 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { return -1; } -static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, const char* idstr) { +static int32_t doGetTableRowSize(SReadHandle* pHandle, uint64_t uid, int32_t* rowLen, const char* idstr) { *rowLen = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, pMeta, 0); - int32_t code = metaGetTableEntryByUid(&mr, uid); + pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pHandle->api.metaFn); + int32_t code = pHandle->api.metaReaderFn.getTableEntryByUid(&mr, uid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr); - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return terrno; } @@ -2144,10 +2174,10 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con } else if (mr.me.type == TSDB_CHILD_TABLE) { uint64_t suid = mr.me.ctbEntry.suid; tDecoderClear(&mr.coder); - code = metaGetTableEntryByUid(&mr, suid); + code = pHandle->api.metaReaderFn.getTableEntryByUid(&mr, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr); - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return terrno; } @@ -2163,7 +2193,7 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con } } - metaReaderClear(&mr); + pHandle->api.metaReaderFn.clearReader(&mr); return TSDB_CODE_SUCCESS; } @@ -2174,16 +2204,17 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { SBlockDistInfo* pBlockScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN}; - int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, - (int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo)); + int32_t code = doGetTableRowSize(&pBlockScanInfo->readHandle, pBlockScanInfo->uid, (int32_t*)&blockDistInfo.rowSize, + GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } - tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); - blockDistInfo.numOfInmemRows = (int32_t)tsdbGetNumOfRowsInMemTable(pBlockScanInfo->pHandle); + pAPI->tsdReader.tsdReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); + blockDistInfo.numOfInmemRows = (int32_t)pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle); SSDataBlock* pBlock = pBlockScanInfo->pResBlock; @@ -2213,7 +2244,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { static void destroyBlockDistScanOperatorInfo(void* param) { SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; blockDataDestroy(pDistInfo->pResBlock); - tsdbReaderClose(pDistInfo->pHandle); + pDistInfo->readHandle.api.tsdReader.tsdReaderClose(pDistInfo->pHandle); tableListDestroy(pDistInfo->pTableListInfo); taosMemoryFreeClear(param); } @@ -2268,7 +2299,8 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi size_t num = tableListGetSize(pTableListInfo); void* pList = tableListGetInfo(pTableListInfo, 0); - code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str, false); + code = readHandle->api.tsdReader.tsdReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, + (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); cleanupQueryTableDataCond(&cond); if (code != 0) { goto _error; @@ -2280,7 +2312,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); - int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2295,4 +2327,4 @@ _error: taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pOperator); return NULL; -} \ No newline at end of file +} diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index fc4e82b57f2e0440c19c81c715e8c731a478e7c7..4e0dff9d4f3474c2b2335d79b0cde362f566dfa1 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -66,20 +66,25 @@ static void setNullRow(SSDataBlock* pBlock, SFillInfo* pFillInfo, int32_t rowInd } static void doSetUserSpecifiedValue(SColumnInfoData* pDst, SVariant* pVar, int32_t rowIndex, int64_t currentKey) { + bool isNull = (TSDB_DATA_TYPE_NULL == pVar->nType) ? true : false; if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { float v = 0; - GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); - colDataSetVal(pDst, rowIndex, (char*)&v, false); + GET_TYPED_DATA(v, float, pVar->nType, &pVar->f); + colDataSetVal(pDst, rowIndex, (char*)&v, isNull); } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { double v = 0; - GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); - colDataSetVal(pDst, rowIndex, (char*)&v, false); + GET_TYPED_DATA(v, double, pVar->nType, &pVar->d); + colDataSetVal(pDst, rowIndex, (char*)&v, isNull); } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { int64_t v = 0; GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); - colDataSetVal(pDst, rowIndex, (char*)&v, false); + colDataSetVal(pDst, rowIndex, (char*)&v, isNull); + } else if (IS_UNSIGNED_NUMERIC_TYPE(pDst->info.type)) { + uint64_t v = 0; + GET_TYPED_DATA(v, uint64_t, pVar->nType, &pVar->u); + colDataSetVal(pDst, rowIndex, (char*)&v, isNull); } else if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - colDataSetVal(pDst, rowIndex, (const char*)¤tKey, false); + colDataSetVal(pDst, rowIndex, (const char*)¤tKey, isNull); } else { // varchar/nchar data colDataSetNULL(pDst, rowIndex); } @@ -247,7 +252,7 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo) { static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull); -static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVal* pRowVal) { +static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVal* pRowVal, bool reset) { SColumnInfoData* pTsCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, pFillInfo->srcTsSlotId); pRowVal->key = ((int64_t*)pTsCol->pData)[rowIndex]; @@ -268,7 +273,7 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVa bool isNull = colDataIsNull_s(pSrcCol, rowIndex); char* p = colDataGetData(pSrcCol, rowIndex); - saveColData(pRowVal->pRowVal, i, p, isNull); + saveColData(pRowVal->pRowVal, i, p, reset ? true : isNull); } else { ASSERT(0); } @@ -293,10 +298,10 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t // set the next value for interpolation if (pFillInfo->currentKey < ts && ascFill) { SRowVal* pRVal = pFillInfo->type == TSDB_FILL_NEXT ? &pFillInfo->next : &pFillInfo->prev; - copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal); + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal, false); } else if (pFillInfo->currentKey > ts && !ascFill) { SRowVal* pRVal = pFillInfo->type == TSDB_FILL_NEXT ? &pFillInfo->prev : &pFillInfo->next; - copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal); + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal, false); } if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && @@ -316,9 +321,14 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t ASSERT(pFillInfo->currentKey == ts); int32_t index = pBlock->info.rows; - if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) { + if (pFillInfo->type == TSDB_FILL_NEXT) { int32_t nextRowIndex = pFillInfo->index + 1; - copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next); + if ((pFillInfo->index + 1) < pFillInfo->numOfRows) { + copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next, false); + } else { + // reset to null after last row + copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next, true); + } } // copy rows to dst buffer @@ -517,10 +527,16 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) return; } + // the endKey is now the aligned time window value. truncate time window isn't correct. pFillInfo->end = endKey; - if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->interval.precision); + +#if 0 + if (pFillInfo->order == TSDB_ORDER_ASC) { + ASSERT(pFillInfo->start <= pFillInfo->end); + } else { + ASSERT(pFillInfo->start >= pFillInfo->end); } +#endif pFillInfo->index = 0; pFillInfo->numOfRows = numOfRows; @@ -530,6 +546,13 @@ void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) pFillInfo->pSrcBlock = (SSDataBlock*)pInput; } +void taosFillUpdateStartTimestampInfo(SFillInfo* pFillInfo, int64_t ts) { + pFillInfo->start = ts; + pFillInfo->currentKey = ts; +} + +bool taosFillNotStarted(const SFillInfo* pFillInfo) {return pFillInfo->start == pFillInfo->currentKey;} + bool taosFillHasMoreResults(SFillInfo* pFillInfo) { int32_t remain = taosNumOfRemainRows(pFillInfo); if (remain > 0) { @@ -561,9 +584,10 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma ASSERT(numOfRes >= numOfRows); } else { // reach the end of data if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || - (ekey1 >= pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { + (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { return 0; } + numOfRes = taosTimeCountInterval(ekey1, pFillInfo->currentKey, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); numOfRes += 1; diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index c59669fc53c9006ae9c535fcb9cb671369adb61f..b01998564513d20d17267a64776d0d20f2461c96 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -18,6 +18,7 @@ #include "functionMgt.h" #include "operator.h" #include "querytask.h" +#include "storageapi.h" #include "tcommon.h" #include "tcompare.h" #include "tdatablock.h" @@ -43,6 +44,8 @@ typedef struct STimeSliceOperatorInfo { uint64_t groupId; SGroupKeys* pPrevGroupKey; SSDataBlock* pNextGroupRes; + SSDataBlock* pRemainRes; // save block unfinished processing + int32_t remainIndex; // the remaining index in the block to be processed } STimeSliceOperatorInfo; static void destroyTimeSliceOperatorInfo(void* param); @@ -209,6 +212,45 @@ static bool isGroupKeyFunc(SExprInfo* pExprInfo) { return (functionType == FUNCTION_TYPE_GROUP_KEY); } +static bool getIgoreNullRes(SExprSupp* pExprSup) { + for (int32_t i = 0; i < pExprSup->numOfExprs; ++i) { + SExprInfo* pExprInfo = &pExprSup->pExprInfo[i]; + + if (isInterpFunc(pExprInfo)) { + for (int32_t j = 0; j < pExprInfo->base.numOfParams; ++j) { + SFunctParam *pFuncParam = &pExprInfo->base.pParam[j]; + if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { + return pFuncParam->param.i ? true : false; + } + } + } + } + + return false; +} + +static bool checkNullRow(SExprSupp* pExprSup, SSDataBlock* pSrcBlock, int32_t index, bool ignoreNull) { + if (!ignoreNull) { + return false; + } + + for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { + SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; + + if (isInterpFunc(pExprInfo)) { + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); + + if (colDataIsNull_s(pSrc, index)) { + return true; + } + } + } + + return false; +} + + static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, SSDataBlock* pSrcBlock, int32_t index, bool beforeTs) { int32_t rows = pResBlock->info.rows; @@ -217,7 +259,8 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp // output the result - bool hasInterp = true; + int32_t fillColIndex = 0; + bool hasInterp = true; for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; @@ -229,7 +272,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp continue; } else if (isIsfilledPseudoColumn(pExprInfo)) { bool isFilled = true; - colDataAppend(pDst, pResBlock->info.rows, (char*)&isFilled, false); + colDataSetVal(pDst, pResBlock->info.rows, (char*)&isFilled, false); continue; } else if (!isInterpFunc(pExprInfo)) { if (isGroupKeyFunc(pExprInfo)) { @@ -267,20 +310,41 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp case TSDB_FILL_SET_VALUE: case TSDB_FILL_SET_VALUE_F: { - SVariant* pVar = &pSliceInfo->pFillColInfo[j].fillVal; + SVariant* pVar = &pSliceInfo->pFillColInfo[fillColIndex].fillVal; + bool isNull = (TSDB_DATA_TYPE_NULL == pVar->nType) ? true : false; if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { float v = 0; - GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); - colDataSetVal(pDst, rows, (char*)&v, false); + if (!IS_VAR_DATA_TYPE(pVar->nType)) { + GET_TYPED_DATA(v, float, pVar->nType, &pVar->f); + } else { + v = taosStr2Float(varDataVal(pVar->pz), NULL); + } + colDataSetVal(pDst, rows, (char*)&v, isNull); } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { double v = 0; - GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); - colDataSetVal(pDst, rows, (char*)&v, false); + if (!IS_VAR_DATA_TYPE(pVar->nType)) { + GET_TYPED_DATA(v, double, pVar->nType, &pVar->d); + } else { + v = taosStr2Double(varDataVal(pVar->pz), NULL); + } + colDataSetVal(pDst, rows, (char*)&v, isNull); } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); - colDataSetVal(pDst, rows, (char*)&v, false); + if (!IS_VAR_DATA_TYPE(pVar->nType)) { + GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); + } else { + v = taosStr2Int64(varDataVal(pVar->pz), NULL, 10); + } + colDataSetVal(pDst, rows, (char*)&v, isNull); + } else if (IS_UNSIGNED_NUMERIC_TYPE(pDst->info.type)) { + uint64_t v = 0; + if (!IS_VAR_DATA_TYPE(pVar->nType)) { + GET_TYPED_DATA(v, uint64_t, pVar->nType, &pVar->u); + } else { + v = taosStr2UInt64(varDataVal(pVar->pz), NULL, 10); + } + colDataSetVal(pDst, rows, (char*)&v, isNull); } else if (IS_BOOLEAN_TYPE(pDst->info.type)) { bool v = false; if (!IS_VAR_DATA_TYPE(pVar->nType)) { @@ -288,8 +352,10 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else { v = taosStr2Int8(varDataVal(pVar->pz), NULL, 10); } - colDataSetVal(pDst, rows, (char*)&v, false); + colDataSetVal(pDst, rows, (char*)&v, isNull); } + + ++fillColIndex; break; } @@ -310,6 +376,11 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp break; } + if (end.key != INT64_MIN && end.key < pSliceInfo->current) { + hasInterp = false; + break; + } + if (start.key == INT64_MIN || end.key == INT64_MIN) { colDataSetNULL(pDst, rows); break; @@ -584,13 +655,47 @@ static int32_t resetKeeperInfo(STimeSliceOperatorInfo* pInfo) { return TSDB_CODE_SUCCESS; } +static bool checkThresholdReached(STimeSliceOperatorInfo* pSliceInfo, int32_t threshold) { + SSDataBlock* pResBlock = pSliceInfo->pRes; + if (pResBlock->info.rows > threshold) { + return true; + } + + return false; +} + +static bool checkWindowBoundReached(STimeSliceOperatorInfo* pSliceInfo) { + if (pSliceInfo->current > pSliceInfo->win.ekey) { + return true; + } + + return false; +} + +static void saveBlockStatus(STimeSliceOperatorInfo* pSliceInfo, SSDataBlock* pBlock, int32_t curIndex) { + SSDataBlock* pResBlock = pSliceInfo->pRes; + + SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId); + if (curIndex < pBlock->info.rows - 1) { + pSliceInfo->pRemainRes = pBlock; + pSliceInfo->remainIndex = curIndex + 1; + return; + } + + // all data in remaining block processed + pSliceInfo->pRemainRes = NULL; + +} + static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pSliceInfo, SSDataBlock* pBlock, - SExecTaskInfo* pTaskInfo) { + SExecTaskInfo* pTaskInfo, bool ignoreNull) { SSDataBlock* pResBlock = pSliceInfo->pRes; SInterval* pInterval = &pSliceInfo->interval; SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId); - for (int32_t i = 0; i < pBlock->info.rows; ++i) { + + int32_t i = (pSliceInfo->pRemainRes == NULL) ? 0 : pSliceInfo->remainIndex; + for (; i < pBlock->info.rows; ++i) { int64_t ts = *(int64_t*)colDataGetData(pTsCol, i); // check for duplicate timestamps @@ -598,8 +703,8 @@ static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pS T_LONG_JMP(pTaskInfo->env, TSDB_CODE_FUNC_DUP_TIMESTAMP); } - if (pSliceInfo->current > pSliceInfo->win.ekey) { - break; + if (checkNullRow(&pOperator->exprSupp, pBlock, i, ignoreNull)) { + continue; } if (ts == pSliceInfo->current) { @@ -610,9 +715,14 @@ static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pS pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (pSliceInfo->current > pSliceInfo->win.ekey) { + + if (checkWindowBoundReached(pSliceInfo)) { break; } + if (checkThresholdReached(pSliceInfo, pOperator->resultInfo.threshold)) { + saveBlockStatus(pSliceInfo, pBlock, i); + return; + } } else if (ts < pSliceInfo->current) { // in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate doKeepPrevRows(pSliceInfo, pBlock, i); @@ -633,9 +743,13 @@ static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pS } } - if (pSliceInfo->current > pSliceInfo->win.ekey) { + if (checkWindowBoundReached(pSliceInfo)) { break; } + if (checkThresholdReached(pSliceInfo, pOperator->resultInfo.threshold)) { + saveBlockStatus(pSliceInfo, pBlock, i); + return; + } } else { // ignore current row, and do nothing } @@ -666,11 +780,20 @@ static void doTimesliceImpl(SOperatorInfo* pOperator, STimeSliceOperatorInfo* pS } doKeepPrevRows(pSliceInfo, pBlock, i); - if (pSliceInfo->current > pSliceInfo->win.ekey) { + if (checkWindowBoundReached(pSliceInfo)) { break; } + if (checkThresholdReached(pSliceInfo, pOperator->resultInfo.threshold)) { + saveBlockStatus(pSliceInfo, pBlock, i); + return; + } } } + + // if reached here, meaning block processing finished naturally, + // or interpolation reach window upper bound + pSliceInfo->pRemainRes = NULL; + } static void genInterpAfterDataBlock(STimeSliceOperatorInfo* pSliceInfo, SOperatorInfo* pOperator, int32_t index) { @@ -717,38 +840,69 @@ static void resetTimesliceInfo(STimeSliceOperatorInfo* pSliceInfo) { resetKeeperInfo(pSliceInfo); } +static void doHandleTimeslice(SOperatorInfo* pOperator, SSDataBlock* pBlock) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + STimeSliceOperatorInfo* pSliceInfo = pOperator->info; + SExprSupp* pSup = &pOperator->exprSupp; + bool ignoreNull = getIgoreNullRes(pSup); + int32_t order = TSDB_ORDER_ASC; + + int32_t code = initKeeperInfo(pSliceInfo, pBlock, &pOperator->exprSupp); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, code); + } + + if (pSliceInfo->scalarSup.pExprInfo != NULL) { + SExprSupp* pExprSup = &pSliceInfo->scalarSup; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true); + doTimesliceImpl(pOperator, pSliceInfo, pBlock, pTaskInfo, ignoreNull); + copyPrevGroupKey(&pOperator->exprSupp, pSliceInfo->pPrevGroupKey, pBlock); +} + static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STimeSliceOperatorInfo* pSliceInfo = pOperator->info; SSDataBlock* pResBlock = pSliceInfo->pRes; - SExprSupp* pSup = &pOperator->exprSupp; - int32_t order = TSDB_ORDER_ASC; - SInterval* pInterval = &pSliceInfo->interval; SOperatorInfo* downstream = pOperator->pDownstream[0]; - blockDataCleanup(pResBlock); while (1) { if (pSliceInfo->pNextGroupRes != NULL) { - setInputDataBlock(pSup, pSliceInfo->pNextGroupRes, order, MAIN_SCAN, true); - doTimesliceImpl(pOperator, pSliceInfo, pSliceInfo->pNextGroupRes, pTaskInfo); - copyPrevGroupKey(&pOperator->exprSupp, pSliceInfo->pPrevGroupKey, pSliceInfo->pNextGroupRes); + doHandleTimeslice(pOperator, pSliceInfo->pNextGroupRes); + if (checkWindowBoundReached(pSliceInfo) || checkThresholdReached(pSliceInfo, pOperator->resultInfo.threshold)) { + doFilter(pResBlock, pOperator->exprSupp.pFilterInfo, NULL); + if (pSliceInfo->pRemainRes == NULL) { + pSliceInfo->pNextGroupRes = NULL; + } + if (pResBlock->info.rows != 0) { + goto _finished; + } else { + // after fillter if result block has 0 rows, go back to + // process pNextGroupRes again for unfinished data + continue; + } + } pSliceInfo->pNextGroupRes = NULL; } while (1) { - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + SSDataBlock* pBlock = pSliceInfo->pRemainRes ? pSliceInfo->pRemainRes : downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { setOperatorCompleted(pOperator); break; } + pResBlock->info.scanFlag = pBlock->info.scanFlag; if (pSliceInfo->groupId == 0 && pBlock->info.id.groupId != 0) { pSliceInfo->groupId = pBlock->info.id.groupId; } else { @@ -759,21 +913,15 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } - if (pSliceInfo->scalarSup.pExprInfo != NULL) { - SExprSupp* pExprSup = &pSliceInfo->scalarSup; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); - } - - int32_t code = initKeeperInfo(pSliceInfo, pBlock, &pOperator->exprSupp); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); + doHandleTimeslice(pOperator, pBlock); + if (checkWindowBoundReached(pSliceInfo) || checkThresholdReached(pSliceInfo, pOperator->resultInfo.threshold)) { + doFilter(pResBlock, pOperator->exprSupp.pFilterInfo, NULL); + if (pResBlock->info.rows != 0) { + goto _finished; + } } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true); - doTimesliceImpl(pOperator, pSliceInfo, pBlock, pTaskInfo); - copyPrevGroupKey(&pOperator->exprSupp, pSliceInfo->pPrevGroupKey, pBlock); } + // post work for a specific group // check if need to interpolate after last datablock // except for fill(next), fill(linear) @@ -786,11 +934,12 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // restore initial value for next group resetTimesliceInfo(pSliceInfo); - if (pResBlock->info.rows >= 4096) { + if (pResBlock->info.rows != 0) { break; } } +_finished: // restore the value setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); if (pResBlock->info.rows == 0) { @@ -812,7 +961,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode int32_t numOfExprs = 0; SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs); - int32_t code = initExprSupp(pSup, pExprInfo, numOfExprs); + int32_t code = initExprSupp(pSup, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -820,7 +969,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode if (pInterpPhyNode->pExprs != NULL) { int32_t num = 0; SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num); - code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -846,6 +995,8 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode pInfo->groupId = 0; pInfo->pPrevGroupKey = NULL; pInfo->pNextGroupRes = NULL; + pInfo->pRemainRes = NULL; + pInfo->remainIndex = 0; if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 652825165cc865983a7dd77d23cd562ed9f52d61..3abb4f44f4e250cf9b05ad906a4b36eddfa0aea8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -28,6 +28,9 @@ #define IS_FINAL_OP(op) ((op)->isFinal) #define DEAULT_DELETE_MARK (1000LL * 60LL * 60LL * 24LL * 365LL * 10LL); +#define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState" +#define STREAM_SESSION_OP_STATE_NAME "StreamSessionHistoryState" +#define STREAM_STATE_OP_STATE_NAME "StreamStateHistoryState" typedef struct SStateWindowInfo { SResultWindowInfo winInfo; @@ -103,9 +106,8 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo return TSDB_CODE_SUCCESS; } -static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) { +static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, int64_t delta) { int64_t* ts = (int64_t*)pColData->pData; - int32_t delta = includeEndpoint ? 1 : 0; int64_t duration = pWin->ekey - pWin->skey + delta; ts[2] = duration; // set the duration @@ -271,43 +273,6 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary return num; } -static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm, NULL); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision); - - tw->ekey -= 1; -} - -void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) { - getNextTimeWindow(pInterval, pInterval->precision, order, tw); -} - void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) { SqlFunctionCtx* pCtx = pSup->pCtx; @@ -343,7 +308,8 @@ void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY pr pCtx[k].end.key = curTs; pCtx[k].end.val = v2; - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR || + pColInfo->info.type == TSDB_DATA_TYPE_GEOMETRY) { if (prevRowIndex == -1) { // pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index]; } else { @@ -391,7 +357,7 @@ static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, in static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, int32_t pos, SSDataBlock* pBlock, const TSKEY* tsCols, STimeWindow* win, SExprSupp* pSup) { - bool ascQuery = (pInfo->inputOrder == TSDB_ORDER_ASC); + bool ascQuery = (pInfo->binfo.inputTsOrder == TSDB_ORDER_ASC); TSKEY curTs = tsCols[pos]; @@ -421,7 +387,7 @@ static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, i static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SExprSupp* pSup, int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { - int32_t order = pInfo->inputOrder; + int32_t order = pInfo->binfo.inputTsOrder; TSKEY actualEndKey = tsCols[endRowIndex]; TSKEY key = (order == TSDB_ORDER_ASC) ? win->ekey : win->skey; @@ -447,15 +413,17 @@ static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SEx return true; } -bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd) { - if (pInterval->interval != pInterval->sliding && (pWin->ekey < calStart || pWin->skey > calEnd)) { +bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType) { + if (pInterval->interval != pInterval->sliding && + ((pWin->ekey < calStart || pWin->skey > calEnd) || (blockType == STREAM_PULL_DATA && pWin->skey < calStart) )) { return false; } + return true; } bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo) { - return inCalSlidingWindow(pInterval, pWin, pBlockInfo->calWin.skey, pBlockInfo->calWin.ekey); + return inCalSlidingWindow(pInterval, pWin, pBlockInfo->calWin.skey, pBlockInfo->calWin.ekey, pBlockInfo->type); } static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, @@ -463,7 +431,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, bool ascQuery = (order == TSDB_ORDER_ASC); int32_t precision = pInterval->precision; - getNextTimeWindow(pInterval, precision, order, pNext); + getNextTimeWindow(pInterval, pNext, order); // next time window is not in current block if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) || @@ -508,7 +476,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, if (ascQuery && primaryKeys[startPos] > pNext->ekey) { TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->skey = taosTimeTruncate(next, pInterval); pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; @@ -517,7 +485,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) { TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->skey = taosTimeTruncate(next, pInterval); pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; @@ -581,7 +549,7 @@ static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataB if (!done) { int32_t endRowIndex = startPos + forwardRows - 1; - TSKEY endKey = (pInfo->inputOrder == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey; + TSKEY endKey = (pInfo->binfo.inputTsOrder == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey; bool interp = setTimeWindowInterpolationEndTs(pInfo, pSup, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); @@ -676,7 +644,7 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP); - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &w, true); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &w, 1); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0, pBlock->info.rows, numOfExprs); @@ -908,7 +876,67 @@ bool needDeleteWindowBuf(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) { return pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark; } -static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, +static bool tsKeyCompFn(void* l, void* r, void* param) { + TSKEY* lTS = (TSKEY*)l; + TSKEY* rTS = (TSKEY*)r; + SIntervalAggOperatorInfo* pInfo = param; + return pInfo->binfo.outputTsOrder == ORDER_ASC ? *lTS < *rTS : *lTS > *rTS; +} + +static bool isCalculatedWin(SIntervalAggOperatorInfo* pInfo, const STimeWindow* win, uint64_t tableGroupId) { + char keyBuf[sizeof(TSKEY) + sizeof(uint64_t)] = {0}; + SET_RES_WINDOW_KEY(keyBuf, (char*)&win->skey, sizeof(TSKEY), tableGroupId); + return tSimpleHashGet(pInfo->aggSup.pResultRowHashTable, keyBuf, GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))) != NULL; +} + +/** + * @brief check if cur window should be filtered out by limit info + * @retval true if should be filtered out + * @retval false if not filtering out + * @note If no limit info, we skip filtering. + * If input/output ts order mismatch, we skip filtering too. + * eg. input ts order: desc, and output ts order: asc, limit: 10 + * IntervalOperator should output the first 10 windows, however, we can't find the first 10 windows until we scan + * every tuple in every block. + * And the boundedQueue keeps refreshing all records with smaller ts key. + */ +static bool filterWindowWithLimit(SIntervalAggOperatorInfo* pOperatorInfo, STimeWindow* win, uint64_t groupId) { + if (!pOperatorInfo->limited // if no limit info, no filter will be applied + || pOperatorInfo->binfo.inputTsOrder != + pOperatorInfo->binfo.outputTsOrder // if input/output ts order mismatch, no filter + ) { + return false; + } + if (pOperatorInfo->limit == 0) return true; + + if (pOperatorInfo->pBQ == NULL) { + pOperatorInfo->pBQ = createBoundedQueue(pOperatorInfo->limit - 1, tsKeyCompFn, taosMemoryFree, pOperatorInfo); + } + + bool shouldFilter = false; + // if BQ has been full, compare it with top of BQ + if (taosBQSize(pOperatorInfo->pBQ) == taosBQMaxSize(pOperatorInfo->pBQ) + 1) { + PriorityQueueNode* top = taosBQTop(pOperatorInfo->pBQ); + shouldFilter = tsKeyCompFn(top->data, &win->skey, pOperatorInfo); + } + if (shouldFilter) { + return true; + } else if (isCalculatedWin(pOperatorInfo, win, groupId)) { + return false; + } + + // cur win not been filtered out and not been pushed into BQ yet, push it into BQ + PriorityQueueNode node = {.data = taosMemoryMalloc(sizeof(TSKEY))}; + *((TSKEY*)node.data) = win->skey; + + if (NULL == taosBQPush(pOperatorInfo->pBQ, &node)) { + taosMemoryFree(node.data); + return true; + } + return false; +} + +static bool hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, int32_t scanFlag) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; @@ -919,12 +947,25 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul int32_t numOfOutput = pSup->numOfExprs; int64_t* tsCols = extractTsCol(pBlock, pInfo); uint64_t tableGroupId = pBlock->info.id.groupId; - bool ascScan = (pInfo->inputOrder == TSDB_ORDER_ASC); + bool ascScan = (pInfo->binfo.inputTsOrder == TSDB_ORDER_ASC); TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); SResultRow* pResult = NULL; + if (tableGroupId != pInfo->curGroupId) { + pInfo->handledGroupNum += 1; + if (pInfo->slimited && pInfo->handledGroupNum > pInfo->slimit) { + return true; + } else { + pInfo->curGroupId = tableGroupId; + destroyBoundedQueue(pInfo->pBQ); + pInfo->pBQ = NULL; + } + } + STimeWindow win = - getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->inputOrder); + getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->binfo.inputTsOrder); + if (filterWindowWithLimit(pInfo, &win, tableGroupId)) return false; + int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { @@ -933,7 +974,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul TSKEY ekey = ascScan ? win.ekey : win.skey; int32_t forwardRows = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder); + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->binfo.inputTsOrder); // prev time window not interpolation yet. if (pInfo->timeWindowInterpo) { @@ -951,7 +992,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup); } - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, 1); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, numOfOutput); @@ -960,8 +1001,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul STimeWindow nextWin = win; while (1) { int32_t prevEndPos = forwardRows - 1 + startPos; - startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->inputOrder); - if (startPos < 0) { + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->binfo.inputTsOrder); + if (startPos < 0 || filterWindowWithLimit(pInfo, &nextWin, tableGroupId)) { break; } // null data, failed to allocate more memory buffer @@ -973,7 +1014,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul ekey = ascScan ? nextWin.ekey : nextWin.skey; forwardRows = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder); + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->binfo.inputTsOrder); // window start(end) key interpolation doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup); // TODO: add to open window? how to close the open windows after input blocks exhausted? @@ -986,7 +1027,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul addToOpenWindowList(pResultRowInfo, pResult, tableGroupId); } #endif - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, numOfOutput); doCloseWindow(pResultRowInfo, pInfo, pResult); @@ -995,6 +1036,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul if (pInfo->timeWindowInterpo) { saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols); } + return false; } void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) { @@ -1066,7 +1108,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { break; } - getTableScanInfo(pOperator, &pInfo->inputOrder, &scanFlag, true); + pInfo->binfo.pRes->info.scanFlag = scanFlag = pBlock->info.scanFlag; if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; @@ -1074,11 +1116,11 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pSup, pBlock, pInfo->inputOrder, scanFlag, true); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag); + setInputDataBlock(pSup, pBlock, pInfo->binfo.inputTsOrder, scanFlag, true); + if (hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag)) break; } - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->resultTsOrder); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->binfo.outputTsOrder); OPTR_SET_OPENED(pOperator); pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; @@ -1153,7 +1195,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, 0); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, pBlock->info.rows, numOfOutput); @@ -1178,7 +1220,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, 0); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, pBlock->info.rows, numOfOutput); } @@ -1192,7 +1234,7 @@ static int32_t openStateWindowAggOptr(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExprSupp* pSup = &pOperator->exprSupp; - int32_t order = TSDB_ORDER_ASC; + int32_t order = pInfo->binfo.inputTsOrder; int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; @@ -1202,6 +1244,7 @@ static int32_t openStateWindowAggOptr(SOperatorInfo* pOperator) { break; } + pInfo->binfo.pRes->info.scanFlag = pBlock->info.scanFlag; setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true); blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId); @@ -1331,13 +1374,17 @@ static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, } static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) { + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamIntervalOperatorInfo* pInfo = pOperator->info; SWinKey key = {.ts = ts, .groupId = groupId}; tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey)); - streamStateDel(pInfo->pState, &key); + pAPI->stateStore.streamStateDel(pInfo->pState, &key); return true; } +static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } + static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDataBlock* pBlock, SArray* pUpWins, SSHashObj* pUpdatedMap) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; @@ -1364,16 +1411,22 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa } do { - if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i])) { - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i], pBlock->info.type)) { + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); continue; } uint64_t winGpId = pGpDatas[i]; SWinKey winRes = {.ts = win.skey, .groupId = winGpId}; void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey)); if (chIds) { - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); - continue; + int32_t childId = getChildIndex(pBlock); + SArray* chArray = *(void**)chIds; + int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ); + if (index != -1) { + qDebug("===stream===try push delete window%" PRId64 "chId:%d ,continue", win.skey, childId); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); + continue; + } } bool res = doDeleteWindow(pOperator, win.skey, winGpId); if (pUpWins && res) { @@ -1382,7 +1435,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa if (pUpdatedMap) { tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); } - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); } while (win.ekey <= endTsCols[i]); } } @@ -1471,7 +1524,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin for (int32_t i = *index; i < size; i++) { SWinKey* pWin = taosArrayGet(pWins, i); void* tbname = NULL; - streamStateGetParName(pInfo->pState, pWin->groupId, &tbname); + pInfo->statestore.streamStateGetParName(pInfo->pState, pWin->groupId, &tbname); if (tbname == NULL) { appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL); } else { @@ -1479,7 +1532,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName); } - streamFreeVal(tbname); + pInfo->statestore.streamStateFreeVal(tbname); (*index)++; } } @@ -1516,6 +1569,7 @@ void destroyIntervalOperatorInfo(void* param) { cleanupGroupResInfo(&pInfo->groupResInfo); colDataDestroy(&pInfo->twAggSup.timeWindowData); + destroyBoundedQueue(pInfo->pBQ); taosMemoryFreeClear(param); } @@ -1529,11 +1583,12 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { taosArrayDestroy(*(void**)pIte); } taosHashCleanup(pInfo->pPullDataMap); + taosHashCleanup(pInfo->pFinalPullDataMap); taosArrayDestroy(pInfo->pPullWins); blockDataDestroy(pInfo->pPullDataRes); taosArrayDestroy(pInfo->pDelWins); blockDataDestroy(pInfo->pDelRes); - streamFileStateDestroy(pInfo->pState->pFileState); + pInfo->statestore.streamFileStateDestroy(pInfo->pState->pFileState); taosMemoryFreeClear(pInfo->pState); nodesDestroyNode((SNode*)pInfo->pPhyNode); @@ -1608,16 +1663,20 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt } void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo) { + SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore; + if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { initIntervalDownStream(downstream->pDownstream[0], type, pInfo); return; } + SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup; - if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark); + if (!pScanInfo->pUpdateInfo) { + pScanInfo->pUpdateInfo = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark); } + pScanInfo->interval = pInfo->interval; pScanInfo->twAggSup = pInfo->twAggSup; pScanInfo->pState = pInfo->pState; @@ -1650,7 +1709,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num); int32_t code = - initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState); + initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1669,16 +1728,27 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh }; pInfo->win = pTaskInfo->window; - pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; + pInfo->binfo.inputTsOrder = pPhyNode->window.node.inputTsOrder; + pInfo->binfo.outputTsOrder = pPhyNode->window.node.outputTsOrder; pInfo->interval = interval; pInfo->twAggSup = as; pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock; + if (pPhyNode->window.node.pLimit) { + SLimitNode* pLimit = (SLimitNode*)pPhyNode->window.node.pLimit; + pInfo->limited = true; + pInfo->limit = pLimit->limit + pLimit->offset; + } + if (pPhyNode->window.node.pSlimit) { + SLimitNode* pLimit = (SLimitNode*)pPhyNode->window.node.pSlimit; + pInfo->slimited = true; + pInfo->slimit = pLimit->limit + pLimit->offset; + pInfo->curGroupId = UINT64_MAX; + } if (pPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1769,7 +1839,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator } // pInfo->numOfRows data belong to the current session window - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, 0); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, pBlock->info.rows, numOfOutput); @@ -1787,7 +1857,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, 0); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, pBlock->info.rows, numOfOutput); } @@ -1821,7 +1891,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { } int64_t st = taosGetTimestampUs(); - int32_t order = TSDB_ORDER_ASC; + int32_t order = pInfo->binfo.inputTsOrder; SOperatorInfo* downstream = pOperator->pDownstream[0]; @@ -1831,6 +1901,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { break; } + pBInfo->pRes->info.scanFlag = pBlock->info.scanFlag; // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true); blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId); @@ -1873,12 +1944,12 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi } int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; - SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; + SColumnNode* pColNode = (SColumnNode*)(pStateNode->pStateKey); if (pStateNode->window.pExprs != NULL) { int32_t numOfScalarExpr = 0; SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalarExpr); - int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); + int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1891,6 +1962,8 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi if (pInfo->stateKey.pData == NULL) { goto _error; } + pInfo->binfo.inputTsOrder = pStateNode->window.node.inputTsOrder; + pInfo->binfo.outputTsOrder = pStateNode->window.node.outputTsOrder; int32_t code = filterInitFromNode((SNode*)pStateNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { @@ -1904,7 +1977,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi initResultSizeInfo(&pOperator->resultInfo, 4096); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1973,7 +2046,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW initBasicInfo(&pInfo->binfo, pResBlock); int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1989,6 +2062,8 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW pInfo->binfo.pRes = pResBlock; pInfo->winSup.prevTs = INT64_MIN; pInfo->reptScan = false; + pInfo->binfo.inputTsOrder = pSessionNode->window.node.inputTsOrder; + pInfo->binfo.outputTsOrder = pSessionNode->window.node.outputTsOrder; code = filterInitFromNode((SNode*)pSessionNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -2045,33 +2120,33 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 } } -bool hasIntervalWindow(SStreamState* pState, SWinKey* pKey) { return streamStateCheck(pState, pKey); } +bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) { return pStore->streamStateCheck(pState, pKey); } -int32_t setIntervalOutputBuf(SStreamState* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId, +int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, - SAggSupporter* pAggSup) { - SWinKey key = { - .ts = win->skey, - .groupId = groupId, - }; + SAggSupporter* pAggSup, SStateStore* pStore) { + + SWinKey key = { .ts = win->skey, .groupId = groupId }; char* value = NULL; int32_t size = pAggSup->resultRowSize; - if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) { + if (pStore->streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) { return TSDB_CODE_OUT_OF_MEMORY; } + *pResult = (SRowBuffPos*)value; SResultRow* res = (SResultRow*)((*pResult)->pRowBuff); + // set time window for current result res->win = (*win); setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset); return TSDB_CODE_SUCCESS; } -bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) { +bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, SStateStore* pStore) { if (pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) { SWinKey key = {.ts = pWin->skey, .groupId = groupId}; - if (!hasIntervalWindow(pState, &key)) { + if (!hasIntervalWindow(pState, &key, pStore)) { return true; } return false; @@ -2095,14 +2170,12 @@ void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) { taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*)); } -static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } - static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) { tSimpleHashClear(pInfo->aggSup.pResultRowHashTable); clearDiskbasedBuf(pInfo->aggSup.pResultBuf); initResultRowInfo(&pInfo->binfo.resultRowInfo); pInfo->aggSup.currentPageId = -1; - streamStateClear(pInfo->pState); + pInfo->statestore.streamStateClear(pInfo->pState); } static void clearSpecialDataBlock(SSDataBlock* pBlock) { @@ -2140,29 +2213,46 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB blockDataUpdateTsWindow(pBlock, 0); } -void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SInterval* pInterval) { - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); +void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, SArray* pPullWins, int32_t numOfCh, SOperatorInfo* pOperator) { + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); TSKEY* tsData = (TSKEY*)pStartCol->pData; - SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); TSKEY* tsEndData = (TSKEY*)pEndCol->pData; SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; int32_t chId = getChildIndex(pBlock); for (int32_t i = 0; i < pBlock->info.rows; i++) { TSKEY winTs = tsData[i]; - while (winTs < tsEndData[i]) { + while (winTs <= tsEndData[i]) { SWinKey winRes = {.ts = winTs, .groupId = groupIdData[i]}; void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey)); if (chIds) { SArray* chArray = *(SArray**)chIds; int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); if (index != -1) { - qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId); + qDebug("===stream===retrive window %" PRId64 " delete child id %d", winRes.ts, chId); taosArrayRemove(chArray, index); if (taosArrayGetSize(chArray) == 0) { // pull data is over taosArrayDestroy(chArray); taosHashRemove(pMap, &winRes, sizeof(SWinKey)); + qDebug("===stream===retrive pull data over.window %" PRId64 , winRes.ts); + + void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey)); + if (pFinalCh) { + taosHashRemove(pFinalMap, &winRes, sizeof(SWinKey)); + doDeleteWindow(pOperator, winRes.ts, winRes.groupId); + STimeWindow nextWin = getFinalTimeWindow(winRes.ts, pInterval); + SPullWindowInfo pull = {.window = nextWin, + .groupId = winRes.groupId, + .calWin.skey = nextWin.skey, + .calWin.ekey = nextWin.skey}; + // add pull data request + if (savePullWindow(&pull, pPullWins) == TSDB_CODE_SUCCESS) { + addPullWindow(pMap, &winRes, numOfCh); + qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh); + } + } } } } @@ -2171,7 +2261,7 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SInterval* pInterval) } } -static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo) { +static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, int32_t childId) { int32_t size = taosArrayGetSize(wins); for (int32_t i = 0; i < size; i++) { SWinKey* winKey = taosArrayGet(wins, i); @@ -2188,6 +2278,14 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo) { addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild); qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild); } + } else { + SArray* chArray = *(void**)chIds; + int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ); + qDebug("===stream===check final retrive %" PRId64",chid:%d", winKey->ts, index); + if (index == -1) { + qDebug("===stream===add final retrive %" PRId64, winKey->ts); + taosHashPut(pInfo->pFinalPullDataMap, winKey, sizeof(SWinKey), NULL, 0); + } } } } @@ -2198,13 +2296,15 @@ static void clearFunctionContext(SExprSupp* pSup) { } } -int32_t getOutputBuf(SStreamState* pState, SRowBuffPos* pPos, SResultRow** pResult) { - return streamStateGetByPos(pState, pPos, (void**)pResult); +int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SStateStore* pStore) { + return pStore->streamStateGetByPos(pState, pPos, (void**)pResult); } -int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, +int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SExprInfo* pExprInfo = pSup->pExprInfo; int32_t numOfExprs = pSup->numOfExprs; int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; @@ -2215,7 +2315,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { SRowBuffPos* pPos = *(SRowBuffPos**)taosArrayGet(pGroupResInfo->pRows, i); SResultRow* pRow = NULL; - int32_t code = getOutputBuf(pState, pPos, &pRow); + int32_t code = getOutputBuf(pState, pPos, &pRow, &pAPI->stateStore); uint64_t groupId = ((SWinKey*)pPos->pKey)->groupId; ASSERT(code == 0); doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); @@ -2227,12 +2327,12 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat if (pBlock->info.id.groupId == 0) { pBlock->info.id.groupId = groupId; void* tbname = NULL; - if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { + if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); } - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } else { // current value belongs to different group, it can't be packed into one datablock if (pBlock->info.id.groupId != groupId) { @@ -2279,7 +2379,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat return TSDB_CODE_SUCCESS; } -void doBuildStreamIntervalResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, +void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // set output datablock version @@ -2339,7 +2439,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p } while (1) { bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); - if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { + if ((pInfo->ignoreExpiredData && isClosed && !IS_FINAL_OP(pInfo)) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); if (startPos < 0) { break; @@ -2354,7 +2454,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p .groupId = groupId, }; void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey)); - if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && isClosed && !chIds) { + if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->statestore) && isClosed && !chIds) { SPullWindowInfo pull = { .window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request @@ -2385,7 +2485,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p } int32_t code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput, - pSup->rowEntryInfoOffset, &pInfo->aggSup); + pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->statestore); pResult = (SResultRow*)pResPos->pRowBuff; if (code != TSDB_CODE_SUCCESS || pResult == NULL) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); @@ -2409,7 +2509,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES); } - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pSDataBlock->info.rows, numOfOutput); key.ts = nextWin.skey; @@ -2464,9 +2564,20 @@ static inline int winPosCmprImpl(const void* pKey1, const void* pKey2) { return 0; } +static void resetUnCloseWinInfo(SSHashObj* winMap) { + void* pIte = NULL; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(winMap, pIte, &iter)) != NULL) { + SRowBuffPos* pPos = *(SRowBuffPos**)pIte; + pPos->beUsed = true; + } +} + static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SOperatorInfo* downstream = pOperator->pDownstream[0]; SExprSupp* pSup = &pOperator->exprSupp; @@ -2495,6 +2606,11 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->binfo.pRes; } + if (pInfo->recvGetAll) { + pInfo->recvGetAll = false; + resetUnCloseWinInfo(pInfo->aggSup.pResultRowHashTable); + } + setOperatorCompleted(pOperator); if (!IS_FINAL_OP(pInfo)) { clearFunctionContext(&pOperator->exprSupp); @@ -2505,8 +2621,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } else { if (pInfo->twAggSup.maxTs > 0 && pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) { - streamStateCommit(pInfo->pState); - streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark); + pAPI->stateStore.streamStateCommit(pInfo->pState); + pAPI->stateStore.streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark); pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs; } qDebug("===stream===interval final close"); @@ -2563,7 +2679,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap); if (IS_FINAL_OP(pInfo)) { - addRetriveWindow(delWins, pInfo); + int32_t chId = getChildIndex(pBlock); + addRetriveWindow(delWins, pInfo, chId); if (pBlock->info.type != STREAM_CLEAR) { taosArrayAddAll(pInfo->pDelWins, delWins); } @@ -2588,6 +2705,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { break; } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) { + pInfo->recvGetAll = true; getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { @@ -2597,7 +2715,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } continue; } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) { - processPullOver(pBlock, pInfo->pPullDataMap, &pInfo->interval); + processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins, pInfo->numOfChild, pOperator); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { return pBlock; @@ -2674,6 +2792,62 @@ TSKEY compareTs(void* pKey) { return pWinKey->ts; } +int32_t getSelectivityBufSize(SqlFunctionCtx* pCtx) { + if (pCtx->subsidiaries.rowLen == 0) { + int32_t rowLen = 0; + for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { + SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; + rowLen += pc->pExpr->base.resSchema.bytes; + } + + return rowLen + pCtx->subsidiaries.num * sizeof(bool); + } else { + return pCtx->subsidiaries.rowLen; + } +} + +int32_t getMaxFunResSize(SExprSupp* pSup, int32_t numOfCols) { + int32_t size = 0; + for (int32_t i = 0; i < numOfCols; ++i) { + int32_t resSize = getSelectivityBufSize(pSup->pCtx + i); + size = TMAX(size, resSize); + } + return size; +} + +void streamIntervalReleaseState(SOperatorInfo* pOperator) { + if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { + SStreamIntervalOperatorInfo* pInfo = pOperator->info; + int32_t resSize = sizeof(TSKEY); + pInfo->statestore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, strlen(STREAM_INTERVAL_OP_STATE_NAME), &pInfo->twAggSup.maxTs, resSize); + } + SStreamIntervalOperatorInfo* pInfo = pOperator->info; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + pAPI->stateStore.streamStateCommit(pInfo->pState); + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.releaseStreamStateFn) { + downstream->fpSet.releaseStreamStateFn(downstream); + } +} + +void streamIntervalReloadState(SOperatorInfo* pOperator) { + if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { + SStreamIntervalOperatorInfo* pInfo = pOperator->info; + int32_t size = 0; + void* pBuf = NULL; + int32_t code = pInfo->statestore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, + strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size); + TSKEY ts = *(TSKEY*)pBuf; + taosMemoryFree(pBuf); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); + pInfo->statestore.streamStateReloadInfo(pInfo->pState, ts); + } + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.reloadStreamStateFn) { + downstream->fpSet.reloadStreamStateFn(downstream); + } +} + SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; @@ -2684,6 +2858,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, } pOperator->pTaskInfo = pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, .intervalUnit = pIntervalPhyNode->intervalUnit, @@ -2709,7 +2885,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, if (pIntervalPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); - int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2720,21 +2896,20 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); initBasicInfo(&pInfo->binfo, pResBlock); + pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); + *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); + + pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1); int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pInfo->pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - - pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); - *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); - streamStateSetNumber(pInfo->pState, -1); - initResultRowInfo(&pInfo->binfo.resultRowInfo); + pInfo->numOfChild = numOfChild; pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode); @@ -2751,10 +2926,12 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, if (!IS_FINAL_OP(pInfo) || numOfChild == 0) { pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; } + pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo)); pInfo->pullIndex = 0; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); + pInfo->pFinalPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE); pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->ignoreExpiredDataSaved = false; @@ -2766,9 +2943,12 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->numOfDatapack = 0; pInfo->pUpdated = NULL; pInfo->pUpdatedMap = NULL; - pInfo->pState->pFileState = streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, - compareTs, pInfo->pState, pInfo->twAggSup.deleteMark); + int32_t funResSize= getMaxFunResSize(&pOperator->exprSupp, numOfCols); + pInfo->pState->pFileState = pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, + compareTs, pInfo->pState, pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo)); pInfo->dataVersion = 0; + pInfo->statestore = pTaskInfo->storageAPI.stateStore; + pInfo->recvGetAll = false; pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -2777,6 +2957,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { initIntervalDownStream(downstream, pPhyNode->type, pInfo); } @@ -2815,19 +2996,22 @@ void destroyStreamSessionAggOperatorInfo(void* param) { } taosArrayDestroy(pInfo->pChildren); } + colDataDestroy(&pInfo->twAggSup.timeWindowData); blockDataDestroy(pInfo->pDelRes); blockDataDestroy(pInfo->pWinBlock); blockDataDestroy(pInfo->pUpdateRes); + tSimpleHashCleanup(pInfo->pStUpdated); tSimpleHashCleanup(pInfo->pStDeleted); + taosArrayDestroy(pInfo->historyWins); taosMemoryFreeClear(param); } int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock) { + SSDataBlock* pResultBlock, SFunctionStateStore* pStore) { initBasicInfo(pBasicInfo, pResultBlock); - int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); + int32_t code = initExprSupp(pSup, pExprInfo, numOfCols, pStore); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2863,14 +3047,14 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->pState = pAggSup->pState; - if ((!pScanInfo->igCheckUpdate || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) && !pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark); + if (!pScanInfo->pUpdateInfo) { + pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark); } pScanInfo->twAggSup = *pTwSup; } int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, int64_t gap, - SStreamState* pState, int32_t keySize, int16_t keyType) { + SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore, SReadHandle* pHandle, SStorageAPI* pApi) { pSup->resultRowSize = keySize + getResultRowSize(pCtx, numOfOutput); pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR); pSup->gap = gap; @@ -2881,10 +3065,12 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, return TSDB_CODE_OUT_OF_MEMORY; } + pSup->stateStore = *pStore; + initDummyFunction(pSup->pDummyCtx, pCtx, numOfOutput); pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState)); *(pSup->pState) = *pState; - streamStateSetNumber(pSup->pState, -1); + pSup->stateStore.streamStateSetNumber(pSup->pState, -1); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pSup->pResultRows = tSimpleHashInit(32, hashFn); @@ -2898,16 +3084,20 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, if (bufSize <= pageSize) { bufSize = pageSize * 4; } + if (!osTempSpaceAvailable()) { terrno = TSDB_CODE_NO_DISKSPACE; qError("Init stream agg supporter failed since %s, tempDir:%s", terrstr(), tsTempDir); return terrno; } + int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, "function", tsTempDir); for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].saveHandle.pBuf = pSup->pResultBuf; } + pSup->pSessionAPI = pApi; + return TSDB_CODE_SUCCESS; } @@ -2927,7 +3117,7 @@ void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT pKey->win.skey = startTs; pKey->win.ekey = endTs; pKey->groupId = groupId; - int32_t code = streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey); + int32_t code = pAggSup->stateStore.streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pKey); } @@ -2935,16 +3125,30 @@ void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT bool isInvalidSessionWin(SResultWindowInfo* pWinInfo) { return pWinInfo->sessionWin.win.skey == 0; } +bool inWinRange(STimeWindow* range, STimeWindow* cur) { + if (cur->skey >= range->skey && cur->ekey <= range->ekey) { + return true; + } + return false; +} + void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, SResultWindowInfo* pCurWin) { pCurWin->sessionWin.groupId = groupId; pCurWin->sessionWin.win.skey = startTs; pCurWin->sessionWin.win.ekey = endTs; int32_t size = pAggSup->resultRowSize; - int32_t code = - streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap, &pCurWin->pOutputBuf, &size); + int32_t code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, + pAggSup->gap, &pCurWin->pOutputBuf, &size); + if (code == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) { + code = TSDB_CODE_FAILED; + releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)pCurWin->pOutputBuf, &pAggSup->pSessionAPI->stateStore); + pCurWin->pOutputBuf = taosMemoryCalloc(1, size); + } + if (code == TSDB_CODE_SUCCESS) { pCurWin->isOutput = true; + pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->sessionWin); } else { pCurWin->sessionWin.win.skey = startTs; pCurWin->sessionWin.win.ekey = endTs; @@ -2953,11 +3157,12 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) { int32_t size = 0; - int32_t code = streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size); if (code != TSDB_CODE_SUCCESS) { return code; } - streamStateCurNext(pAggSup->pState, pCur); + + pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur); return TSDB_CODE_SUCCESS; } void saveDeleteInfo(SArray* pWins, SSessionKey key) { @@ -3029,20 +3234,20 @@ static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pR static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput, - SOperatorInfo* pOperator) { + SOperatorInfo* pOperator, int64_t winDelta) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset); if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, false); + updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, winDelta); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput); return TSDB_CODE_SUCCESS; } static bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) { - streamStateSessionDel(pAggSup->pState, pKey); + pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey); SSessionKey hashKey = {0}; getSessionHashKey(pKey, &hashKey); tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey)); @@ -3060,12 +3265,12 @@ static int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin) { - SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin); + SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin); pNextWin->isOutput = true; setSessionWinOutputInfo(pStUpdated, pNextWin); int32_t size = 0; pNextWin->sessionWin = pCurWin->sessionWin; - int32_t code = streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size); + int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size); if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pNextWin->pOutputBuf); SET_SESSION_WIN_INVALID(*pNextWin); @@ -3073,10 +3278,13 @@ SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* return pCur; } -static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated, - SSHashObj* pStDeleted) { - SExprSupp* pSup = &pOperator->exprSupp; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; +static int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated, + SSHashObj* pStDeleted, bool addGap) { + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + int32_t winNum = 0; + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SResultRow* pCurResult = NULL; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; @@ -3086,15 +3294,20 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC while (1) { SResultWindowInfo winInfo = {0}; SStreamStateCur* pCur = getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo); - if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap)) { + if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap) || + !inWinRange(&pAggSup->winRange, &winInfo.sessionWin.win)) { taosMemoryFree(winInfo.pOutputBuf); - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); break; } SResultRow* pWinResult = NULL; initSessionOutputBuf(&winInfo, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset); pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, winInfo.sessionWin.win.ekey); - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->sessionWin.win, true); + int64_t winDelta = 0; + if (addGap) { + winDelta = pAggSup->gap; + } + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->sessionWin.win, winDelta); compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); tSimpleHashRemove(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey)); if (winInfo.isOutput && pStDeleted) { @@ -3102,18 +3315,21 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC } removeSessionResult(pStUpdated, pAggSup->pResultRows, winInfo.sessionWin); doDeleteSessionWindow(pAggSup, &winInfo.sessionWin); - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); taosMemoryFree(winInfo.pOutputBuf); + winNum++; } + return winNum; } int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) { - saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize); + saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize, &pAggSup->stateStore); + pWinInfo->pOutputBuf = NULL; return TSDB_CODE_SUCCESS; } static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated, - SSHashObj* pStDeleted, bool hasEndTs) { + SSHashObj* pStDeleted, bool hasEndTs, bool addGap) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; @@ -3122,8 +3338,13 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData SResultRow* pResult = NULL; int32_t rows = pSDataBlock->info.rows; int32_t winRows = 0; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); + pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; + if (pAggSup->winRange.ekey <= 0) { + pAggSup->winRange.ekey = INT64_MAX; + } SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; @@ -3135,7 +3356,6 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData } TSKEY* endTsCols = (int64_t*)pEndTsCol->pData; - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; for (int32_t i = 0; i < rows;) { if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) { i++; @@ -3151,12 +3371,16 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } + int64_t winDelta = 0; + if (addGap) { + winDelta = pAggSup->gap; + } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput, - pOperator); + pOperator, winDelta); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } - compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted); + compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted, addGap); saveSessionOutputBuf(pAggSup, &winInfo); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { @@ -3228,6 +3452,8 @@ static int32_t copyUpdateResult(SSHashObj* pStUpdated, SArray* pUpdated) { } void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) { + SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; + blockDataCleanup(pBlock); int32_t size = tSimpleHashGetSize(pStDeleted); if (size == 0) { @@ -3256,14 +3482,14 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); void* tbname = NULL; - streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname); + pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname); if (tbname == NULL) { colDataSetNULL(pTableCol, pBlock->info.rows); } else { char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); - streamFreeVal(tbname); + pAPI->stateStore.streamStateFreeVal(tbname); } pBlock->info.rows += 1; } @@ -3275,6 +3501,8 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + int32_t size = taosArrayGetSize(pWinArray); SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; @@ -3291,15 +3519,19 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS SStreamAggSupporter* pChAggSup = &pChInfo->streamAggSup; SSessionKey chWinKey = {0}; getSessionHashKey(pWinKey, &chWinKey); - SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey); + SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey); SResultRow* pResult = NULL; SResultRow* pChResult = NULL; while (1) { SResultWindowInfo childWin = {0}; childWin.sessionWin = *pWinKey; int32_t code = getSessionWinBuf(pChAggSup, pCur, &childWin); - if (code == TSDB_CODE_SUCCESS && pWinKey->win.skey <= childWin.sessionWin.win.skey && - childWin.sessionWin.win.ekey <= pWinKey->win.ekey) { + + if (code == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &childWin.sessionWin.win)) { + continue; + } + + if (code == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) { if (num == 0) { setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin); code = initSessionOutputBuf(&parentWin, &pResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); @@ -3308,17 +3540,17 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS } } num++; - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin.sessionWin.win, true); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin.sessionWin.win, pAggSup->gap); initSessionOutputBuf(&childWin, &pChResult, pChild->exprSupp.pCtx, numOfOutput, pChild->exprSupp.rowEntryInfoOffset); compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); - compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL); + compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL, true); saveResult(parentWin, pStUpdated); } else { break; } } - streamStateFreeCur(pCur); + pAPI->stateStore.streamStateFreeCur(pCur); } if (num > 0) { saveSessionOutputBuf(pAggSup, &parentWin); @@ -3383,7 +3615,7 @@ void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayL pGroupResInfo->pBuf = NULL; } -void doBuildSessionResult(SOperatorInfo* pOperator, SStreamState* pState, SGroupResInfo* pGroupResInfo, +void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // set output datablock version @@ -3398,6 +3630,29 @@ void doBuildSessionResult(SOperatorInfo* pOperator, SStreamState* pState, SGroup // clear the existed group id pBlock->info.id.groupId = 0; buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo); + if (pBlock->info.rows == 0) { + cleanupGroupResInfo(pGroupResInfo); + } +} +void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins) { + int32_t size = taosArrayGetSize(pAllWins); + if (size == 0) { + return; + } + + SSessionKey* pSeKey = taosArrayGet(pAllWins, size - 1); + taosArrayPush(pMaxWins, pSeKey); + if (pSeKey->groupId == 0) { + return; + } + uint64_t preGpId = pSeKey->groupId; + for (int32_t i = size - 2; i >= 0; i--) { + pSeKey = taosArrayGet(pAllWins, i); + if (preGpId != pSeKey->groupId) { + taosArrayPush(pMaxWins, pSeKey); + preGpId = pSeKey->groupId; + } + } } static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { @@ -3405,6 +3660,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + qDebug("===stream=== stream session agg"); if (pOperator->status == OP_EXEC_DONE) { return NULL; } else if (pOperator->status == OP_RES_TO_RETURN) { @@ -3470,14 +3726,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo)); + doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo), true); if (IS_FINAL_OP(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); // if chIndex + 1 - size > 0, add new child for (int32_t i = 0; i < chIndex + 1 - size; i++) { SOperatorInfo* pChildOp = - createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0); + createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0, NULL); if (!pChildOp) { T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } @@ -3485,7 +3741,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true); + doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true, false); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark); @@ -3499,6 +3755,9 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { removeSessionResults(pInfo->pStDeleted, pInfo->pUpdated); tSimpleHashCleanup(pInfo->pStUpdated); pInfo->pStUpdated = NULL; + if(pInfo->isHistoryOp) { + getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + } initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); @@ -3525,8 +3784,69 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return NULL; } +void streamSessionReleaseState(SOperatorInfo* pOperator) { + if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION) { + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; + int32_t resSize = taosArrayGetSize(pInfo->historyWins) * sizeof(SSessionKey); + pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_STATE_NAME, strlen(STREAM_SESSION_OP_STATE_NAME), pInfo->historyWins->pData, resSize); + } + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.releaseStreamStateFn) { + downstream->fpSet.releaseStreamStateFn(downstream); + } +} + +void resetWinRange(STimeWindow* winRange) { + winRange->skey = INT64_MIN; + winRange->ekey = INT64_MAX; +} + +void streamSessionReloadState(SOperatorInfo* pOperator) { + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + resetWinRange(&pAggSup->winRange); + + SResultWindowInfo winInfo = {0}; + int32_t size = 0; + void* pBuf = NULL; + int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + int32_t num = size / sizeof(SSessionKey); + SSessionKey* pSeKeyBuf = (SSessionKey*) pBuf; + ASSERT(size == num * sizeof(SSessionKey)); + if (!pInfo->pStUpdated && num > 0) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pStUpdated = tSimpleHashInit(64, hashFn); + } + for (int32_t i = 0; i < num; i++) { + SResultWindowInfo winInfo = {0}; + setSessionOutputBuf(pAggSup, pSeKeyBuf[i].win.skey, pSeKeyBuf[i].win.ekey, pSeKeyBuf[i].groupId, &winInfo); + int32_t winNum = compactSessionWindow(pOperator, &winInfo, pInfo->pStUpdated, pInfo->pStDeleted, true); + if (winNum > 0) { + qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, winInfo.sessionWin.win.skey, winInfo.sessionWin.groupId); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { + saveResult(winInfo, pInfo->pStUpdated); + } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + if (!isCloseWindow(&winInfo.sessionWin.win, &pInfo->twAggSup)) { + saveDeleteRes(pInfo->pStDeleted, winInfo.sessionWin); + } + SSessionKey key = {0}; + getSessionHashKey(&winInfo.sessionWin, &key); + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); + } + } + saveSessionOutputBuf(pAggSup, &winInfo); + } + taosMemoryFree(pBuf); + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.reloadStreamStateFn) { + downstream->fpSet.reloadStreamStateFn(downstream); + } +} + SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo) { + SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; int32_t numOfCols = 0; int32_t code = TSDB_CODE_OUT_OF_MEMORY; @@ -3542,7 +3862,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh if (pSessionNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -3551,13 +3871,13 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); - code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); + code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, pSessionNode->gap, - pTaskInfo->streamInfo.pState, 0, 0); + pTaskInfo->streamInfo.pState, 0, 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -3589,11 +3909,19 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->pUpdated = NULL; pInfo->pStUpdated = NULL; pInfo->dataVersion = 0; + pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); + if (!pInfo->historyWins) { + goto _error; + } + if (pHandle) { + pInfo->isHistoryOp = pHandle->fillHistory; + } setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); if (downstream) { initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); @@ -3613,7 +3941,7 @@ _error: static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) { tSimpleHashClear(pInfo->streamAggSup.pResultRows); - streamStateSessionClear(pInfo->streamAggSup.pState); + pInfo->streamAggSup.stateStore.streamStateSessionClear(pInfo->streamAggSup.pState); } static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { @@ -3623,6 +3951,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + qDebug("===stream=== stream session semi agg"); if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -3690,7 +4019,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, NULL, false); + doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, NULL, false, false); maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); } @@ -3701,7 +4030,6 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { removeSessionResults(pInfo->pStDeleted, pInfo->pUpdated); tSimpleHashCleanup(pInfo->pStUpdated); pInfo->pStUpdated = NULL; - initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); @@ -3732,13 +4060,14 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo, int32_t numOfChild) { + SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle) { int32_t code = TSDB_CODE_OUT_OF_MEMORY; - SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo); + SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle); if (pOperator == NULL) { goto _error; } + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; pInfo->isFinal = (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION); @@ -3750,20 +4079,20 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAgg, NULL, destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL); } - + setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); setOperatorInfo(pOperator, name, pPhyNode->type, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->operatorType = pPhyNode->type; if (numOfChild > 0) { pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*)); for (int32_t i = 0; i < numOfChild; i++) { - SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0); + SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0, NULL); if (pChildOp == NULL) { goto _error; } SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info; pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; - streamStateSetNumber(pChInfo->streamAggSup.pState, i); + pAPI->stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i); taosArrayPush(pInfo->pChildren, &pChildOp); } } @@ -3798,6 +4127,8 @@ void destroyStreamStateOperatorInfo(void* param) { } colDataDestroy(&pInfo->twAggSup.timeWindowData); blockDataDestroy(pInfo->pDelRes); + taosArrayDestroy(pInfo->historyWins); + tSimpleHashCleanup(pInfo->pSeUpdated); tSimpleHashCleanup(pInfo->pSeDeleted); taosMemoryFreeClear(param); } @@ -3814,11 +4145,21 @@ bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) { } bool compareStateKey(void* data, void* key) { + if (!data || !key) { + return true; + } SStateKeys* stateKey = (SStateKeys*)key; stateKey->pData = (char*)key + sizeof(SStateKeys); return compareVal(data, stateKey); } +bool compareWinStateKey(SStateKeys* left, SStateKeys* right) { + if (!left || !right) { + return false; + } + return compareVal(left->pData, right); +} + void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData, SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin) { int32_t size = pAggSup->resultRowSize; @@ -3826,7 +4167,7 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pCurWin->winInfo.sessionWin.win.skey = ts; pCurWin->winInfo.sessionWin.win.ekey = ts; int32_t code = - streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize, + pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize, compareStateKey, &pCurWin->winInfo.pOutputBuf, &size); pCurWin->pStateKey = (SStateKeys*)((char*)pCurWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize)); @@ -3835,9 +4176,26 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys); pCurWin->pStateKey->isNull = false; + if (code == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->winInfo.sessionWin.win)) { + code = TSDB_CODE_FAILED; + releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)pCurWin->winInfo.pOutputBuf, &pAggSup->pSessionAPI->stateStore); + pCurWin->winInfo.pOutputBuf = taosMemoryCalloc(1, size); + pCurWin->pStateKey = + (SStateKeys*)((char*)pCurWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize)); + pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys); + pCurWin->pStateKey->type = pAggSup->stateKeyType; + pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys); + pCurWin->pStateKey->isNull = false; + pCurWin->winInfo.sessionWin.groupId = groupId; + pCurWin->winInfo.sessionWin.win.skey = ts; + pCurWin->winInfo.sessionWin.win.ekey = ts; + qDebug("===stream===reset state win key. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey, pCurWin->winInfo.sessionWin.win.ekey); + } + if (code == TSDB_CODE_SUCCESS) { pCurWin->winInfo.isOutput = true; - } else { + pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin); + } else if (pKeyData) { if (IS_VAR_DATA_TYPE(pAggSup->stateKeyType)) { varDataCopy(pCurWin->pStateKey->pData, pKeyData); } else { @@ -3846,13 +4204,21 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, } pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin; - pNextWin->winInfo.pOutputBuf = NULL; - SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); - code = streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0); + SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin); + int32_t nextSize = pAggSup->resultRowSize; + code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, &pNextWin->winInfo.pOutputBuf, &nextSize); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_INVALID(pNextWin->winInfo); + } else { + pNextWin->pStateKey = + (SStateKeys*)((char*)pNextWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize)); + pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys); + pNextWin->pStateKey->type = pAggSup->stateKeyType; + pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys); + pNextWin->pStateKey->isNull = false; + pNextWin->winInfo.isOutput = true; } - streamStateFreeCur(pCur); + pAggSup->stateStore.streamStateFreeCur(pCur); } int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, TSKEY* pTs, uint64_t groupId, @@ -3891,7 +4257,9 @@ int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNex static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated, SSHashObj* pStDeleted) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamStateAggOperatorInfo* pInfo = pOperator->info; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; uint64_t groupId = pSDataBlock->info.id.groupId; @@ -3899,8 +4267,13 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl TSKEY* tsCols = NULL; SResultRow* pResult = NULL; int32_t winRows = 0; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); + pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; + if (pAggSup->winRange.ekey <= 0) { + pAggSup->winRange.ekey = INT64_MAX; + } if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); @@ -3909,7 +4282,6 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl return; } - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; int32_t rows = pSDataBlock->info.rows; blockDataEnsureCapacity(pAggSup->pScanBlock, rows); SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); @@ -3924,6 +4296,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl SStateWindowInfo curWin = {0}; SStateWindowInfo nextWin = {0}; setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin); + if (IS_VALID_SESSION_WIN(nextWin.winInfo)) { + releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)nextWin.winInfo.pOutputBuf, &pAPI->stateStore); + } setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); winRows = updateStateWindowInfo(&curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual, pAggSup->pResultRows, pSeUpdated, pStDeleted); @@ -3933,11 +4308,11 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin); - releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf); + releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf, &pAPI->stateStore); continue; } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, - pOperator); + pOperator, 0); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } @@ -3966,6 +4341,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SStreamStateAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; + qDebug("===stream=== stream state agg"); if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { @@ -4033,6 +4409,10 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { tSimpleHashCleanup(pInfo->pSeUpdated); pInfo->pSeUpdated = NULL; + if(pInfo->isHistoryOp) { + getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + } + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); @@ -4058,11 +4438,108 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return NULL; } +void streamStateReleaseState(SOperatorInfo* pOperator) { + SStreamStateAggOperatorInfo* pInfo = pOperator->info; + int32_t resSize = taosArrayGetSize(pInfo->historyWins) * sizeof(SSessionKey); + qDebug("===stream=== relase state. save result count:%d", (int32_t)taosArrayGetSize(pInfo->historyWins)); + pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_STATE_NAME, strlen(STREAM_STATE_OP_STATE_NAME), pInfo->historyWins->pData, resSize); + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.releaseStreamStateFn) { + downstream->fpSet.releaseStreamStateFn(downstream); + } +} + +static void compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, + SSHashObj* pStUpdated, SSHashObj* pStDeleted) { + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + + SStreamStateAggOperatorInfo* pInfo = pOperator->info; + SResultRow* pCurResult = NULL; + int32_t numOfOutput = pOperator->exprSupp.numOfExprs; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); + SResultRow* pWinResult = NULL; + initSessionOutputBuf(pNextWin, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset); + pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, pNextWin->sessionWin.win.ekey); + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->sessionWin.win, 1); + compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); + tSimpleHashRemove(pStUpdated, &pNextWin->sessionWin, sizeof(SSessionKey)); + if (pNextWin->isOutput && pStDeleted) { + qDebug("===stream=== save delete window info %" PRId64 ", %" PRIu64, pNextWin->sessionWin.win.skey, pNextWin->sessionWin.groupId); + saveDeleteRes(pStDeleted, pNextWin->sessionWin); + } + removeSessionResult(pStUpdated, pAggSup->pResultRows, pNextWin->sessionWin); + doDeleteSessionWindow(pAggSup, &pNextWin->sessionWin); + taosMemoryFree(pNextWin->pOutputBuf); +} + +void streamStateReloadState(SOperatorInfo* pOperator) { + SStreamStateAggOperatorInfo* pInfo = pOperator->info; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + resetWinRange(&pAggSup->winRange); + + SSessionKey seKey = {.win.skey = INT64_MIN, .win.ekey = INT64_MIN, .groupId = 0}; + int32_t size = 0; + void* pBuf = NULL; + int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME, + strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); + int32_t num = size / sizeof(SSessionKey); + qDebug("===stream=== reload state. get result count:%d", num); + SSessionKey* pSeKeyBuf = (SSessionKey*) pBuf; + ASSERT(size == num * sizeof(SSessionKey)); + if (!pInfo->pSeUpdated && num > 0) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pSeUpdated = tSimpleHashInit(64, hashFn); + } + if (!pInfo->pSeDeleted && num > 0) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pSeDeleted = tSimpleHashInit(64, hashFn); + } + for (int32_t i = 0; i < num; i++) { + SStateWindowInfo curInfo = {0}; + SStateWindowInfo nextInfo = {0}; + SStateWindowInfo dummy = {0}; + qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, pSeKeyBuf[i].groupId, i); + setStateOutputBuf(pAggSup, pSeKeyBuf[i].win.skey, pSeKeyBuf[i].groupId, NULL, &curInfo, &nextInfo); + bool cpRes = compareWinStateKey(curInfo.pStateKey,nextInfo.pStateKey); + qDebug("===stream=== reload state. next window info %" PRId64 ", %" PRIu64 ", compare:%d", nextInfo.winInfo.sessionWin.win.skey, nextInfo.winInfo.sessionWin.groupId, cpRes); + if (cpRes) { + compactStateWindow(pOperator, &curInfo.winInfo, &nextInfo.winInfo, pInfo->pSeUpdated, pInfo->pSeDeleted); + qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, curInfo.winInfo.sessionWin.groupId); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { + saveResult(curInfo.winInfo, pInfo->pSeUpdated); + } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) { + saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin); + } + SSessionKey key = {0}; + getSessionHashKey(&curInfo.winInfo.sessionWin, &key); + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo)); + } + } else if (IS_VALID_SESSION_WIN(nextInfo.winInfo)) { + releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)nextInfo.winInfo.pOutputBuf, &pAggSup->pSessionAPI->stateStore); + } + + if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { + saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + } + } + taosMemoryFree(pBuf); + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.reloadStreamStateFn) { + downstream->fpSet.reloadStreamStateFn(downstream); + } +} + SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo) { + SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) { SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode; int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; - SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; + SColumnNode* pColNode = (SColumnNode*)(pStateNode->pStateKey); int32_t code = TSDB_CODE_SUCCESS; SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo)); @@ -4077,7 +4554,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys if (pStateNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4096,14 +4573,14 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); - code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); + code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } int32_t keySize = sizeof(SStateKeys) + pColNode->node.resType.bytes; int16_t type = pColNode->node.resType.type; code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, - type); + type, &pTaskInfo->storageAPI.stateStore, pHandle, &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4119,11 +4596,19 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pUpdated = NULL; pInfo->pSeUpdated = NULL; pInfo->dataVersion = 0; + pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); + if (!pInfo->historyWins) { + goto _error; + } + if (pHandle) { + pInfo->isHistoryOp = pHandle->fillHistory; + } setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState); initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4210,7 +4695,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR continue; } - updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true); + updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, 1); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos, pBlock->info.rows, pSup->numOfExprs); @@ -4230,7 +4715,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR miaInfo->curTs = currWin.skey; } - updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true); + updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, 1); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos, pBlock->info.rows, pSup->numOfExprs); } @@ -4251,7 +4736,6 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { SSDataBlock* pRes = pIaInfo->binfo.pRes; SResultRowInfo* pResultRowInfo = &pIaInfo->binfo.resultRowInfo; SOperatorInfo* downstream = pOperator->pDownstream[0]; - int32_t scanFlag = MAIN_SCAN; while (1) { SSDataBlock* pBlock = NULL; @@ -4271,6 +4755,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { finalizeResultRows(pIaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pRes, pTaskInfo); resetResultRow(pMiaInfo->pResultRow, pIaInfo->aggSup.resultRowSize - sizeof(SResultRow)); cleanupAfterGroupResultGen(pMiaInfo, pRes); + doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL); } setOperatorCompleted(pOperator); @@ -4291,6 +4776,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { pMiaInfo->prefetchedBlock = pBlock; cleanupAfterGroupResultGen(pMiaInfo, pRes); + doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL); break; } else { // continue @@ -4298,8 +4784,8 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { } } - getTableScanInfo(pOperator, &pIaInfo->inputOrder, &scanFlag, false); - setInputDataBlock(pSup, pBlock, pIaInfo->inputOrder, scanFlag, true); + pRes->info.scanFlag = pBlock->info.scanFlag; + setInputDataBlock(pSup, pBlock, pIaInfo->binfo.inputTsOrder, pBlock->info.scanFlag, true); doMergeAlignedIntervalAggImpl(pOperator, &pIaInfo->binfo.resultRowInfo, pBlock, pRes); doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL); @@ -4372,7 +4858,8 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, miaInfo->curTs = INT64_MIN; iaInfo->win = pTaskInfo->window; - iaInfo->inputOrder = TSDB_ORDER_ASC; + iaInfo->binfo.inputTsOrder = pNode->window.node.inputTsOrder; + iaInfo->binfo.outputTsOrder = pNode->window.node.outputTsOrder; iaInfo->interval = interval; iaInfo->primaryTsIndex = ((SColumnNode*)pNode->window.pTspk)->slotId; iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock; @@ -4384,7 +4871,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num); code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4449,7 +4936,7 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t STimeWindow* newWin) { SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; - bool ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC); + bool ascScan = (iaInfo->binfo.inputTsOrder == TSDB_ORDER_ASC); SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin}; tdListAppend(miaInfo->groupIntervals, &groupTimeWindow); @@ -4484,12 +4971,12 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* int32_t numOfOutput = pExprSup->numOfExprs; int64_t* tsCols = extractTsCol(pBlock, iaInfo); uint64_t tableGroupId = pBlock->info.id.groupId; - bool ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC); + bool ascScan = (iaInfo->binfo.inputTsOrder == TSDB_ORDER_ASC); TSKEY blockStartTs = getStartTsKey(&pBlock->info.window, tsCols); SResultRow* pResult = NULL; STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval, - iaInfo->inputOrder); + iaInfo->binfo.inputTsOrder); int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, @@ -4500,7 +4987,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* TSKEY ekey = ascScan ? win.ekey : win.skey; int32_t forwardRows = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder); + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->binfo.inputTsOrder); ASSERT(forwardRows > 0); // prev time window not interpolation yet. @@ -4519,7 +5006,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* doWindowBorderInterpolation(iaInfo, pBlock, pResult, &win, startPos, forwardRows, pExprSup); } - updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true); + updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, 1); applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, numOfOutput); doCloseWindow(pResultRowInfo, iaInfo, pResult); @@ -4531,7 +5018,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* while (1) { int32_t prevEndPos = forwardRows - 1 + startPos; startPos = - getNextQualifiedWindow(&iaInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, iaInfo->inputOrder); + getNextQualifiedWindow(&iaInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, iaInfo->binfo.inputTsOrder); if (startPos < 0) { break; } @@ -4546,12 +5033,12 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* ekey = ascScan ? nextWin.ekey : nextWin.skey; forwardRows = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->inputOrder); + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->binfo.inputTsOrder); // window start(end) key interpolation doWindowBorderInterpolation(iaInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pExprSup); - updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true); + updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, 1); applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, numOfOutput); doCloseWindow(pResultRowInfo, iaInfo, pResult); @@ -4582,7 +5069,6 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { if (!miaInfo->inputBlocksFinished) { SOperatorInfo* downstream = pOperator->pDownstream[0]; - int32_t scanFlag = MAIN_SCAN; while (1) { SSDataBlock* pBlock = NULL; if (miaInfo->prefetchedBlock == NULL) { @@ -4607,9 +5093,9 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { break; } - getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag, false); - setInputDataBlock(pExpSupp, pBlock, iaInfo->inputOrder, scanFlag, true); - doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); + pRes->info.scanFlag = pBlock->info.scanFlag; + setInputDataBlock(pExpSupp, pBlock, iaInfo->binfo.inputTsOrder, pBlock->info.scanFlag, true); + doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, pBlock->info.scanFlag, pRes); if (pRes->info.rows >= pOperator->resultInfo.threshold) { break; @@ -4659,10 +5145,11 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge SIntervalAggOperatorInfo* pIntervalInfo = &pMergeIntervalInfo->intervalAggOperatorInfo; pIntervalInfo->win = pTaskInfo->window; - pIntervalInfo->inputOrder = TSDB_ORDER_ASC; + pIntervalInfo->binfo.inputTsOrder = pIntervalPhyNode->window.node.inputTsOrder; pIntervalInfo->interval = interval; pIntervalInfo->binfo.mergeResultBlock = pIntervalPhyNode->window.mergeDataBlock; pIntervalInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; + pIntervalInfo->binfo.outputTsOrder = pIntervalPhyNode->window.node.outputTsOrder; SExprSupp* pExprSupp = &pOperator->exprSupp; @@ -4670,7 +5157,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge initResultSizeInfo(&pOperator->resultInfo, 4096); int32_t code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4713,7 +5200,9 @@ _error: static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SExprSupp* pSup = &pOperator->exprSupp; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + + SExprSupp* pSup = &pOperator->exprSupp; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -4731,11 +5220,17 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { printDataBlock(pInfo->binfo.pRes, "single interval"); return pInfo->binfo.pRes; } + + if (pInfo->recvGetAll) { + pInfo->recvGetAll = false; + resetUnCloseWinInfo(pInfo->aggSup.pResultRowHashTable); + } + setOperatorCompleted(pOperator); if (pInfo->twAggSup.maxTs > 0 && pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) { - streamStateCommit(pInfo->pState); - streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark); + pAPI->stateStore.streamStateCommit(pInfo->pState); + pAPI->stateStore.streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark); setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId); pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs; } @@ -4770,6 +5265,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { continue; } else if (pBlock->info.type == STREAM_GET_ALL) { qDebug("===stream===single interval recv|block type STREAM_GET_ALL"); + pInfo->recvGetAll = true; getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { @@ -4877,6 +5373,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); pOperator->pTaskInfo = pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->ignoreExpiredDataSaved = false; pInfo->isFinal = false; @@ -4889,9 +5387,13 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); + pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); + *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); + pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1); + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); + pInfo->pState, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4899,7 +5401,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys if (pIntervalPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4912,12 +5414,9 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); initResultRowInfo(&pInfo->binfo.resultRowInfo); - pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); - *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); - streamStateSetNumber(pInfo->pState, -1); - pInfo->pPhyNode = NULL; // create new child pInfo->pPullDataMap = NULL; + pInfo->pFinalPullDataMap = NULL; pInfo->pPullWins = NULL; // SPullWindowInfo pInfo->pullIndex = 0; pInfo->pPullDataRes = NULL; @@ -4928,13 +5427,20 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->numOfDatapack = 0; pInfo->pUpdated = NULL; pInfo->pUpdatedMap = NULL; - pInfo->pState->pFileState = streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, - compareTs, pInfo->pState, pInfo->twAggSup.deleteMark); + int32_t funResSize= getMaxFunResSize(pSup, numOfCols); + + pInfo->pState->pFileState = pTaskInfo->storageAPI.stateStore.streamFileStateInit( + tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, + pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo)); setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState); + + pInfo->statestore = pTaskInfo->storageAPI.stateStore; + pInfo->recvGetAll = false; initIntervalDownStream(downstream, pPhyNode->type, pInfo); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 211f78b9814cedcc7f8e32f9babf179a072df4df..1891e93c61d458e500d46e68b43188356ed9a066 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -19,10 +19,12 @@ #include "tcompare.h" #include "tdatablock.h" #include "tdef.h" +#include "theap.h" #include "tlosertree.h" #include "tpagedbuf.h" #include "tsort.h" #include "tutil.h" +#include "tsimplehash.h" struct STupleHandle { SSDataBlock* pBlock; @@ -41,11 +43,22 @@ struct SSortHandle { int64_t startTs; uint64_t totalElapsed; + uint64_t pqMaxRows; + uint32_t pqMaxTupleLength; + uint32_t pqSortBufSize; + bool forceUsePQSort; + BoundedQueue* pBoundedQueue; + uint32_t tmpRowIdx; + + int64_t mergeLimit; + int64_t currMergeLimitTs; + int32_t sourceId; SSDataBlock* pDataBlock; SMsortComparParam cmpParam; int32_t numOfCompletedSources; bool opened; + int8_t closed; const char* idStr; bool inMemSort; bool needAdjust; @@ -56,21 +69,122 @@ struct SSortHandle { _sort_fetch_block_fn_t fetchfp; _sort_merge_compar_fn_t comparFn; SMultiwayMergeTreeInfo* pMergeTree; + + bool singleTableMerge; }; +void setSingleTableMerge(SSortHandle* pHandle) { + pHandle->singleTableMerge = true; +} + static int32_t msortComparFn(const void* pLeft, const void* pRight, void* param); +// | offset[0] | offset[1] |....| nullbitmap | data |...| +static void* createTuple(uint32_t columnNum, uint32_t tupleLen) { + uint32_t totalLen = sizeof(uint32_t) * columnNum + BitmapLen(columnNum) + tupleLen; + return taosMemoryCalloc(1, totalLen); +} +static void destoryAllocatedTuple(void* t) { taosMemoryFree(t); } + +#define tupleOffset(tuple, colIdx) ((uint32_t*)(tuple + sizeof(uint32_t) * colIdx)) +#define tupleSetOffset(tuple, colIdx, offset) (*tupleOffset(tuple, colIdx) = offset) +#define tupleSetNull(tuple, colIdx, colNum) colDataSetNull_f((char*)tuple + sizeof(uint32_t) * colNum, colIdx) +#define tupleColIsNull(tuple, colIdx, colNum) colDataIsNull_f((char*)tuple + sizeof(uint32_t) * colNum, colIdx) +#define tupleGetDataStartOffset(colNum) (sizeof(uint32_t) * colNum + BitmapLen(colNum)) +#define tupleSetData(tuple, offset, data, length) memcpy(tuple + offset, data, length) + +/** + * @param t the tuple pointer addr, if realloced, *t is changed to the new addr + * @param offset copy data into pTuple start from offset + * @param colIndex the columnIndex, for setting null bitmap + * @return the next offset to add field + * */ +static inline size_t tupleAddField(char** t, uint32_t colNum, uint32_t offset, uint32_t colIdx, void* data, size_t length, + bool isNull, uint32_t tupleLen) { + tupleSetOffset(*t, colIdx, offset); + if (isNull) { + tupleSetNull(*t, colIdx, colNum); + } else { + if (offset + length > tupleLen + tupleGetDataStartOffset(colNum)) { + *t = taosMemoryRealloc(*t, offset + length); + } + tupleSetData(*t, offset, data, length); + } + return offset + length; +} + +static void* tupleGetField(char* t, uint32_t colIdx, uint32_t colNum) { + if (tupleColIsNull(t, colIdx, colNum)) return NULL; + return t + *tupleOffset(t, colIdx); +} + SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle) { return createOneDataBlock(pSortHandle->pDataBlock, false); } +#define AllocatedTupleType 0 +#define ReferencedTupleType 1 // tuple references to one row in pDataBlock +typedef struct TupleDesc { + uint8_t type; + char* data; // if type is AllocatedTuple, then points to the created tuple, otherwise points to the DataBlock +} TupleDesc; + +typedef struct ReferencedTuple { + TupleDesc desc; + size_t rowIndex; +} ReferencedTuple; + +static TupleDesc* createAllocatedTuple(SSDataBlock* pBlock, size_t colNum, uint32_t tupleLen, size_t rowIdx) { + TupleDesc* t = taosMemoryCalloc(1, sizeof(TupleDesc)); + void* pTuple = createTuple(colNum, tupleLen); + if (!pTuple) { + taosMemoryFree(t); + return NULL; + } + size_t colLen = 0; + uint32_t offset = tupleGetDataStartOffset(colNum); + for (size_t colIdx = 0; colIdx < colNum; ++colIdx) { + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, colIdx); + if (colDataIsNull_s(pCol, rowIdx)) { + offset = tupleAddField((char**)&pTuple, colNum, offset, colIdx, 0, 0, true, tupleLen); + } else { + colLen = colDataGetRowLength(pCol, rowIdx); + offset = + tupleAddField((char**)&pTuple, colNum, offset, colIdx, colDataGetData(pCol, rowIdx), colLen, false, tupleLen); + } + } + t->type = AllocatedTupleType; + t->data = pTuple; + return t; +} + +void* tupleDescGetField(const TupleDesc* pDesc, int32_t colIdx, uint32_t colNum) { + if (pDesc->type == ReferencedTupleType) { + ReferencedTuple* pRefTuple = (ReferencedTuple*)pDesc; + SColumnInfoData* pCol = taosArrayGet(((SSDataBlock*)pDesc->data)->pDataBlock, colIdx); + if (colDataIsNull_s(pCol, pRefTuple->rowIndex)) return NULL; + return colDataGetData(pCol, pRefTuple->rowIndex); + } else { + return tupleGetField(pDesc->data, colIdx, colNum); + } +} + +void destroyTuple(void* t) { + TupleDesc* pDesc = t; + if (pDesc->type == AllocatedTupleType) { + destoryAllocatedTuple(pDesc->data); + taosMemoryFree(pDesc); + } +} + /** * * @param type * @return */ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t pageSize, int32_t numOfPages, - SSDataBlock* pBlock, const char* idstr) { + SSDataBlock* pBlock, const char* idstr, uint64_t pqMaxRows, uint32_t pqMaxTupleLength, + uint32_t pqSortBufSize) { SSortHandle* pSortHandle = taosMemoryCalloc(1, sizeof(SSortHandle)); pSortHandle->type = type; @@ -79,14 +193,29 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page pSortHandle->pSortInfo = pSortInfo; pSortHandle->loops = 0; + pSortHandle->pqMaxTupleLength = pqMaxTupleLength; + if (pqMaxRows != 0) { + pSortHandle->pqSortBufSize = pqSortBufSize; + pSortHandle->pqMaxRows = pqMaxRows; + } + pSortHandle->forceUsePQSort = false; + if (pBlock != NULL) { pSortHandle->pDataBlock = createOneDataBlock(pBlock, false); } + pSortHandle->mergeLimit = -1; + pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES); pSortHandle->cmpParam.orderInfo = pSortInfo; pSortHandle->cmpParam.cmpGroupId = false; - + pSortHandle->cmpParam.sortType = type; + if (type == SORT_BLOCK_TS_MERGE) { + SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pSortInfo, 0); + pSortHandle->cmpParam.tsSlotId = pOrder->slotId; + pSortHandle->cmpParam.order = pOrder->order; + pSortHandle->cmpParam.cmpFn = (pOrder->order == TSDB_ORDER_ASC) ? compareInt64Val : compareInt64ValDesc; + } tsortSetComparFp(pSortHandle, msortComparFn); if (idstr != NULL) { @@ -101,7 +230,11 @@ static int32_t sortComparCleanup(SMsortComparParam* cmpParam) { for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { SSortSource* pSource = cmpParam->pSources[i]; blockDataDestroy(pSource->src.pBlock); + if (pSource->pageIdList) { + taosArrayDestroy(pSource->pageIdList); + } taosMemoryFreeClear(pSource); + cmpParam->pSources[i] = NULL; } cmpParam->numOfSources = 0; @@ -123,9 +256,11 @@ void tsortClearOrderdSource(SArray* pOrderedSource, int64_t *fetchUs, int64_t *f // release pageIdList if ((*pSource)->pageIdList) { taosArrayDestroy((*pSource)->pageIdList); + (*pSource)->pageIdList = NULL; } if ((*pSource)->param && !(*pSource)->onlyRef) { taosMemoryFree((*pSource)->param); + (*pSource)->param = NULL; } if (!(*pSource)->onlyRef && (*pSource)->src.pBlock) { @@ -143,15 +278,15 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { if (pSortHandle == NULL) { return; } - tsortClose(pSortHandle); if (pSortHandle->pMergeTree != NULL) { - tMergeTreeDestroy(pSortHandle->pMergeTree); + tMergeTreeDestroy(&pSortHandle->pMergeTree); } destroyDiskbasedBuf(pSortHandle->pBuf); taosMemoryFreeClear(pSortHandle->idStr); blockDataDestroy(pSortHandle->pDataBlock); + if (pSortHandle->pBoundedQueue) destroyBoundedQueue(pSortHandle->pBoundedQueue); int64_t fetchUs = 0, fetchNum = 0; tsortClearOrderdSource(pSortHandle->pOrderedSource, &fetchUs, &fetchNum); @@ -352,11 +487,14 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT if (pHandle->type == SORT_SINGLESOURCE_SORT) { pSource->pageIndex++; if (pSource->pageIndex >= taosArrayGetSize(pSource->pageIdList)) { + qDebug("adjust merge tree. %d source completed %d", *numOfCompleted, pSource->pageIndex); (*numOfCompleted) += 1; pSource->src.rowIndex = -1; pSource->pageIndex = -1; pSource->src.pBlock = blockDataDestroy(pSource->src.pBlock); } else { + if (pSource->pageIndex % 512 == 0) qDebug("begin source %p page %d", pSource, pSource->pageIndex); + int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex); void* pPage = getBufPage(pHandle->pBuf, *pPgId); @@ -369,7 +507,6 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT if (code != TSDB_CODE_SUCCESS) { return code; } - releaseBufPage(pHandle->pBuf, pPage); } } else { @@ -380,6 +517,7 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT if (pSource->src.pBlock == NULL) { (*numOfCompleted) += 1; pSource->src.rowIndex = -1; + qDebug("adjust merge tree. %d source completed", *numOfCompleted); } } } @@ -460,53 +598,63 @@ int32_t msortComparFn(const void* pLeft, const void* pRight, void* param) { } } - for (int32_t i = 0; i < pInfo->size; ++i) { - SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); - SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId); + if (pParam->sortType == SORT_BLOCK_TS_MERGE) { + SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pParam->tsSlotId); + SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pParam->tsSlotId); + int64_t* left1 = (int64_t*)(pLeftColInfoData->pData) + pLeftSource->src.rowIndex; + int64_t* right1 = (int64_t*)(pRightColInfoData->pData) + pRightSource->src.rowIndex; - bool leftNull = false; - if (pLeftColInfoData->hasNull) { - if (pLeftBlock->pBlockAgg == NULL) { - leftNull = colDataIsNull_s(pLeftColInfoData, pLeftSource->src.rowIndex); - } else { - leftNull = - colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg[i]); + int ret = pParam->cmpFn(left1, right1); + return ret; + } else { + for (int32_t i = 0; i < pInfo->size; ++i) { + SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); + SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId); + SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->slotId); + + bool leftNull = false; + if (pLeftColInfoData->hasNull) { + if (pLeftBlock->pBlockAgg == NULL) { + leftNull = colDataIsNull_s(pLeftColInfoData, pLeftSource->src.rowIndex); + } else { + leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, + pLeftBlock->pBlockAgg[i]); + } } - } - SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->slotId); - bool rightNull = false; - if (pRightColInfoData->hasNull) { - if (pRightBlock->pBlockAgg == NULL) { - rightNull = colDataIsNull_s(pRightColInfoData, pRightSource->src.rowIndex); - } else { - rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, - pRightBlock->pBlockAgg[i]); + bool rightNull = false; + if (pRightColInfoData->hasNull) { + if (pRightBlock->pBlockAgg == NULL) { + rightNull = colDataIsNull_s(pRightColInfoData, pRightSource->src.rowIndex); + } else { + rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, + pRightBlock->pBlockAgg[i]); + } } - } - if (leftNull && rightNull) { - continue; // continue to next slot - } + if (leftNull && rightNull) { + continue; // continue to next slot + } - if (rightNull) { - return pOrder->nullFirst ? 1 : -1; - } + if (rightNull) { + return pOrder->nullFirst ? 1 : -1; + } - if (leftNull) { - return pOrder->nullFirst ? -1 : 1; - } + if (leftNull) { + return pOrder->nullFirst ? -1 : 1; + } - void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex); - void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex); + void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex); + void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex); - __compar_fn_t fn = getKeyComparFunc(pLeftColInfoData->info.type, pOrder->order); + __compar_fn_t fn = getKeyComparFunc(pLeftColInfoData->info.type, pOrder->order); - int ret = fn(left1, right1); - if (ret == 0) { - continue; - } else { - return ret; + int ret = fn(left1, right1); + if (ret == 0) { + continue; + } else { + return ret; + } } } return 0; @@ -533,7 +681,8 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { pHandle->numOfPages); } - int32_t numOfRows = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize); + int32_t numOfRows = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize, + blockDataGetSerialMetaSize(taosArrayGetSize(pHandle->pDataBlock->pDataBlock))); blockDataEnsureCapacity(pHandle->pDataBlock, numOfRows); // the initial pass + sortPass + final mergePass @@ -550,6 +699,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { // Only *numOfInputSources* can be loaded into buffer to perform the external sort. for (int32_t i = 0; i < sortGroup; ++i) { + qDebug("internal merge sort pass %d group %d. num input sources %d ", t, i, numOfInputSources); pHandle->sourceId += 1; int32_t end = (i + 1) * numOfInputSources - 1; @@ -572,8 +722,15 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { return code; } + int nMergedRows = 0; + SArray* pPageIdList = taosArrayInit(4, sizeof(int32_t)); while (1) { + if (tsortIsClosed(pHandle)) { + code = terrno = TSDB_CODE_TSC_QUERY_CANCELLED; + return code; + } + SSDataBlock* pDataBlock = getSortedBlockDataInner(pHandle, &pHandle->cmpParam, numOfRows); if (pDataBlock == NULL) { break; @@ -597,12 +754,16 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); + nMergedRows += pDataBlock->info.rows; blockDataCleanup(pDataBlock); + if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) { + break; + } } sortComparCleanup(&pHandle->cmpParam); - tMergeTreeDestroy(pHandle->pMergeTree); + tMergeTreeDestroy(&pHandle->pMergeTree); pHandle->numOfCompletedSources = 0; SSDataBlock* pBlock = createOneDataBlock(pHandle->pDataBlock, false); @@ -646,127 +807,404 @@ int32_t getProperSortPageSize(size_t rowSize, uint32_t numOfCols) { return pgSize; } -static int32_t createInitialSources(SSortHandle* pHandle) { - size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize; - int32_t code = 0; +static int32_t createPageBuf(SSortHandle* pHandle) { + if (pHandle->pBuf == NULL) { + if (!osTempSpaceAvailable()) { + terrno = TSDB_CODE_NO_DISKSPACE; + qError("create page buf failed since %s, tempDir:%s", terrstr(), tsTempDir); + return terrno; + } - if (pHandle->type == SORT_SINGLESOURCE_SORT) { - SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0); - SSortSource* source = *pSource; - *pSource = NULL; + int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, + "tableBlocksBuf", tsTempDir); + dBufSetPrintInfo(pHandle->pBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + return 0; +} - tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); +typedef struct SBlkMergeSupport { + int64_t** aTs; + int32_t* aRowIdx; + int32_t order; +} SBlkMergeSupport; - while (1) { - SSDataBlock* pBlock = pHandle->fetchfp(source->param); - if (pBlock == NULL) { - break; - } +static int32_t blockCompareTsFn(const void* pLeft, const void* pRight, void* param) { + int32_t left = *(int32_t*)pLeft; + int32_t right = *(int32_t*)pRight; - if (pHandle->pDataBlock == NULL) { - uint32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - pHandle->pageSize = getProperSortPageSize(blockDataGetRowSize(pBlock), numOfCols); + SBlkMergeSupport* pSup = (SBlkMergeSupport*)param; + if (pSup->aRowIdx[left] == -1) { + return 1; + } else if (pSup->aRowIdx[right] == -1) { + return -1; + } - // todo, number of pages are set according to the total available sort buffer - pHandle->numOfPages = 1024; - sortBufSize = pHandle->numOfPages * pHandle->pageSize; - pHandle->pDataBlock = createOneDataBlock(pBlock, false); - } + int64_t leftTs = pSup->aTs[left][pSup->aRowIdx[left]]; + int64_t rightTs = pSup->aTs[right][pSup->aRowIdx[right]]; - if (pHandle->beforeFp != NULL) { - pHandle->beforeFp(pBlock, pHandle->param); - } + int32_t ret = leftTs>rightTs ? 1 : ((leftTs < rightTs) ? -1 : 0); + if (pSup->order == TSDB_ORDER_DESC) { + ret = -1 * ret; + } + return ret; +} - code = blockDataMerge(pHandle->pDataBlock, pBlock); - if (code != TSDB_CODE_SUCCESS) { - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); +static int32_t appendDataBlockToPageBuf(SSortHandle* pHandle, SSDataBlock* blk, SArray* aPgId) { + int32_t pageId = -1; + void* pPage = getNewBufPage(pHandle->pBuf, &pageId); + taosArrayPush(aPgId, &pageId); + + int32_t size = blockDataGetSize(blk) + sizeof(int32_t) + taosArrayGetSize(blk->pDataBlock) * sizeof(int32_t); + ASSERT(size <= getBufPageSize(pHandle->pBuf)); + + blockDataToBuf(pPage, blk); + + setBufPageDirty(pPage, true); + releaseBufPage(pHandle->pBuf, pPage); + + return 0; +} + +static int32_t getPageBufIncForRow(SSDataBlock* blk, int32_t row, int32_t rowIdxInPage) { + int sz = 0; + int numCols = taosArrayGetSize(blk->pDataBlock); + if (!blk->info.hasVarCol) { + sz += numCols * ((rowIdxInPage & 0x7) == 0 ? 1: 0); + sz += blockDataGetRowSize(blk); + } else { + for (int32_t i = 0; i < numCols; ++i) { + SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(blk->pDataBlock, i); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + if (pColInfoData->varmeta.offset[row] != -1) { + char* p = colDataGetData(pColInfoData, row); + sz += varDataTLen(p); } - if (!source->onlyRef && source->src.pBlock) { - blockDataDestroy(source->src.pBlock); - source->src.pBlock = NULL; + + sz += sizeof(pColInfoData->varmeta.offset[0]); + } else { + sz += pColInfoData->info.bytes; + + if (((rowIdxInPage) & 0x07) == 0) { + sz += 1; // bitmap } - taosMemoryFree(source); - return code; } + } + } + return sz; +} + +static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockOrderInfo* order, SArray* aExtSrc) { + int pgHeaderSz = sizeof(int32_t) + sizeof(int32_t) * taosArrayGetSize(pHandle->pDataBlock->pDataBlock); + int32_t rowCap = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize, pgHeaderSz); + blockDataEnsureCapacity(pHandle->pDataBlock, rowCap); + blockDataCleanup(pHandle->pDataBlock); + int32_t numBlks = taosArrayGetSize(aBlk); + + SBlkMergeSupport sup; + sup.aRowIdx = taosMemoryCalloc(numBlks, sizeof(int32_t)); + sup.aTs = taosMemoryCalloc(numBlks, sizeof(int64_t*)); + sup.order = order->order; + for (int i = 0; i < numBlks; ++i) { + SSDataBlock* blk = taosArrayGetP(aBlk, i); + SColumnInfoData* col = taosArrayGet(blk->pDataBlock, order->slotId); + sup.aTs[i] = (int64_t*)col->pData; + sup.aRowIdx[i] = 0; + } + + int32_t totalRows = 0; + for (int i = 0; i < numBlks; ++i) { + SSDataBlock* blk = taosArrayGetP(aBlk, i); + totalRows += blk->info.rows; + } - size_t size = blockDataGetSize(pHandle->pDataBlock); - if (size > sortBufSize) { - // Perform the in-memory sort and then flush data in the buffer into disk. - int64_t p = taosGetTimestampUs(); - code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); - if (code != 0) { - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); + SArray* aPgId = taosArrayInit(8, sizeof(int32_t)); + + SMultiwayMergeTreeInfo* pTree = NULL; + tMergeTreeCreate(&pTree, taosArrayGetSize(aBlk), &sup, blockCompareTsFn); + int32_t nRows = 0; + int32_t nMergedRows = 0; + bool mergeLimitReached = false; + size_t blkPgSz = pgHeaderSz; + int64_t lastPageBufTs = (order->order == TSDB_ORDER_ASC) ? INT64_MAX : INT64_MIN; + int64_t currTs = (order->order == TSDB_ORDER_ASC) ? INT64_MAX : INT64_MIN; + while (nRows < totalRows) { + int32_t minIdx = tMergeTreeGetChosenIndex(pTree); + SSDataBlock* minBlk = taosArrayGetP(aBlk, minIdx); + int32_t minRow = sup.aRowIdx[minIdx]; + int32_t bufInc = getPageBufIncForRow(minBlk, minRow, pHandle->pDataBlock->info.rows); + + if (blkPgSz <= pHandle->pageSize && blkPgSz + bufInc > pHandle->pageSize) { + SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId); + lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; + appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + nMergedRows += pHandle->pDataBlock->info.rows; + + blockDataCleanup(pHandle->pDataBlock); + blkPgSz = pgHeaderSz; + bufInc = getPageBufIncForRow(minBlk, minRow, 0); + + if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) { + mergeLimitReached = true; + if ((lastPageBufTs < pHandle->currMergeLimitTs && order->order == TSDB_ORDER_ASC) || + (lastPageBufTs > pHandle->currMergeLimitTs && order->order == TSDB_ORDER_DESC)) { + pHandle->currMergeLimitTs = lastPageBufTs; } - if (!source->onlyRef && source->src.pBlock) { - blockDataDestroy(source->src.pBlock); - source->src.pBlock = NULL; + break; + } + } + blockDataEnsureCapacity(pHandle->pDataBlock, pHandle->pDataBlock->info.rows + 1); + appendOneRowToDataBlock(pHandle->pDataBlock, minBlk, &minRow); + blkPgSz += bufInc; + + ++nRows; + + if (sup.aRowIdx[minIdx] == minBlk->info.rows - 1) { + sup.aRowIdx[minIdx] = -1; + } else { + ++sup.aRowIdx[minIdx]; + } + tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree)); + } + if (pHandle->pDataBlock->info.rows > 0) { + if (!mergeLimitReached) { + SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId); + lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; + appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + nMergedRows += pHandle->pDataBlock->info.rows; + if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) { + mergeLimitReached = true; + if ((lastPageBufTs < pHandle->currMergeLimitTs && order->order == TSDB_ORDER_ASC) || + (lastPageBufTs > pHandle->currMergeLimitTs && order->order == TSDB_ORDER_DESC)) { + pHandle->currMergeLimitTs = lastPageBufTs; } + } + } + blockDataCleanup(pHandle->pDataBlock); + } + SSDataBlock* pMemSrcBlk = createOneDataBlock(pHandle->pDataBlock, false); + doAddNewExternalMemSource(pHandle->pBuf, aExtSrc, pMemSrcBlk, &pHandle->sourceId, aPgId); - taosMemoryFree(source); - return code; - } + taosMemoryFree(sup.aRowIdx); + taosMemoryFree(sup.aTs); - int64_t el = taosGetTimestampUs() - p; - pHandle->sortElapsed += el; + tMergeTreeDestroy(&pTree); - code = doAddToBuf(pHandle->pDataBlock, pHandle); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + return 0; +} + +static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { + SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0); + size_t nSrc = taosArrayGetSize(pHandle->pOrderedSource); + SArray* aExtSrc = taosArrayInit(nSrc, POINTER_BYTES); + + size_t maxBufSize = pHandle->numOfPages * pHandle->pageSize; + + int32_t code = createPageBuf(pHandle); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(aExtSrc); + return code; + } + + SSortSource* pSrc = taosArrayGetP(pHandle->pOrderedSource, 0); + int32_t szSort = 0; + + if (pOrder->order == TSDB_ORDER_ASC) { + pHandle->currMergeLimitTs = INT64_MAX; + } else { + pHandle->currMergeLimitTs = INT64_MIN; + } + + SArray* aBlkSort = taosArrayInit(8, POINTER_BYTES); + SSHashObj* mUidBlk = tSimpleHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT)); + while (1) { + SSDataBlock* pBlk = pHandle->fetchfp(pSrc->param); + if (pBlk != NULL) { + SColumnInfoData* tsCol = taosArrayGet(pBlk->pDataBlock, pOrder->slotId); + int64_t firstRowTs = *(int64_t*)tsCol->pData; + if ((pOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs) || + (pOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) { + continue; + } + } + if (pBlk != NULL) { + szSort += blockDataGetSize(pBlk); + + void* ppBlk = tSimpleHashGet(mUidBlk, &pBlk->info.id.uid, sizeof(pBlk->info.id.uid)); + if (ppBlk != NULL) { + SSDataBlock* tBlk = *(SSDataBlock**)(ppBlk); + blockDataMerge(tBlk, pBlk); + } else { + SSDataBlock* tBlk = createOneDataBlock(pBlk, true); + tSimpleHashPut(mUidBlk, &pBlk->info.id.uid, sizeof(pBlk->info.id.uid), &tBlk, POINTER_BYTES); + taosArrayPush(aBlkSort, &tBlk); + } + } + + if ((pBlk != NULL && szSort > maxBufSize) || (pBlk == NULL && szSort > 0)) { + tSimpleHashClear(mUidBlk); + + int64_t p = taosGetTimestampUs(); + sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc); + int64_t el = taosGetTimestampUs() - p; + pHandle->sortElapsed += el; + + for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) { + blockDataDestroy(taosArrayGetP(aBlkSort, i)); } + taosArrayClear(aBlkSort); + szSort = 0; + qDebug("source %zu created", taosArrayGetSize(aExtSrc)); + } + if (pBlk == NULL) { + break; } - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); + if (tsortIsClosed(pHandle)) { + tSimpleHashClear(mUidBlk); + for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) { + blockDataDestroy(taosArrayGetP(aBlkSort, i)); + } + taosArrayClear(aBlkSort); + break; } + } + + tSimpleHashCleanup(mUidBlk); + taosArrayDestroy(aBlkSort); + tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); + if (!tsortIsClosed(pHandle)) { + taosArrayAddAll(pHandle->pOrderedSource, aExtSrc); + } + taosArrayDestroy(aExtSrc); - taosMemoryFree(source); + pHandle->type = SORT_SINGLESOURCE_SORT; + return TSDB_CODE_SUCCESS; +} + +static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { + int32_t code = 0; + size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize; + + SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0); + SSortSource* source = *pSource; + *pSource = NULL; + + tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); + + while (1) { + SSDataBlock* pBlock = pHandle->fetchfp(source->param); + if (pBlock == NULL) { + break; + } - if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) { - size_t size = blockDataGetSize(pHandle->pDataBlock); + if (pHandle->pDataBlock == NULL) { + uint32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + pHandle->pageSize = getProperSortPageSize(blockDataGetRowSize(pBlock), numOfCols); + // todo, number of pages are set according to the total available sort buffer + pHandle->numOfPages = 1024; + sortBufSize = pHandle->numOfPages * pHandle->pageSize; + pHandle->pDataBlock = createOneDataBlock(pBlock, false); + } + + if (pHandle->beforeFp != NULL) { + pHandle->beforeFp(pBlock, pHandle->param); + } + + code = blockDataMerge(pHandle->pDataBlock, pBlock); + if (code != TSDB_CODE_SUCCESS) { + if (source->param && !source->onlyRef) { + taosMemoryFree(source->param); + } + if (!source->onlyRef && source->src.pBlock) { + blockDataDestroy(source->src.pBlock); + source->src.pBlock = NULL; + } + taosMemoryFree(source); + return code; + } + + size_t size = blockDataGetSize(pHandle->pDataBlock); + if (size > sortBufSize) { // Perform the in-memory sort and then flush data in the buffer into disk. int64_t p = taosGetTimestampUs(); - code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); if (code != 0) { + if (source->param && !source->onlyRef) { + taosMemoryFree(source->param); + } + if (!source->onlyRef && source->src.pBlock) { + blockDataDestroy(source->src.pBlock); + source->src.pBlock = NULL; + } + + taosMemoryFree(source); return code; } int64_t el = taosGetTimestampUs() - p; pHandle->sortElapsed += el; - - // All sorted data can fit in memory, external memory sort is not needed. Return to directly - if (size <= sortBufSize && pHandle->pBuf == NULL) { - pHandle->cmpParam.numOfSources = 1; - pHandle->inMemSort = true; - - pHandle->loops = 1; - pHandle->tupleHandle.rowIndex = -1; - pHandle->tupleHandle.pBlock = pHandle->pDataBlock; - return 0; - } else { - code = doAddToBuf(pHandle->pDataBlock, pHandle); + if (pHandle->pqMaxRows > 0) blockDataKeepFirstNRows(pHandle->pDataBlock, pHandle->pqMaxRows); + code = doAddToBuf(pHandle->pDataBlock, pHandle); + if (code != TSDB_CODE_SUCCESS) { + return code; } } } + if (source->param && !source->onlyRef) { + taosMemoryFree(source->param); + } + + taosMemoryFree(source); + + if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) { + size_t size = blockDataGetSize(pHandle->pDataBlock); + + // Perform the in-memory sort and then flush data in the buffer into disk. + int64_t p = taosGetTimestampUs(); + + code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); + if (code != 0) { + return code; + } + + if (pHandle->pqMaxRows > 0) blockDataKeepFirstNRows(pHandle->pDataBlock, pHandle->pqMaxRows); + int64_t el = taosGetTimestampUs() - p; + pHandle->sortElapsed += el; + + // All sorted data can fit in memory, external memory sort is not needed. Return to directly + if (size <= sortBufSize && pHandle->pBuf == NULL) { + pHandle->cmpParam.numOfSources = 1; + pHandle->inMemSort = true; + + pHandle->loops = 1; + pHandle->tupleHandle.rowIndex = -1; + pHandle->tupleHandle.pBlock = pHandle->pDataBlock; + return 0; + } else { + code = doAddToBuf(pHandle->pDataBlock, pHandle); + } + } return code; } -int32_t tsortOpen(SSortHandle* pHandle) { - if (pHandle->opened) { - return 0; - } +static int32_t createInitialSources(SSortHandle* pHandle) { + int32_t code = 0; - if (pHandle->fetchfp == NULL || pHandle->comparFn == NULL) { - return -1; + if (pHandle->type == SORT_SINGLESOURCE_SORT) { + code = createBlocksQuickSortInitialSources(pHandle); + } else if (pHandle->type == SORT_BLOCK_TS_MERGE) { + code = createBlocksMergeSortInitialSources(pHandle); } + qDebug("%zu sources created", taosArrayGetSize(pHandle->pOrderedSource)); + return code; +} - pHandle->opened = true; - +static bool tsortOpenForBufMergeSort(SSortHandle* pHandle) { int32_t code = createInitialSources(pHandle); if (code != TSDB_CODE_SUCCESS) { return code; @@ -796,10 +1234,23 @@ int32_t tsortOpen(SSortHandle* pHandle) { } int32_t tsortClose(SSortHandle* pHandle) { - // do nothing + atomic_val_compare_exchange_8(&pHandle->closed, 0, 1); + taosMsleep(10); return TSDB_CODE_SUCCESS; } +bool tsortIsClosed(SSortHandle* pHandle) { + return atomic_val_compare_exchange_8(&pHandle->closed, 1, 2); +} + +void tsortSetClosed(SSortHandle* pHandle) { + atomic_store_8(&pHandle->closed, 2); +} + +void tsortSetMergeLimit(SSortHandle* pHandle, int64_t mergeLimit) { + pHandle->mergeLimit = mergeLimit; +} + int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fetchFp, void (*fp)(SSDataBlock*, void*), void* param) { pHandle->fetchfp = fetchFp; @@ -818,7 +1269,10 @@ int32_t tsortSetCompareGroupId(SSortHandle* pHandle, bool compareGroupId) { return TSDB_CODE_SUCCESS; } -STupleHandle* tsortNextTuple(SSortHandle* pHandle) { +static STupleHandle* tsortBufMergeSortNextTuple(SSortHandle* pHandle) { + if (tsortIsClosed(pHandle)) { + return NULL; + } if (pHandle->cmpParam.numOfSources == pHandle->numOfCompletedSources) { return NULL; } @@ -865,6 +1319,196 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle) { return &pHandle->tupleHandle; } +static bool tsortIsForceUsePQSort(SSortHandle* pHandle) { + return pHandle->forceUsePQSort == true; +} + +void tsortSetForceUsePQSort(SSortHandle* pHandle) { + pHandle->forceUsePQSort = true; +} + +static bool tsortIsPQSortApplicable(SSortHandle* pHandle) { + if (pHandle->type != SORT_SINGLESOURCE_SORT) return false; + if (tsortIsForceUsePQSort(pHandle)) return true; + uint64_t maxRowsFitInMemory = pHandle->pqSortBufSize / (pHandle->pqMaxTupleLength + sizeof(char*)); + return maxRowsFitInMemory > pHandle->pqMaxRows; +} + +static bool tsortPQCompFn(void* a, void* b, void* param) { + SSortHandle* pHandle = param; + int32_t res = pHandle->comparFn(a, b, param); + if (res < 0) return 1; + return 0; +} + +static bool tsortPQComFnReverse(void*a, void* b, void* param) { + SSortHandle* pHandle = param; + int32_t res = pHandle->comparFn(a, b, param); + if (res > 0) return 1; + return 0; +} + +static int32_t tupleComparFn(const void* pLeft, const void* pRight, void* param) { + TupleDesc* pLeftDesc = (TupleDesc*)pLeft; + TupleDesc* pRightDesc = (TupleDesc*)pRight; + + SSortHandle* pHandle = (SSortHandle*)param; + SArray* orderInfo = (SArray*)pHandle->pSortInfo; + uint32_t colNum = blockDataGetNumOfCols(pHandle->pDataBlock); + for (int32_t i = 0; i < orderInfo->size; ++i) { + SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(orderInfo, i); + void *lData = tupleDescGetField(pLeftDesc, pOrder->slotId, colNum); + void *rData = tupleDescGetField(pRightDesc, pOrder->slotId, colNum); + if (!lData && !rData) continue; + if (!lData) return pOrder->nullFirst ? -1 : 1; + if (!rData) return pOrder->nullFirst ? 1 : -1; + + int type = ((SColumnInfoData*)taosArrayGet(pHandle->pDataBlock->pDataBlock, pOrder->slotId))->info.type; + __compar_fn_t fn = getKeyComparFunc(type, pOrder->order); + + int ret = fn(lData, rData); + if (ret == 0) { + continue; + } else { + return ret; + } + } + return 0; +} + +static int32_t tsortOpenForPQSort(SSortHandle* pHandle) { + pHandle->pBoundedQueue = createBoundedQueue(pHandle->pqMaxRows, tsortPQCompFn, destroyTuple, pHandle); + if (NULL == pHandle->pBoundedQueue) return TSDB_CODE_OUT_OF_MEMORY; + tsortSetComparFp(pHandle, tupleComparFn); + + SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0); + SSortSource* source = *pSource; + + pHandle->pDataBlock = NULL; + uint32_t tupleLen = 0; + PriorityQueueNode pqNode; + while (1) { + // fetch data + SSDataBlock* pBlock = pHandle->fetchfp(source->param); + if (NULL == pBlock) break; + + if (pHandle->beforeFp != NULL) { + pHandle->beforeFp(pBlock, pHandle->param); + } + if (pHandle->pDataBlock == NULL) { + pHandle->pDataBlock = createOneDataBlock(pBlock, false); + } + if (pHandle->pDataBlock == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + size_t colNum = blockDataGetNumOfCols(pBlock); + + if (tupleLen == 0) { + for (size_t colIdx = 0; colIdx < colNum; ++colIdx) { + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, colIdx); + tupleLen += pCol->info.bytes; + if (IS_VAR_DATA_TYPE(pCol->info.type)) { + tupleLen += sizeof(VarDataLenT); + } + } + } + ReferencedTuple refTuple = {.desc.data = (char*)pBlock, .desc.type = ReferencedTupleType, .rowIndex = 0}; + for (size_t rowIdx = 0; rowIdx < pBlock->info.rows; ++rowIdx) { + refTuple.rowIndex = rowIdx; + pqNode.data = &refTuple; + PriorityQueueNode* pPushedNode = taosBQPush(pHandle->pBoundedQueue, &pqNode); + if (!pPushedNode) { + // do nothing if push failed + } else { + pPushedNode->data = createAllocatedTuple(pBlock, colNum, tupleLen, rowIdx); + if (pPushedNode->data == NULL) return TSDB_CODE_OUT_OF_MEMORY; + } + } + } + return TSDB_CODE_SUCCESS; +} + +static STupleHandle* tsortPQSortNextTuple(SSortHandle* pHandle) { + if (pHandle->pDataBlock == NULL) { // when no input stream datablock + return NULL; + } + blockDataCleanup(pHandle->pDataBlock); + blockDataEnsureCapacity(pHandle->pDataBlock, 1); + // abandon the top tuple if queue size bigger than max size + if (taosBQSize(pHandle->pBoundedQueue) == taosBQMaxSize(pHandle->pBoundedQueue) + 1) { + taosBQPop(pHandle->pBoundedQueue); + } + if (pHandle->tmpRowIdx == 0) { + // sort the results + taosBQSetFn(pHandle->pBoundedQueue, tsortPQComFnReverse); + taosBQBuildHeap(pHandle->pBoundedQueue); + } + if (taosBQSize(pHandle->pBoundedQueue) > 0) { + uint32_t colNum = blockDataGetNumOfCols(pHandle->pDataBlock); + PriorityQueueNode* node = taosBQTop(pHandle->pBoundedQueue); + char* pTuple = ((TupleDesc*)node->data)->data; + + for (uint32_t i = 0; i < colNum; ++i) { + void* pData = tupleGetField(pTuple, i, colNum); + if (!pData) { + colDataSetNULL(bdGetColumnInfoData(pHandle->pDataBlock, i), 0); + } else { + colDataSetVal(bdGetColumnInfoData(pHandle->pDataBlock, i), 0, pData, false); + } + } + pHandle->pDataBlock->info.rows++; + pHandle->tmpRowIdx++; + taosBQPop(pHandle->pBoundedQueue); + } + if (pHandle->pDataBlock->info.rows == 0) return NULL; + pHandle->tupleHandle.pBlock = pHandle->pDataBlock; + return &pHandle->tupleHandle; +} + +static STupleHandle* tsortSingleTableMergeNextTuple(SSortHandle* pHandle) { + if (1 == pHandle->numOfCompletedSources) return NULL; + if (pHandle->tupleHandle.pBlock && pHandle->tupleHandle.rowIndex + 1 < pHandle->tupleHandle.pBlock->info.rows) { + pHandle->tupleHandle.rowIndex++; + } else { + if (pHandle->tupleHandle.rowIndex == -1) return NULL; + SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0); + SSortSource* source = *pSource; + SSDataBlock* pBlock = pHandle->fetchfp(source->param); + if (!pBlock || pBlock->info.rows == 0) { + setCurrentSourceDone(source, pHandle); + pHandle->tupleHandle.pBlock = NULL; + return NULL; + } + pHandle->tupleHandle.pBlock = pBlock; + pHandle->tupleHandle.rowIndex = 0; + } + return &pHandle->tupleHandle; +} + +int32_t tsortOpen(SSortHandle* pHandle) { + if (pHandle->opened) { + return 0; + } + + if (pHandle->fetchfp == NULL || pHandle->comparFn == NULL) { + return -1; + } + + pHandle->opened = true; + if (tsortIsPQSortApplicable(pHandle)) + return tsortOpenForPQSort(pHandle); + else + return tsortOpenForBufMergeSort(pHandle); +} + +STupleHandle* tsortNextTuple(SSortHandle* pHandle) { + if (pHandle->singleTableMerge) + return tsortSingleTableMergeNextTuple(pHandle); + else if (pHandle->pBoundedQueue) + return tsortPQSortNextTuple(pHandle); + else + return tsortBufMergeSortNextTuple(pHandle); +} + bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex) { SColumnInfoData* pColInfoSrc = taosArrayGet(pVHandle->pBlock->pDataBlock, colIndex); return colDataIsNull_s(pColInfoSrc, pVHandle->rowIndex); @@ -880,6 +1524,7 @@ void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) { } uint64_t tsortGetGroupId(STupleHandle* pVHandle) { return pVHandle->pBlock->info.id.groupId; } +void* tsortGetBlockInfo(STupleHandle* pVHandle) { return &pVHandle->pBlock->info; } SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) { SSortExecInfo info = {0}; diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index 8122d7d6a91fed15eddce1fa74150600475043c0..b4d188459733d3c06e6e37e53122f24ddb01768c 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -66,7 +66,7 @@ SSDataBlock* getSingleColDummyBlock(void* param) { colInfo.info.type = pInfo->type; if (pInfo->type == TSDB_DATA_TYPE_NCHAR) { colInfo.info.bytes = TSDB_NCHAR_SIZE * VARCOUNT + VARSTR_HEADER_SIZE; - } else if (pInfo->type == TSDB_DATA_TYPE_BINARY) { + } else if (pInfo->type == TSDB_DATA_TYPE_BINARY || pInfo->type == TSDB_DATA_TYPE_GEOMETRY) { colInfo.info.bytes = VARCOUNT + VARSTR_HEADER_SIZE; } else { colInfo.info.bytes = tDataTypes[pInfo->type].bytes; @@ -94,7 +94,7 @@ SSDataBlock* getSingleColDummyBlock(void* param) { colDataSetVal(pColInfo, i, reinterpret_cast(str), false); pBlock->info.hasVarCol = true; printf("nchar: %s\n", strOri); - } else if (pInfo->type == TSDB_DATA_TYPE_BINARY) { + } else if (pInfo->type == TSDB_DATA_TYPE_BINARY || pInfo->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t size = taosRand() % VARCOUNT; char str[64] = {0}; taosRandStr(varDataVal(str), size); @@ -317,7 +317,7 @@ TEST(testCase, external_mem_sort_Test) { char buf[128] = {0}; int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(v), varDataLen(v), buf); printf("%d: %s\n", row++, buf); - }else if(pInfo[i].type == TSDB_DATA_TYPE_BINARY){ + }else if(pInfo[i].type == TSDB_DATA_TYPE_BINARY || pInfo[i]->type == TSDB_DATA_TYPE_GEOMETRY){ char buf[128] = {0}; memcpy(buf, varDataVal(v), varDataLen(v)); printf("%d: %s\n", row++, buf); diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2894c665870e79dc93d01565f7fffe87d8552c86 --- /dev/null +++ b/source/libs/executor/test/timewindowTest.cpp @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "taos.h" +#include "thash.h" +#include "tsimplehash.h" +#include "executor.h" +#include "ttime.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +namespace { +SInterval createInterval(int64_t interval, int64_t sliding, int64_t offset, char intervalUnit, char slidingUnit, + char offsetUnit, int8_t precision) { + SInterval v = {0}; + v.interval = interval; + v.intervalUnit = intervalUnit; + v.sliding = sliding; + v.slidingUnit = slidingUnit; + v.offset = offset; + v.offsetUnit = offsetUnit; + v.precision = precision; + return v; +} + +void printTimeWindow(STimeWindow* pWindow, int8_t precision, int64_t ts) { + char buf[64] = {0}; + char bufs[64] = {0}; + char bufe[64] = {0}; + + taosFormatUtcTime(buf, tListLen(buf), ts, precision); + + taosFormatUtcTime(bufs, tListLen(bufs), pWindow->skey, precision); + taosFormatUtcTime(bufe, tListLen(bufe), pWindow->ekey, precision); + + printf("%s [%s - %s]\n", buf, bufs, bufe); +} +} // namespace + +TEST(testCase, timewindow_gen) { + // set correct time zone + osSetTimezone("UTC"); + int32_t precision = TSDB_TIME_PRECISION_MILLI; + + SInterval interval = + createInterval(10 * 86400 * 1000, 10 * 86400 * 1000, 0, 'd', 'd', 'd', precision); + + int64_t key = 1659312000L * 1000; // 2022-8-1 00:00:00 // UTC+8 (ms) + + STimeWindow w = {0}; + getInitialStartTimeWindow(&interval, key, &w, true); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&interval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, precision, key); + + printf("---------------------------------------------------\n"); + SInterval monthInterval = + createInterval(1, 1, 0, 'n', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&monthInterval, key, &w, true); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&monthInterval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, precision, key); + + printf("----------------------------------------------------------\n"); + SInterval slidingInterval = createInterval(1, 10*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&slidingInterval, key, &w, true); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + printf("-----------------calendar_interval_1n_sliding_1d-------\n"); + SInterval calendar_interval_1n = createInterval(1, 1*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + int64_t k1 = 1664409600 * 1000L; + getInitialStartTimeWindow(&calendar_interval_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + + printf("next\n"); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + printf("----------------interval_1d_clendar_sliding_1n---------\n"); + SInterval interval_1d_calendar_sliding_1n = createInterval(1*86400*1000L, 1, 0, 'd', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + + k1 = 1664409600 * 1000L; + getInitialStartTimeWindow(&interval_1d_calendar_sliding_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + + printf("next time window:\n"); + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + printf("----------------interval_1d_sliding_1d_calendar_offset_1n---------\n"); + SInterval offset_1n = createInterval(10*86400*1000L, 10*86400*1000L, 1, 'd', 'd', 'n', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&offset_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + + +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index f23b4d3e8762951a791529930781f657294c7d13..3a68648d493358c986b4fd52c736a85155c9d38b 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -32,8 +32,8 @@ target_link_libraries( PRIVATE nodes PRIVATE qcom PRIVATE scalar + PRIVATE geometry PRIVATE transport - PRIVATE stream ${LINK_JEMALLOC} PUBLIC uv_a ) diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index a07038384e0b2f1d8ff7e5f833e8b2d0f4defa24..6d23f65cf376f4f0dcb8e772cef49cfa468ce68c 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -50,6 +50,8 @@ extern "C" { #define FUNC_MGT_KEEP_ORDER_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(21) #define FUNC_MGT_CUMULATIVE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(22) #define FUNC_MGT_INTERP_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(23) +#define FUNC_MGT_GEOMETRY_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(24) +#define FUNC_MGT_FORBID_SYSTABLE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(25) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index fe98a1dd537af5fc48b66c431dcfe524fdefe972..cee40001552d68fe62f585666b5c550b346bac13 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -18,6 +18,7 @@ #include "cJSON.h" #include "querynodes.h" #include "scalar.h" +#include "geomFunc.h" #include "taoserror.h" #include "ttime.h" @@ -467,7 +468,8 @@ static int32_t translateStddevMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // pseudo column do not need to check parameters - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT, + .precision = pFunc->node.resType.precision}; return TSDB_CODE_SUCCESS; } @@ -490,7 +492,8 @@ static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, in // pseudo column do not need to check parameters pFunc->node.resType = - (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; + (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP, + .precision = pFunc->node.resType.precision}; return TSDB_CODE_SUCCESS; } @@ -1568,17 +1571,32 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); uint8_t dbPrec = pFunc->node.resType.precision; - // if (1 != numOfParams && 3 != numOfParams && 4 != numOfParams) { - if (1 != numOfParams) { + if (2 < numOfParams) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } uint8_t nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, 0)); uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if ((!IS_NUMERIC_TYPE(paraType) && !IS_BOOLEAN_TYPE(paraType))|| QUERY_NODE_VALUE == nodeType) { + if ((!IS_NUMERIC_TYPE(paraType) && !IS_BOOLEAN_TYPE(paraType)) || QUERY_NODE_VALUE == nodeType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + if (2 == numOfParams) { + nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, 1)); + paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; + if (!IS_INTEGER_TYPE(paraType) || QUERY_NODE_VALUE != nodeType) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); + if (pValue->datum.i != 0 && pValue->datum.i != 1) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "INTERP function second parameter should be 0/1"); + } + + pValue->notReserved = true; + } + #if 0 if (3 <= numOfParams) { int64_t timeVal[2] = {0}; @@ -1624,6 +1642,15 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return TSDB_CODE_SUCCESS; } +static EFuncReturnRows interpEstReturnRows(SFunctionNode* pFunc) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (1 < numOfParams && 1 == ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 1))->datum.i) { + return FUNC_RETURN_ROWS_INDEFINITE; + } else { + return FUNC_RETURN_ROWS_N; + } +} + static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // forbid null as first/last input, since first(c0, null, 1) may have different number of input int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); @@ -2083,6 +2110,70 @@ static int32_t translateToJson(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return TSDB_CODE_SUCCESS; } +static int32_t translateInStrOutGeom(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_STR_DATA_TYPE(para1Type) && !IS_NULL_TYPE(para1Type)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_GEOMETRY].bytes, .type = TSDB_DATA_TYPE_GEOMETRY}; + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateInGeomOutStr(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (para1Type != TSDB_DATA_TYPE_GEOMETRY && !IS_NULL_TYPE(para1Type)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_VARCHAR].bytes, .type = TSDB_DATA_TYPE_VARCHAR}; + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateIn2NumOutGeom(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (2 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; + if ((!IS_NUMERIC_TYPE(para1Type) && !IS_NULL_TYPE(para1Type)) || + (!IS_NUMERIC_TYPE(para2Type) && !IS_NULL_TYPE(para2Type))) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_GEOMETRY].bytes, .type = TSDB_DATA_TYPE_GEOMETRY}; + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateIn2GeomOutBool(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (2 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; + if ((para1Type != TSDB_DATA_TYPE_GEOMETRY && !IS_NULL_TYPE(para1Type)) || + (para2Type != TSDB_DATA_TYPE_GEOMETRY && !IS_NULL_TYPE(para2Type))) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes, .type = TSDB_DATA_TYPE_BOOL}; + + return TSDB_CODE_SUCCESS; +} + static int32_t translateSelectValue(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; return TSDB_CODE_SUCCESS; @@ -2259,7 +2350,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "leastsquares", .type = FUNCTION_TYPE_LEASTSQUARES, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateLeastSQR, .getEnvFunc = getLeastSQRFuncEnv, .initFunc = leastSQRFunctionSetup, @@ -2367,7 +2458,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "top", .type = FUNCTION_TYPE_TOP, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_FILL_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | + FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_FILL_FUNC, .translateFunc = translateTopBot, .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, @@ -2382,7 +2474,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "bottom", .type = FUNCTION_TYPE_BOTTOM, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_FILL_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | + FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_FILL_FUNC, .translateFunc = translateTopBot, .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, @@ -2439,7 +2532,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "elapsed", .type = FUNCTION_TYPE_ELAPSED, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | + FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, .dataRequiredFunc = statisDataRequired, .translateFunc = translateElapsed, .getEnvFunc = getElapsedFuncEnv, @@ -2479,18 +2573,19 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "interp", .type = FUNCTION_TYPE_INTERP, .classification = FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | - FUNC_MGT_FORBID_STREAM_FUNC|FUNC_MGT_KEEP_ORDER_FUNC, + FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, .translateFunc = translateInterp, .getEnvFunc = getSelectivityFuncEnv, .initFunc = functionSetup, .processFunc = NULL, - .finalizeFunc = NULL + .finalizeFunc = NULL, + .estimateReturnRowsFunc = interpEstReturnRows }, { .name = "derivative", .type = FUNCTION_TYPE_DERIVATIVE, .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | - FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_CUMULATIVE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_CUMULATIVE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateDerivative, .getEnvFunc = getDerivativeFuncEnv, .initFunc = derivativeFuncSetup, @@ -2502,7 +2597,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "irate", .type = FUNCTION_TYPE_IRATE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | + FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateIrate, .getEnvFunc = getIrateFuncEnv, .initFunc = irateFuncSetup, @@ -2513,7 +2609,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "last_row", .type = FUNCTION_TYPE_LAST_ROW, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | + FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateFirstLast, .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, @@ -2528,7 +2625,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_cache_last_row", .type = FUNCTION_TYPE_CACHE_LAST_ROW, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | + FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -2538,7 +2636,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_cache_last", .type = FUNCTION_TYPE_CACHE_LAST, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -2548,7 +2646,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_last_row_partial", .type = FUNCTION_TYPE_LAST_PARTIAL, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | + FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateFirstLastPartial, .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, @@ -2559,7 +2658,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_last_row_merge", .type = FUNCTION_TYPE_LAST_MERGE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | + FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateFirstLastMerge, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -2569,7 +2669,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "first", .type = FUNCTION_TYPE_FIRST, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | + FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateFirstLast, .dynDataRequiredFunc = firstDynDataReq, .getEnvFunc = getFirstLastFuncEnv, @@ -2584,7 +2685,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_first_partial", .type = FUNCTION_TYPE_FIRST_PARTIAL, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | + FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateFirstLastPartial, .dynDataRequiredFunc = firstDynDataReq, .getEnvFunc = getFirstLastFuncEnv, @@ -2596,7 +2698,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_first_merge", .type = FUNCTION_TYPE_FIRST_MERGE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | + FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateFirstLastMerge, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -2607,7 +2710,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "last", .type = FUNCTION_TYPE_LAST, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | + FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateFirstLast, .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, @@ -2622,7 +2726,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_last_partial", .type = FUNCTION_TYPE_LAST_PARTIAL, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | + FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateFirstLastPartial, .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, @@ -2634,7 +2739,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_last_merge", .type = FUNCTION_TYPE_LAST_MERGE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | + FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateFirstLastMerge, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -2645,7 +2751,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "twa", .type = FUNCTION_TYPE_TWA, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | + FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateInNumOutDou, .dataRequiredFunc = statisDataRequired, .getEnvFunc = getTwaFuncEnv, @@ -2736,7 +2843,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "diff", .type = FUNCTION_TYPE_DIFF, .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | - FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_CUMULATIVE_FUNC, + FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_CUMULATIVE_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateDiff, .getEnvFunc = getDiffFuncEnv, .initFunc = diffFunctionSetup, @@ -2749,7 +2856,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "statecount", .type = FUNCTION_TYPE_STATE_COUNT, .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | - FUNC_MGT_FORBID_STREAM_FUNC, + FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateStateCount, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, @@ -2761,7 +2868,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "stateduration", .type = FUNCTION_TYPE_STATE_DURATION, .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | - FUNC_MGT_FORBID_STREAM_FUNC, + FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateStateDuration, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, @@ -2773,7 +2880,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "csum", .type = FUNCTION_TYPE_CSUM, .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | - FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_CUMULATIVE_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, + FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_CUMULATIVE_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateCsum, .getEnvFunc = getCsumFuncEnv, .initFunc = functionSetup, @@ -2786,7 +2893,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "mavg", .type = FUNCTION_TYPE_MAVG, .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | - FUNC_MGT_FORBID_STREAM_FUNC, + FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateMavg, .getEnvFunc = getMavgFuncEnv, .initFunc = mavgFunctionSetup, @@ -2797,7 +2904,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "sample", .type = FUNCTION_TYPE_SAMPLE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_FILL_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | + FUNC_MGT_FORBID_FILL_FUNC, .translateFunc = translateSample, .getEnvFunc = getSampleFuncEnv, .initFunc = sampleFunctionSetup, @@ -2808,8 +2916,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "tail", .type = FUNCTION_TYPE_TAIL, - .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | - FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateTail, .getEnvFunc = getTailFuncEnv, .initFunc = tailFunctionSetup, @@ -2820,8 +2927,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "unique", .type = FUNCTION_TYPE_UNIQUE, - .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | - FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateUnique, .getEnvFunc = getUniqueFuncEnv, .initFunc = uniqueFunctionSetup, @@ -3337,6 +3443,96 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = NULL, .finalizeFunc = NULL }, + { + .name = "st_geomfromtext", + .type = FUNCTION_TYPE_GEOM_FROM_TEXT, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_GEOMETRY_FUNC, + .translateFunc = translateInStrOutGeom, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = geomFromTextFunction, + .finalizeFunc = NULL + }, + { + .name = "st_astext", + .type = FUNCTION_TYPE_AS_TEXT, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_GEOMETRY_FUNC, + .translateFunc = translateInGeomOutStr, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = asTextFunction, + .finalizeFunc = NULL + }, + { + .name = "st_makepoint", + .type = FUNCTION_TYPE_MAKE_POINT, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_GEOMETRY_FUNC, + .translateFunc = translateIn2NumOutGeom, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = makePointFunction, + .finalizeFunc = NULL + }, + { + .name = "st_intersects", + .type = FUNCTION_TYPE_INTERSECTS, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_GEOMETRY_FUNC, + .translateFunc = translateIn2GeomOutBool, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = intersectsFunction, + .finalizeFunc = NULL + }, + { + .name = "st_equals", + .type = FUNCTION_TYPE_EQUALS, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_GEOMETRY_FUNC, + .translateFunc = translateIn2GeomOutBool, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = equalsFunction, + .finalizeFunc = NULL + }, + { + .name = "st_touches", + .type = FUNCTION_TYPE_TOUCHES, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_GEOMETRY_FUNC, + .translateFunc = translateIn2GeomOutBool, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = touchesFunction, + .finalizeFunc = NULL + }, + { + .name = "st_covers", + .type = FUNCTION_TYPE_COVERS, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_GEOMETRY_FUNC, + .translateFunc = translateIn2GeomOutBool, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = coversFunction, + .finalizeFunc = NULL + }, + { + .name = "st_contains", + .type = FUNCTION_TYPE_CONTAINS, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_GEOMETRY_FUNC, + .translateFunc = translateIn2GeomOutBool, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = containsFunction, + .finalizeFunc = NULL + }, + { + .name = "st_containsproperly", + .type = FUNCTION_TYPE_CONTAINS_PROPERLY, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_GEOMETRY_FUNC, + .translateFunc = translateIn2GeomOutBool, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = containsProperlyFunction, + .finalizeFunc = NULL + }, }; // clang-format on diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 80b26bd39b5f06b560c51343141606ee8c0c2bae..fad8c9ca5bb4869a9d3d869ba6a36d127c0613fa 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -18,7 +18,6 @@ #include "function.h" #include "query.h" #include "querynodes.h" -#include "streamState.h" #include "tcompare.h" #include "tdatablock.h" #include "tdigest.h" @@ -500,7 +499,7 @@ static int64_t getNumOfElems(SqlFunctionCtx* pCtx) { */ SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows && !IS_VAR_DATA_TYPE(pInputCol->info.type)) { + if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows) { numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull; } else { if (pInputCol->hasNull) { @@ -883,10 +882,6 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu } pStart += pDstCol->info.bytes; } - - if (pCtx->saveHandle.pState) { - streamFreeVal((void*)p); - } } return TSDB_CODE_SUCCESS; @@ -925,6 +920,7 @@ void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) { *pDestPos = *pSourcePos; } +#define COMPARE_MINMAX_DATA(type) (( (*(type*)&pDBuf->v) < (*(type*)&pSBuf->v) ) ^ isMinFunc) int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t isMinFunc) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SMinmaxResInfo* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); @@ -932,18 +928,57 @@ int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SMinmaxResInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); int16_t type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type; - if (IS_FLOAT_TYPE(type)) { - if (pSBuf->assign && ((((*(double*)&pDBuf->v) < (*(double*)&pSBuf->v)) ^ isMinFunc) || !pDBuf->assign)) { - *(double*)&pDBuf->v = *(double*)&pSBuf->v; - replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); - pDBuf->assign = true; - } - } else { - if (pSBuf->assign && (((pDBuf->v < pSBuf->v) ^ isMinFunc) || !pDBuf->assign)) { - pDBuf->v = pSBuf->v; - replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); - pDBuf->assign = true; + + switch (type) { + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_BIGINT: + if (pSBuf->assign && (COMPARE_MINMAX_DATA(int64_t) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_INT: + if (pSBuf->assign && (COMPARE_MINMAX_DATA(int32_t) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_SMALLINT: + if (pSBuf->assign && (COMPARE_MINMAX_DATA(int16_t) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: + if (pSBuf->assign && (COMPARE_MINMAX_DATA(int8_t) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; + case TSDB_DATA_TYPE_FLOAT: { + if (pSBuf->assign && (COMPARE_MINMAX_DATA(double) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; } + default: + if (pSBuf->assign && (strcmp((char*)&pDBuf->v, (char*)&pSBuf->v) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; } pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); pDResInfo->isNullRes &= pSResInfo->isNullRes; @@ -1701,7 +1736,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { GET_TYPED_DATA(v, double, pVal->nType, &pVal->i); - int32_t code = getPercentile(pMemBucket, v, &ppInfo->result); + code = getPercentile(pMemBucket, v, &ppInfo->result); if (code != TSDB_CODE_SUCCESS) { goto _fin_error; } @@ -1717,7 +1752,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); varDataSetLen(buf, len); - colDataAppend(pCol, pBlock->info.rows, buf, false); + colDataSetVal(pCol, pBlock->info.rows, buf, false); tMemBucketDestroy(pMemBucket); return pResInfo->numOfRes; @@ -2420,6 +2455,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } static int32_t firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) { + if (!pInput->hasResult) { + return TSDB_CODE_FAILED; + } + if (pOutput->hasResult) { if (isFirst) { if (pInput->ts > pOutput->ts) { @@ -2665,7 +2704,7 @@ bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { } else { pDiffInfo->ignoreNegative = false; } - pDiffInfo->includeNull = false; + pDiffInfo->includeNull = true; pDiffInfo->firstOutput = false; return true; } @@ -2703,13 +2742,12 @@ static int32_t doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv, } static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SColumnInfoData* pOutput, int32_t pos, - int32_t order, int64_t ts) { - int32_t factor = (order == TSDB_ORDER_ASC) ? 1 : -1; + int64_t ts) { pDiffInfo->prevTs = ts; switch (type) { case TSDB_DATA_TYPE_INT: { int32_t v = *(int32_t*)pv; - int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null + int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null if (delta < 0 && pDiffInfo->ignoreNegative) { colDataSetNull_f_s(pOutput, pos); } else { @@ -2722,7 +2760,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: { int8_t v = *(int8_t*)pv; - int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null + int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null if (delta < 0 && pDiffInfo->ignoreNegative) { colDataSetNull_f_s(pOutput, pos); } else { @@ -2733,7 +2771,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, } case TSDB_DATA_TYPE_SMALLINT: { int16_t v = *(int16_t*)pv; - int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null + int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null if (delta < 0 && pDiffInfo->ignoreNegative) { colDataSetNull_f_s(pOutput, pos); } else { @@ -2745,7 +2783,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: { int64_t v = *(int64_t*)pv; - int64_t delta = factor * (v - pDiffInfo->prev.i64); // direct previous may be null + int64_t delta = v - pDiffInfo->prev.i64; // direct previous may be null if (delta < 0 && pDiffInfo->ignoreNegative) { colDataSetNull_f_s(pOutput, pos); } else { @@ -2756,7 +2794,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, } case TSDB_DATA_TYPE_FLOAT: { float v = *(float*)pv; - double delta = factor * (v - pDiffInfo->prev.d64); // direct previous may be null + double delta = v - pDiffInfo->prev.d64; // direct previous may be null if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) { // check for overflow colDataSetNull_f_s(pOutput, pos); } else { @@ -2767,7 +2805,7 @@ static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, } case TSDB_DATA_TYPE_DOUBLE: { double v = *(double*)pv; - double delta = factor * (v - pDiffInfo->prev.d64); // direct previous may be null + double delta = v - pDiffInfo->prev.d64; // direct previous may be null if ((delta < 0 && pDiffInfo->ignoreNegative) || isinf(delta) || isnan(delta)) { // check for overflow colDataSetNull_f_s(pOutput, pos); } else { @@ -2798,82 +2836,42 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; - if (pCtx->order == TSDB_ORDER_ASC) { - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { - int32_t pos = startOffset + numOfElems; + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + int32_t pos = startOffset + numOfElems; - if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - if (pDiffInfo->includeNull) { - colDataSetNull_f_s(pOutput, pos); + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + if (pDiffInfo->includeNull) { + colDataSetNull_f_s(pOutput, pos); - numOfElems += 1; - } - continue; + numOfElems += 1; } + continue; + } - char* pv = colDataGetData(pInputCol, i); + char* pv = colDataGetData(pInputCol, i); - if (pDiffInfo->hasPrev) { - if (tsList[i] == pDiffInfo->prevTs) { - return TSDB_CODE_FUNC_DUP_TIMESTAMP; - } - int32_t code = doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, pCtx->order, tsList[i]); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, pos); - } - - numOfElems++; - } else { - int32_t code = doSetPrevVal(pDiffInfo, pInputCol->info.type, pv, tsList[i]); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + if (pDiffInfo->hasPrev) { + if (tsList[i] == pDiffInfo->prevTs) { + return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - - pDiffInfo->hasPrev = true; - } - } else { - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { - int32_t pos = startOffset + numOfElems; - - if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - if (pDiffInfo->includeNull) { - colDataSetNull_f_s(pOutput, pos); - numOfElems += 1; - } - continue; + int32_t code = doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, tsList[i]); + if (code != TSDB_CODE_SUCCESS) { + return code; } - - char* pv = colDataGetData(pInputCol, i); - - // there is a row of previous data block to be handled in the first place. - if (pDiffInfo->hasPrev) { - if (tsList[i] == pDiffInfo->prevTs) { - return TSDB_CODE_FUNC_DUP_TIMESTAMP; - } - int32_t code = doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, pCtx->order, tsList[i]); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, pos); - } - - numOfElems++; - } else { - int32_t code = doSetPrevVal(pDiffInfo, pInputCol->info.type, pv, tsList[i]); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + // handle selectivity + if (pCtx->subsidiaries.num > 0) { + appendSelectivityValue(pCtx, i, pos); } - pDiffInfo->hasPrev = true; + numOfElems++; + } else { + int32_t code = doSetPrevVal(pDiffInfo, pInputCol->info.type, pv, tsList[i]); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } + + pDiffInfo->hasPrev = true; } pResInfo->numOfRes = numOfElems; @@ -3123,8 +3121,8 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid return buf; } -static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STupleKey* key, - STuplePos* pPos) { +static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, SWinKey* key, + STuplePos* pPos, SFunctionStateStore* pStore) { STuplePos p = {0}; if (pHandle->pBuf != NULL) { SFilePage* pPage = NULL; @@ -3157,9 +3155,8 @@ static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, pPage->num += length; setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); - } else { - // other tuple save policy - if (streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) { + } else { // other tuple save policy + if (pStore->streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) { p.streamTupleKey = *key; } } @@ -3171,7 +3168,7 @@ static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { prepareBuf(pCtx); - STupleKey key; + SWinKey key; if (pCtx->saveHandle.pBuf == NULL) { SColumnInfoData* pColInfo = taosArrayGet(pSrcBlock->pDataBlock, 0); if (pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { @@ -3179,15 +3176,14 @@ int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* key.groupId = pSrcBlock->info.id.groupId; key.ts = skey; - key.exprIdx = pCtx->exprIdx; } } char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); - return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, &key, pPos); + return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, &key, pPos, pCtx->pStore); } -static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos) { +static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos, SFunctionStateStore* pStore) { if (pHandle->pBuf != NULL) { SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId); if (pPage == NULL) { @@ -3197,7 +3193,7 @@ static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); } else { - streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length); + pStore->streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length); } return TSDB_CODE_SUCCESS; @@ -3207,10 +3203,10 @@ int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBloc prepareBuf(pCtx); char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); - return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos); + return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos, pCtx->pStore); } -static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos) { +static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos, SFunctionStateStore* pStore) { if (pHandle->pBuf != NULL) { SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId); if (pPage == NULL) { @@ -3222,13 +3218,13 @@ static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPo } else { void* value = NULL; int32_t vLen; - streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen); + pStore->streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen); return (char*)value; } } const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos) { - return doLoadTupleData(&pCtx->saveHandle, pPos); + return doLoadTupleData(&pCtx->saveHandle, pPos, pCtx->pStore); } int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { @@ -4996,7 +4992,7 @@ static int32_t saveModeTupleData(SqlFunctionCtx* pCtx, char* data, SModeInfo *pI memcpy(pInfo->buf, data, pInfo->colBytes); } - return doSaveTupleData(&pCtx->saveHandle, pInfo->buf, pInfo->colBytes, NULL, pPos); + return doSaveTupleData(&pCtx->saveHandle, pInfo->buf, pInfo->colBytes, NULL, pPos, pCtx->pStore); } static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, char* data) { @@ -5025,7 +5021,7 @@ static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCt } else { pHashItem->count += 1; if (pCtx->subsidiaries.num > 0) { - int32_t code = updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &pHashItem->tuplePos); + code = updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &pHashItem->tuplePos); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 18f6e8050b56e1c73a28cf64bb24c1da3844651f..345020cee23ca27ce7d0ec1a05103042b5b6fc23 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -219,6 +219,8 @@ bool fmIsKeepOrderFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, F bool fmIsCumulativeFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_CUMULATIVE_FUNC); } +bool fmIsForbidSysTableFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_SYSTABLE_FUNC); } + bool fmIsInterpFunc(int32_t funcId) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; @@ -344,7 +346,7 @@ static int32_t getFuncInfo(SFunctionNode* pFunc) { return fmGetFuncInfo(pFunc, msg, sizeof(msg)); } -static SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList) { +SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList) { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == pFunc) { return NULL; diff --git a/source/libs/function/src/thistogram.c b/source/libs/function/src/thistogram.c index e7d631f638da769fe0d9eabb03762bb983410a56..b56691f35d2933da298b74b13af0d2eee32ebb61 100644 --- a/source/libs/function/src/thistogram.c +++ b/source/libs/function/src/thistogram.c @@ -474,8 +474,8 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { } ASSERTS(total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem, - "tHistogramUniform Error, total:%d, numOfElem:%d, elems[%d].num:%d", - total, numOfElem, j + 1, pHisto->elems[j + 1].num); + "tHistogramUniform Error, total:%ld, numOfElem:%ld, elems[%d].num:%ld", + total, (int64_t)numOfElem, j + 1, pHisto->elems[j + 1].num); double delta = numOfElem - total; if (fabs(delta) < FLT_EPSILON) { diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 3ec802a7cebfc29f33b4c9e61b909908807ae501..8101b342a44e3132c969b3055839fbf7eb55c8b4 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -39,6 +39,7 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) if (p != NULL) { pIdList = *(SArray **)p; } else { + taosMemoryFree(buffer); return NULL; } @@ -48,6 +49,7 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); if (pg == NULL) { + taosMemoryFree(buffer); return NULL; } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 6b70422ac8fe20bf8089eb8efba11ecc3c5e6ae5..5b9f44c8126b520715bf32144b99f9af17638174 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -255,6 +255,18 @@ int32_t udfStopUdfd() { return 0; } +int32_t udfGetUdfdPid(int32_t* pUdfdPid) { + SUdfdData *pData = &udfdGlobal; + if (pData->spawnErr) { + return pData->spawnErr; + } + uv_pid_t pid = uv_process_get_pid(&pData->process); + if (pUdfdPid) { + *pUdfdPid = (int32_t)pid; + } + return TSDB_CODE_SUCCESS; +} + //============================================================================================== /* Copyright (c) 2013, Ben Noordhuis * The QUEUE is copied from queue.h under libuv @@ -791,7 +803,21 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo memcpy(udfCol->colData.varLenCol.varOffsets, col->varmeta.offset, udfCol->colData.varLenCol.varOffsetsLen); udfCol->colData.varLenCol.payloadLen = colDataGetLength(col, udfBlock->numOfRows); udfCol->colData.varLenCol.payload = taosMemoryMalloc(udfCol->colData.varLenCol.payloadLen); - memcpy(udfCol->colData.varLenCol.payload, col->pData, udfCol->colData.varLenCol.payloadLen); + if (col->reassigned) { + for (int32_t row = 0; row < udfCol->colData.numOfRows; ++row) { + char* pColData = col->pData + col->varmeta.offset[row]; + int32_t colSize = 0; + if (col->info.type == TSDB_DATA_TYPE_JSON) { + colSize = getJsonValueLen(pColData); + } else { + colSize = varDataTLen(pColData); + } + memcpy(udfCol->colData.varLenCol.payload, pColData, colSize); + udfCol->colData.varLenCol.payload += colSize; + } + } else { + memcpy(udfCol->colData.varLenCol.payload, col->pData, udfCol->colData.varLenCol.payloadLen); + } } else { udfCol->colData.fixLenCol.nullBitmapLen = BitmapLen(udfCol->colData.numOfRows); int32_t bitmapLen = udfCol->colData.fixLenCol.nullBitmapLen; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index d2553ba96dbc242025fa9489da25fbbbee70e7f8..7371017111f5662fd765d3c7afb74fedca377b16 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -29,6 +29,7 @@ #include "tmsg.h" #include "trpc.h" #include "tmisce.h" +#include "tversion.h" // clang-format on #define UDFD_MAX_SCRIPT_PLUGINS 64 @@ -54,41 +55,36 @@ int32_t udfdCPluginOpen(SScriptUdfEnvItem *items, int numItems) { return 0; } int32_t udfdCPluginClose() { return 0; } const char *udfdCPluginUdfInitLoadInitDestoryFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) { - char initFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; + char initFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; char *initSuffix = "_init"; - strcpy(initFuncName, udfName); - strncat(initFuncName, initSuffix, strlen(initSuffix)); + snprintf(initFuncName, sizeof(initFuncName), "%s%s", udfName, initSuffix); uv_dlsym(&udfCtx->lib, initFuncName, (void **)(&udfCtx->initFunc)); - char destroyFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; + char destroyFuncName[TSDB_FUNC_NAME_LEN + 9] = {0}; char *destroySuffix = "_destroy"; - strcpy(destroyFuncName, udfName); - strncat(destroyFuncName, destroySuffix, strlen(destroySuffix)); + snprintf(destroyFuncName, sizeof(destroyFuncName), "%s%s", udfName, destroySuffix); uv_dlsym(&udfCtx->lib, destroyFuncName, (void **)(&udfCtx->destroyFunc)); return udfName; } void udfdCPluginUdfInitLoadAggFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) { char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; - strcpy(processFuncName, udfName); + strncpy(processFuncName, udfName, sizeof(processFuncName)); uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->aggProcFunc)); - char startFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; + char startFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; char *startSuffix = "_start"; - strncpy(startFuncName, processFuncName, sizeof(startFuncName)); - strncat(startFuncName, startSuffix, strlen(startSuffix)); + snprintf(startFuncName, sizeof(startFuncName), "%s%s", processFuncName, startSuffix); uv_dlsym(&udfCtx->lib, startFuncName, (void **)(&udfCtx->aggStartFunc)); - char finishFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; + char finishFuncName[TSDB_FUNC_NAME_LEN + 8] = {0}; char *finishSuffix = "_finish"; - strncpy(finishFuncName, processFuncName, sizeof(finishFuncName)); - strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); + snprintf(finishFuncName, sizeof(finishFuncName), "%s%s", processFuncName, finishSuffix); uv_dlsym(&udfCtx->lib, finishFuncName, (void **)(&udfCtx->aggFinishFunc)); - char mergeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; + char mergeFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; char *mergeSuffix = "_merge"; - strncpy(mergeFuncName, processFuncName, sizeof(mergeFuncName)); - strncat(mergeFuncName, mergeSuffix, strlen(mergeSuffix)); + snprintf(mergeFuncName, sizeof(mergeFuncName), "%s%s", processFuncName, mergeSuffix); uv_dlsym(&udfCtx->lib, mergeFuncName, (void **)(&udfCtx->aggMergeFunc)); } @@ -98,6 +94,7 @@ int32_t udfdCPluginUdfInit(SScriptUdfInfo *udf, void **pUdfCtx) { err = uv_dlopen(udf->path, &udfCtx->lib); if (err != 0) { fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); + taosMemoryFree(udfCtx); return TSDB_CODE_UDF_LOAD_UDF_FAILURE; } const char *udfName = udf->name; @@ -106,7 +103,7 @@ int32_t udfdCPluginUdfInit(SScriptUdfInfo *udf, void **pUdfCtx) { if (udf->funcType == UDF_FUNC_TYPE_SCALAR) { char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; - strcpy(processFuncName, udfName); + strncpy(processFuncName, udfName, sizeof(processFuncName)); uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->scalarProcFunc)); } else if (udf->funcType == UDF_FUNC_TYPE_AGG) { udfdCPluginUdfInitLoadAggFuncs(udfCtx, udfName); @@ -604,9 +601,9 @@ SUdf *udfdGetOrCreateUdf(const char *udfName) { return udf; } else { (*pUdfHash)->expired = true; - taosHashRemove(global.udfsHash, udfName, strlen(udfName)); fnInfo("udfd expired, check for new version. existing udf %s udf version %d, udf created time %" PRIx64, (*pUdfHash)->name, (*pUdfHash)->version, (*pUdfHash)->createdTime); + taosHashRemove(global.udfsHash, udfName, strlen(udfName)); } } @@ -969,40 +966,6 @@ int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) { return code; } -int32_t udfdConnectToMnode() { - SConnectReq connReq = {0}; - connReq.connType = CONN_TYPE__UDFD; - tstrncpy(connReq.app, "udfd", sizeof(connReq.app)); - tstrncpy(connReq.user, TSDB_DEFAULT_USER, sizeof(connReq.user)); - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - taosEncryptPass_c((uint8_t *)(TSDB_DEFAULT_PASS), strlen(TSDB_DEFAULT_PASS), pass); - tstrncpy(connReq.passwd, pass, sizeof(connReq.passwd)); - connReq.pid = taosGetPId(); - connReq.startTime = taosGetTimestampMs(); - strcpy(connReq.sVer, version); - - int32_t contLen = tSerializeSConnectReq(NULL, 0, &connReq); - void *pReq = rpcMallocCont(contLen); - tSerializeSConnectReq(pReq, contLen, &connReq); - - SUdfdRpcSendRecvInfo *msgInfo = taosMemoryCalloc(1, sizeof(SUdfdRpcSendRecvInfo)); - msgInfo->rpcType = UDFD_RPC_MNODE_CONNECT; - uv_sem_init(&msgInfo->resultSem, 0); - - SRpcMsg rpcMsg = {0}; - rpcMsg.msgType = TDMT_MND_CONNECT; - rpcMsg.pCont = pReq; - rpcMsg.contLen = contLen; - rpcMsg.info.ahandle = msgInfo; - rpcSendRequest(global.clientRpc, &global.mgmtEp.epSet, &rpcMsg, NULL); - - uv_sem_wait(&msgInfo->resultSem); - int32_t code = msgInfo->code; - uv_sem_destroy(&msgInfo->resultSem); - taosMemoryFree(msgInfo); - return code; -} - static bool udfdRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED || code == TSDB_CODE_SYN_RESTORING || @@ -1076,7 +1039,7 @@ int32_t udfdOpenClientRpc() { connLimitNum = TMIN(connLimitNum, 500); rpcInit.connLimitNum = connLimitNum; rpcInit.timeToGetConn = tsTimeToGetAvailableConn; - + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); global.clientRpc = rpcOpen(&rpcInit); if (global.clientRpc == NULL) { fnError("failed to init dnode rpc client"); @@ -1382,23 +1345,6 @@ static int32_t udfdRun() { return 0; } -void udfdConnectMnodeThreadFunc(void *args) { - int32_t retryMnodeTimes = 0; - int32_t code = 0; - while (retryMnodeTimes++ <= TSDB_MAX_REPLICA) { - uv_sleep(100 * (1 << retryMnodeTimes)); - code = udfdConnectToMnode(); - if (code == 0) { - break; - } - fnError("udfd can not connect to mnode, code: %s. retry", tstrerror(code)); - } - - if (code != 0) { - fnError("udfd can not connect to mnode"); - } -} - int32_t udfdInitResidentFuncs() { if (strlen(tsUdfdResFuncs) == 0) { return TSDB_CODE_SUCCESS; @@ -1501,9 +1447,6 @@ int main(int argc, char *argv[]) { udfdInitResidentFuncs(); - uv_thread_t mnodeConnectThread; - uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL); - udfdRun(); removeListeningPipe(); diff --git a/source/libs/geometry/CMakeLists.txt b/source/libs/geometry/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..679c73f508689fa61ec74a894688e4dd0f14006f --- /dev/null +++ b/source/libs/geometry/CMakeLists.txt @@ -0,0 +1,17 @@ +aux_source_directory(src GEOMETRY_SRC) + +add_library(geometry STATIC ${GEOMETRY_SRC}) +target_include_directories( + geometry + PUBLIC "${TD_SOURCE_DIR}/include/libs/geometry" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../scalar/inc" +) + +target_link_libraries( + geometry + PRIVATE os util nodes function scalar qcom +) + +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) diff --git a/source/libs/geometry/src/geomFunc.c b/source/libs/geometry/src/geomFunc.c new file mode 100644 index 0000000000000000000000000000000000000000..3588bf8b7d932b9ffc2ce94a47971883380cea4a --- /dev/null +++ b/source/libs/geometry/src/geomFunc.c @@ -0,0 +1,444 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include "geosWrapper.h" +#include "geomFunc.h" +#include "querynodes.h" +#include "tdatablock.h" +#include "sclInt.h" +#include "sclvector.h" + +typedef int32_t (*_geomDoRelationFunc_t)(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, + bool swapped, char *res); + +typedef int32_t (*_geomInitCtxFunc_t)(); +typedef int32_t (*_geomExecuteOneParamFunc_t)(SColumnInfoData *pInputData, int32_t i, SColumnInfoData *pOutputData); +typedef int32_t (*_geomExecuteTwoParamsFunc_t)(SColumnInfoData *pInputData[], int32_t iLeft, int32_t iRight, + SColumnInfoData *pOutputData); + +// output is with VARSTR format +// need to call taosMemoryFree(*output) later +int32_t doMakePointFunc(double x, double y, unsigned char **output) { + int32_t code = TSDB_CODE_FAILED; + + unsigned char *outputGeom = NULL; + size_t size = 0; + code = doMakePoint(x, y, &outputGeom, &size); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + + *output = taosMemoryCalloc(1, size + VARSTR_HEADER_SIZE); + if (*output == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + memcpy(varDataVal(*output), outputGeom, size); + varDataSetLen(*output, size); + code = TSDB_CODE_SUCCESS; + +_exit: + geosFreeBuffer(outputGeom); + + return code; +} + +// both input and output are with VARSTR format +// need to call taosMemoryFree(*output) later +int32_t doGeomFromTextFunc(const char *input, unsigned char **output) { + int32_t code = TSDB_CODE_FAILED; + + if ((varDataLen(input)) == 0) { //empty value + *output = NULL; + return TSDB_CODE_SUCCESS; + } + + // make input as a zero ending string + char *end = varDataVal(input) + varDataLen(input); + char endValue = *end; + *end = 0; + + unsigned char *outputGeom = NULL; + size_t size = 0; + + code = doGeomFromText(varDataVal(input), &outputGeom, &size); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + + *output = taosMemoryCalloc(1, size + VARSTR_HEADER_SIZE); + if (*output == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + memcpy(varDataVal(*output), outputGeom, size); + varDataSetLen(*output, size); + code = TSDB_CODE_SUCCESS; + +_exit: + geosFreeBuffer(outputGeom); + + *end = endValue; //recover the input string + + return code; +} + +// both input and output are with VARSTR format +// need to call taosMemoryFree(*output) later +int32_t doAsTextFunc(unsigned char *input, char **output) { + int32_t code = TSDB_CODE_FAILED; + + if ((varDataLen(input)) == 0) { //empty value + *output = NULL; + return TSDB_CODE_SUCCESS; + } + + char *outputWKT = NULL; + code = doAsText(varDataVal(input), varDataLen(input), &outputWKT); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + + size_t size = strlen(outputWKT); + *output = taosMemoryCalloc(1, size + VARSTR_HEADER_SIZE); + if (*output == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + memcpy(varDataVal(*output), outputWKT, size); + varDataSetLen(*output, size); + code = TSDB_CODE_SUCCESS; + +_exit: + geosFreeBuffer(outputWKT); + + return code; +} + +int32_t executeMakePointFunc(SColumnInfoData *pInputData[], int32_t iLeft, int32_t iRight, + SColumnInfoData *pOutputData) { + int32_t code = TSDB_CODE_FAILED; + + _getDoubleValue_fn_t getDoubleValueFn[2]; + getDoubleValueFn[0]= getVectorDoubleValueFn(pInputData[0]->info.type); + getDoubleValueFn[1]= getVectorDoubleValueFn(pInputData[1]->info.type); + + unsigned char *output = NULL; + code = doMakePointFunc(getDoubleValueFn[0](pInputData[0]->pData, iLeft), getDoubleValueFn[1](pInputData[1]->pData, iRight), &output); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + + colDataSetVal(pOutputData, TMAX(iLeft, iRight), output, (output == NULL)); + +_exit: + if (output) { + taosMemoryFree(output); + } + + return code; +} + +int32_t executeGeomFromTextFunc(SColumnInfoData *pInputData, int32_t i, SColumnInfoData *pOutputData) { + int32_t code = TSDB_CODE_FAILED; + + char *input = colDataGetData(pInputData, i); + unsigned char *output = NULL; + code = doGeomFromTextFunc(input, &output); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + + colDataSetVal(pOutputData, i, output, (output == NULL)); + +_exit: + if (output) { + taosMemoryFree(output); + } + + return code; +} + +int32_t executeAsTextFunc(SColumnInfoData *pInputData, int32_t i, SColumnInfoData *pOutputData) { + int32_t code = TSDB_CODE_FAILED; + + unsigned char *input = colDataGetData(pInputData, i); + char *output = NULL; + code = doAsTextFunc(input, &output); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + + colDataSetVal(pOutputData, i, output, (output == NULL)); + +_exit: + if (output) { + taosMemoryFree(output); + } + + return code; +} + +int32_t executeRelationFunc(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, + const GEOSGeometry *geom2, int32_t i, + bool swapped, SColumnInfoData *pOutputData, + _geomDoRelationFunc_t doRelationFn) { + int32_t code = TSDB_CODE_FAILED; + char res = 0; + + if (!geom1 || !geom2) { //if empty input value + res = -1; + code = TSDB_CODE_SUCCESS; + } + else { + code = doRelationFn(geom1, preparedGeom1, geom2, swapped, &res); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + colDataSetVal(pOutputData, i, &res, (res==-1)); + + return code; +} + +int32_t geomOneParamFunction(SScalarParam *pInput, SScalarParam *pOutput, + _geomInitCtxFunc_t initCtxFn, _geomExecuteOneParamFunc_t executeOneParamFn) { + int32_t code = TSDB_CODE_FAILED; + + code = initCtxFn(); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + pOutput->numOfRows = pInput->numOfRows; + + if (IS_NULL_TYPE(GET_PARAM_TYPE(pInput))) { + colDataSetNNULL(pOutputData, 0, pInput->numOfRows); + code = TSDB_CODE_SUCCESS; + } + else { + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + colDataSetNULL(pOutputData, i); + code = TSDB_CODE_SUCCESS; + continue; + } + + code = executeOneParamFn(pInputData, i, pOutputData); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } + + return code; +} + +int32_t geomTwoParamsFunction(SScalarParam *pInput, SScalarParam *pOutput, + _geomInitCtxFunc_t initCtxFn, _geomExecuteTwoParamsFunc_t executeTwoParamsFn) { + int32_t code = TSDB_CODE_FAILED; + + code = initCtxFn(); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + SColumnInfoData *pInputData[2]; + SColumnInfoData *pOutputData = pOutput->columnData; + pInputData[0] = pInput[0].columnData; + pInputData[1] = pInput[1].columnData; + + bool hasNullType = (IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0])) || + IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[1]))); + bool isConstantLeft = (pInput[0].numOfRows == 1); + bool isConstantRight = (pInput[1].numOfRows == 1); + int32_t numOfRows = TMAX(pInput[0].numOfRows, pInput[1].numOfRows); + pOutput->numOfRows = numOfRows; + + if (hasNullType || // one of operant is NULL type + (isConstantLeft && colDataIsNull_s(pInputData[0], 0)) || // left operand is constant NULL + (isConstantRight && colDataIsNull_s(pInputData[1], 0))) { // right operand is constant NULL + colDataSetNNULL(pOutputData, 0, numOfRows); + code = TSDB_CODE_SUCCESS; + } + else { + int32_t iLeft = 0; + int32_t iRight = 0; + for (int32_t i = 0; i < numOfRows; ++i) { + iLeft = isConstantLeft ? 0 : i; + iRight = isConstantRight ? 0 : i; + + if ((!isConstantLeft && colDataIsNull_s(pInputData[0], iLeft)) || + (!isConstantRight && colDataIsNull_s(pInputData[1], iRight))) { + colDataSetNULL(pOutputData, i); + code = TSDB_CODE_SUCCESS; + continue; + } + + code = executeTwoParamsFn(pInputData, iLeft, iRight, pOutputData); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } + + return code; +} + +int32_t geomRelationFunction(SScalarParam *pInput, SScalarParam *pOutput, + bool swapAllowed, _geomDoRelationFunc_t doRelationFn) { + int32_t code = TSDB_CODE_FAILED; + + code = initCtxRelationFunc(); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // handle with all NULL output + bool hasNullType = (IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0])) || + IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[1]))); + bool isConstant1 = (pInput[0].numOfRows == 1); + bool isConstant2 = (pInput[1].numOfRows == 1); + int32_t numOfRows = TMAX(pInput[0].numOfRows, pInput[1].numOfRows); + pOutput->numOfRows = numOfRows; + SColumnInfoData *pOutputData = pOutput->columnData; + + if (hasNullType || // at least one of operant is NULL type + (isConstant1 && colDataIsNull_s(pInput[0].columnData, 0)) || // left operand is constant NULL + (isConstant2 && colDataIsNull_s(pInput[1].columnData, 0))) { // right operand is constant NULL + colDataSetNNULL(pOutputData, 0, numOfRows); + code = TSDB_CODE_SUCCESS; + return code; + } + + bool swapped = false; + SColumnInfoData *pInputData[2]; + + // swap two input data to make sure input data 0 is constant if swapAllowed and only isConstant2 is true + if (swapAllowed && + !isConstant1 && isConstant2) { + pInputData[0] = pInput[1].columnData; + pInputData[1] = pInput[0].columnData; + + isConstant1 = true; + isConstant2 = false; + swapped = true; + } + else { + pInputData[0] = pInput[0].columnData; + pInputData[1] = pInput[1].columnData; + } + + GEOSGeometry *geom1 = NULL; + GEOSGeometry *geom2 = NULL; + const GEOSPreparedGeometry *preparedGeom1 = NULL; + + // if there is constant, make PreparedGeometry from pInputData 0 + if (isConstant1) { + code = readGeometry(colDataGetData(pInputData[0], 0), &geom1, &preparedGeom1); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + } + if (isConstant2) { + code = readGeometry(colDataGetData(pInputData[1], 0), &geom2, NULL); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + } + + for (int32_t i = 0; i < numOfRows; ++i) { + if ((!isConstant1 && colDataIsNull_s(pInputData[0], i)) || + (!isConstant2 && colDataIsNull_s(pInputData[1], i))) { + colDataSetNULL(pOutputData, i); + code = TSDB_CODE_SUCCESS; + continue; + } + + if (!isConstant1) { + code = readGeometry(colDataGetData(pInputData[0], i), &geom1, &preparedGeom1); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + } + if (!isConstant2) { + code = readGeometry(colDataGetData(pInputData[1], i), &geom2, NULL); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + } + + code = executeRelationFunc(geom1, preparedGeom1, geom2, i, swapped, pOutputData, doRelationFn); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + + if (!isConstant1) { + destroyGeometry(&geom1, &preparedGeom1); + } + if (!isConstant2) { + destroyGeometry(&geom2, NULL); + } + } + +_exit: + destroyGeometry(&geom1, &preparedGeom1); + destroyGeometry(&geom2, NULL); + + return code; +} + +int32_t makePointFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return geomTwoParamsFunction(pInput, pOutput, initCtxMakePoint, executeMakePointFunc); +} + +int32_t geomFromTextFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return geomOneParamFunction(pInput, pOutput, initCtxGeomFromText, executeGeomFromTextFunc); +} + +int32_t asTextFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return geomOneParamFunction(pInput, pOutput, initCtxAsText, executeAsTextFunc); +} + +int32_t intersectsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return geomRelationFunction(pInput, pOutput, true, doIntersects); +} + +int32_t equalsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return geomRelationFunction(pInput, pOutput, true, doEquals); +} + +int32_t touchesFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return geomRelationFunction(pInput, pOutput, true, doTouches); +} + +int32_t coversFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return geomRelationFunction(pInput, pOutput, true, doCovers); +} + +int32_t containsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return geomRelationFunction(pInput, pOutput, true, doContains); +} + +int32_t containsProperlyFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return geomRelationFunction(pInput, pOutput, false, doContainsProperly); +} diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c new file mode 100644 index 0000000000000000000000000000000000000000..993178e2b02f3ba75a875df39edb861f5f1f240a --- /dev/null +++ b/source/libs/geometry/src/geosWrapper.c @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "geosWrapper.h" +#include "tdef.h" +#include "types.h" + +typedef char (*_geosRelationFunc_t)(GEOSContextHandle_t handle, const GEOSGeometry *g1, const GEOSGeometry *g2); +typedef char (*_geosPreparedRelationFunc_t)(GEOSContextHandle_t handle, const GEOSPreparedGeometry *pg1, const GEOSGeometry *g2); + +void geosFreeBuffer(void *buffer) { + if (buffer) { + GEOSFree_r(getThreadLocalGeosCtx()->handle, buffer); + } +} + +void geosErrMsgeHandler(const char *errMsg, void *userData) { + char* targetErrMsg = userData; + snprintf(targetErrMsg, 512, "%s", errMsg); +} + +int32_t initCtxMakePoint() { + int32_t code = TSDB_CODE_FAILED; + SGeosContext* geosCtx = getThreadLocalGeosCtx(); + + if (geosCtx->handle == NULL) { + geosCtx->handle = GEOS_init_r(); + if (geosCtx->handle == NULL) { + return code; + } + + GEOSContext_setErrorMessageHandler_r(geosCtx->handle, geosErrMsgeHandler, geosCtx->errMsg); + } + + if (geosCtx->WKBWriter == NULL) { + geosCtx->WKBWriter = GEOSWKBWriter_create_r(geosCtx->handle); + if (geosCtx->WKBWriter == NULL) { + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + +// outputWKT is a zero ending string +// need to call geosFreeBuffer(*outputGeom) later +int32_t doMakePoint(double x, double y, unsigned char **outputGeom, size_t *size) { + int32_t code = TSDB_CODE_FAILED; + SGeosContext* geosCtx = getThreadLocalGeosCtx(); + + GEOSGeometry *geom = NULL; + unsigned char *wkb = NULL; + + geom = GEOSGeom_createPointFromXY_r(geosCtx->handle, x, y); + if (geom == NULL) { + code = TSDB_CODE_FUNC_FUNTION_PARA_VALUE; + goto _exit; + } + + wkb = GEOSWKBWriter_write_r(geosCtx->handle, geosCtx->WKBWriter, geom, size); + if (wkb == NULL) { + goto _exit; + } + *outputGeom = wkb; + + code = TSDB_CODE_SUCCESS; + +_exit: + if (geom) { + GEOSGeom_destroy_r(geosCtx->handle, geom); + geom = NULL; + } + + return code; +} + +int32_t initCtxGeomFromText() { + int32_t code = TSDB_CODE_FAILED; + SGeosContext* geosCtx = getThreadLocalGeosCtx(); + + if (geosCtx->handle == NULL) { + geosCtx->handle = GEOS_init_r(); + if (geosCtx->handle == NULL) { + return code; + } + + GEOSContext_setErrorMessageHandler_r(geosCtx->handle, geosErrMsgeHandler, geosCtx->errMsg); + } + + if (geosCtx->WKTReader == NULL) { + geosCtx->WKTReader = GEOSWKTReader_create_r(geosCtx->handle); + if (geosCtx->WKTReader == NULL) { + return code; + } + } + + if (geosCtx->WKBWriter == NULL) { + geosCtx->WKBWriter = GEOSWKBWriter_create_r(geosCtx->handle); + if (geosCtx->WKBWriter == NULL) { + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + +// inputWKT is a zero ending string +// need to call geosFreeBuffer(*outputGeom) later +int32_t doGeomFromText(const char *inputWKT, unsigned char **outputGeom, size_t *size) { + int32_t code = TSDB_CODE_FAILED; + SGeosContext* geosCtx = getThreadLocalGeosCtx(); + + GEOSGeometry *geom = NULL; + unsigned char *wkb = NULL; + + geom = GEOSWKTReader_read_r(geosCtx->handle, geosCtx->WKTReader, inputWKT); + if (geom == NULL) { + code = TSDB_CODE_FUNC_FUNTION_PARA_VALUE; + goto _exit; + } + + wkb = GEOSWKBWriter_write_r(geosCtx->handle, geosCtx->WKBWriter, geom, size); + if (wkb == NULL) { + goto _exit; + } + *outputGeom = wkb; + + code = TSDB_CODE_SUCCESS; + +_exit: + if (geom) { + GEOSGeom_destroy_r(geosCtx->handle, geom); + geom = NULL; + } + + return code; +} + +int32_t initCtxAsText() { + int32_t code = TSDB_CODE_FAILED; + SGeosContext* geosCtx = getThreadLocalGeosCtx(); + + if (geosCtx->handle == NULL) { + geosCtx->handle = GEOS_init_r(); + if (geosCtx->handle == NULL) { + return code; + } + + GEOSContext_setErrorMessageHandler_r(geosCtx->handle, geosErrMsgeHandler, geosCtx->errMsg); + } + + if (geosCtx->WKBReader == NULL) { + geosCtx->WKBReader = GEOSWKBReader_create_r(geosCtx->handle); + if (geosCtx->WKBReader == NULL) { + return code; + } + } + + if (geosCtx->WKTWriter == NULL) { + geosCtx->WKTWriter = GEOSWKTWriter_create_r(geosCtx->handle); + + if (geosCtx->WKTWriter) { + GEOSWKTWriter_setRoundingPrecision_r(geosCtx->handle, geosCtx->WKTWriter, 6); + GEOSWKTWriter_setTrim_r(geosCtx->handle, geosCtx->WKTWriter, 0); + } else { + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + +// outputWKT is a zero ending string +// need to call geosFreeBuffer(*outputWKT) later +int32_t doAsText(const unsigned char *inputGeom, size_t size, char **outputWKT) { + int32_t code = TSDB_CODE_FAILED; + SGeosContext* geosCtx = getThreadLocalGeosCtx(); + + GEOSGeometry *geom = NULL; + unsigned char *wkt = NULL; + + geom = GEOSWKBReader_read_r(geosCtx->handle, geosCtx->WKBReader, inputGeom, size); + if (geom == NULL) { + code = TSDB_CODE_FUNC_FUNTION_PARA_VALUE; + goto _exit; + } + + wkt = GEOSWKTWriter_write_r(geosCtx->handle, geosCtx->WKTWriter, geom); + if (wkt == NULL) { + goto _exit; + } + *outputWKT = wkt; + + code = TSDB_CODE_SUCCESS; + +_exit: + if (geom) { + GEOSGeom_destroy_r(geosCtx->handle, geom); + geom = NULL; + } + + return code; +} + +int32_t initCtxRelationFunc() { + int32_t code = TSDB_CODE_FAILED; + SGeosContext* geosCtx = getThreadLocalGeosCtx(); + + if (geosCtx->handle == NULL) { + geosCtx->handle = GEOS_init_r(); + if (geosCtx->handle == NULL) { + return code; + } + + GEOSContext_setErrorMessageHandler_r(geosCtx->handle, geosErrMsgeHandler, geosCtx->errMsg); + } + + if (geosCtx->WKBReader == NULL) { + geosCtx->WKBReader = GEOSWKBReader_create_r(geosCtx->handle); + if (geosCtx->WKBReader == NULL) { + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t doGeosRelation(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, + bool swapped, char *res, + _geosRelationFunc_t relationFn, + _geosRelationFunc_t swappedRelationFn, + _geosPreparedRelationFunc_t preparedRelationFn, + _geosPreparedRelationFunc_t swappedPreparedRelationFn) { + int32_t code = TSDB_CODE_FAILED; + SGeosContext* geosCtx = getThreadLocalGeosCtx(); + + if (!preparedGeom1) { + if (!swapped) { + ASSERT(relationFn); + *res = relationFn(geosCtx->handle, geom1, geom2); + } + else { + ASSERT(swappedRelationFn); + *res = swappedRelationFn(geosCtx->handle, geom1, geom2); + } + } + else { + if (!swapped) { + ASSERT(preparedRelationFn); + *res = preparedRelationFn(geosCtx->handle, preparedGeom1, geom2); + } + else { + ASSERT(swappedPreparedRelationFn); + *res = swappedPreparedRelationFn(geosCtx->handle, preparedGeom1, geom2); + } + } + + code = TSDB_CODE_SUCCESS; + return code; +} + +int32_t doIntersects(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, + bool swapped, char *res) { + return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, + GEOSIntersects_r, GEOSIntersects_r, GEOSPreparedIntersects_r, GEOSPreparedIntersects_r); +} + +int32_t doEquals(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, + bool swapped, char *res) { + return doGeosRelation(geom1, NULL, geom2, swapped, res, + GEOSEquals_r, GEOSEquals_r, NULL, NULL); // no prepared version for eguals() +} + +int32_t doTouches(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, + bool swapped, char *res) { + return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, + GEOSTouches_r, GEOSTouches_r, GEOSPreparedTouches_r, GEOSPreparedTouches_r); +} + +int32_t doCovers(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, + bool swapped, char *res) { + return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, + GEOSCovers_r, GEOSCoveredBy_r, GEOSPreparedCovers_r, GEOSPreparedCoveredBy_r); +} + +int32_t doContains(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, + bool swapped, char *res) { + return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, + GEOSContains_r, GEOSWithin_r, GEOSPreparedContains_r, GEOSPreparedWithin_r); +} + +int32_t doContainsProperly(const GEOSGeometry *geom1, const GEOSPreparedGeometry *preparedGeom1, const GEOSGeometry *geom2, + bool swapped, char *res) { + return doGeosRelation(geom1, preparedGeom1, geom2, swapped, res, + NULL, NULL, GEOSPreparedContainsProperly_r, NULL); +} + +// input is with VARSTR format +// need to call destroyGeometry(outputGeom, outputPreparedGeom) later +int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom, const GEOSPreparedGeometry **outputPreparedGeom) { + SGeosContext* geosCtx = getThreadLocalGeosCtx(); + + ASSERT(outputGeom); //it is not allowed if outputGeom is NULL + *outputGeom = NULL; + + if (outputPreparedGeom) { //it means not to generate PreparedGeometry if outputPreparedGeom is NULL + *outputPreparedGeom = NULL; + } + + if (varDataLen(input) == 0) { //empty value + return TSDB_CODE_SUCCESS; + } + + *outputGeom = GEOSWKBReader_read_r(geosCtx->handle, geosCtx->WKBReader, varDataVal(input), varDataLen(input)); + if (*outputGeom == NULL) { + return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; + } + + if (outputPreparedGeom) { + *outputPreparedGeom = GEOSPrepare_r(geosCtx->handle, *outputGeom); + if (*outputPreparedGeom == NULL) { + return TSDB_CODE_FAILED; + } + } + + return TSDB_CODE_SUCCESS; +} + +void destroyGeometry(GEOSGeometry **geom, const GEOSPreparedGeometry **preparedGeom) { + SGeosContext* geosCtx = getThreadLocalGeosCtx(); + + if (preparedGeom && *preparedGeom) { + GEOSPreparedGeom_destroy_r(geosCtx->handle, *preparedGeom); + *preparedGeom = NULL; + } + + if (geom && *geom) { + GEOSGeom_destroy_r(geosCtx->handle, *geom); + *geom = NULL; + } +} diff --git a/source/libs/geometry/test/CMakeLists.txt b/source/libs/geometry/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..ba849a9dc8642bc83d09fc2249069bd3af48639f --- /dev/null +++ b/source/libs/geometry/test/CMakeLists.txt @@ -0,0 +1,19 @@ +MESSAGE(STATUS "build geometry unit test") + +IF(NOT TD_DARWIN) + # GoogleTest requires at least C++11 + SET(CMAKE_CXX_STANDARD 11) + + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + ADD_EXECUTABLE(geomTest ${SOURCE_LIST}) + + TARGET_LINK_LIBRARIES( + geomTest + PUBLIC os util gtest qcom nodes geometry scalar function scalar + ) + + add_test( + NAME geomTest + COMMAND geomTest + ) +ENDIF() diff --git a/source/libs/geometry/test/geomFuncTest.cpp b/source/libs/geometry/test/geomFuncTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..459c7499cc0b5f038689461e1650c29428337643 --- /dev/null +++ b/source/libs/geometry/test/geomFuncTest.cpp @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include "os.h" + +int main(int argc, char **argv) { + taosSeedRand(taosGetTimestampSec()); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/source/libs/geometry/test/geomFuncTestUtil.cpp b/source/libs/geometry/test/geomFuncTestUtil.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0918781499e98ef32e9ffa2a9a1e717d9266c4eb --- /dev/null +++ b/source/libs/geometry/test/geomFuncTestUtil.cpp @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "geomFuncTestUtil.h" + +void setColumnInfo(SColumnInfo *info, int32_t colId, int32_t type, int32_t bytes) { + memset(info, 0, sizeof(SColumnInfo)); + info->colId = colId; + info->type = type; + info->bytes = bytes; +} + +void setScalarParam(SScalarParam *sclParam, int32_t type, void *valueArray, TDRowValT valTypeArray[], int32_t rowNum) { + int32_t bytes = 0; + switch (type) { + case TSDB_DATA_TYPE_NULL: { + bytes = -1; + break; + } + case TSDB_DATA_TYPE_BOOL: { + bytes = sizeof(int8_t); + break; + } + case TSDB_DATA_TYPE_TINYINT: { + bytes = sizeof(int8_t); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + bytes = sizeof(int16_t); + break; + } + case TSDB_DATA_TYPE_INT: { + bytes = sizeof(int32_t); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + bytes = sizeof(int64_t); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + bytes = sizeof(float); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + bytes = sizeof(double); + break; + } + case TSDB_DATA_TYPE_VARCHAR: { + bytes = TSDB_MAX_BINARY_LEN; + break; + } + case TSDB_DATA_TYPE_GEOMETRY: { + bytes = TSDB_MAX_GEOMETRY_LEN; + break; + } + default: { + ASSERT(0); + break; + } + } + + sclParam->columnData = (SColumnInfoData *)taosMemoryCalloc(1, sizeof(SColumnInfoData)); + sclParam->numOfRows = rowNum; + + setColumnInfo(&sclParam->columnData->info, 0, type, bytes); + colInfoDataEnsureCapacity(sclParam->columnData, rowNum, false); + + if (type != TSDB_DATA_TYPE_NULL && valueArray) { + for (int32_t i = 0; i < rowNum; ++i) { + if (tdValTypeIsNull(valTypeArray[i])) { + colDataSetNULL(sclParam->columnData, i); + } + else { + const char *val = (const char *)valueArray + (i * bytes); + colDataSetVal(sclParam->columnData, i, val, false); + } + } + } +} + +void destroyScalarParam(SScalarParam *sclParam, int32_t colNum) { + for (int32_t i = 0; i < colNum; ++i) { + colDataDestroy((sclParam + i)->columnData); + taosMemoryFree((sclParam + i)->columnData); + } + taosMemoryFree(sclParam); +} + +void makeOneScalarParam(SScalarParam **pSclParam, int32_t type, void *valueArray, TDRowValT valTypeArray[], int32_t rowNum) { + *pSclParam = (SScalarParam *)taosMemoryCalloc(1, sizeof(SScalarParam)); + setScalarParam(*pSclParam, type, valueArray, valTypeArray, rowNum); +} + +bool compareVarData(unsigned char *varData1, unsigned char *varData2) { + if (varDataLen(varData1) == 0 || varDataLen(varData2) == 0) { + return false; + } + if(varDataLen(varData1) != varDataLen(varData2)) { + return false; + } + + return (memcmp(varDataVal(varData1), varDataVal(varData2), varDataLen(varData1)) == 0); +} + +void compareVarDataColumn(SColumnInfoData *columnData1, SColumnInfoData *columnData2, int32_t rowNum) { + for (int32_t i = 0; i < rowNum; ++i) { + bool isNull1 = colDataIsNull_s(columnData1, i); + bool isNull2 = colDataIsNull_s(columnData2, i); + ASSERT_EQ((isNull1 == isNull2), true); + + if (!isNull1) { + bool res = compareVarData((unsigned char *)colDataGetData(columnData1, i), + (unsigned char *)colDataGetData(columnData2, i)); + ASSERT_EQ(res, true); + } + } +} diff --git a/source/libs/geometry/test/geomFuncTestUtil.h b/source/libs/geometry/test/geomFuncTestUtil.h new file mode 100644 index 0000000000000000000000000000000000000000..1d7c6d77be4f0d37e15711aaf74569233202066d --- /dev/null +++ b/source/libs/geometry/test/geomFuncTestUtil.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include "tdatablock.h" +#include "geomFunc.h" + +void setScalarParam(SScalarParam *sclParam, int32_t type, void *valueArray, TDRowValT valTypeArray[], int32_t rowNum); +void destroyScalarParam(SScalarParam *sclParam, int32_t colNum); + +void makeOneScalarParam(SScalarParam **pSclParam, int32_t type, void *valueArray, TDRowValT valTypeArray[], int32_t rowNum); + +void compareVarDataColumn(SColumnInfoData *columnData1, SColumnInfoData *columnData2, int32_t rowNum); + +void callGeomFromTextWrapper5(void *strArray, TDRowValT valTypeArray[], int32_t rowNum, SScalarParam *pOutputGeomFromText); diff --git a/source/libs/geometry/test/geomIoFuncTest.cpp b/source/libs/geometry/test/geomIoFuncTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fcb7a4d12a7bd753ef32f505a7c72c761e3e5c8a --- /dev/null +++ b/source/libs/geometry/test/geomIoFuncTest.cpp @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "geomFuncTestUtil.h" + +void callGeomFromText(int32_t inputType, void *strArray, TDRowValT valTypeArray[], int32_t rowNum, + SScalarParam **pInputGeomFromText, SScalarParam **pOutputGeomFromText, + int32_t expectedCode) { + makeOneScalarParam(pInputGeomFromText, inputType, strArray, valTypeArray, rowNum); + makeOneScalarParam(pOutputGeomFromText, TSDB_DATA_TYPE_GEOMETRY, 0, 0, rowNum); + + int32_t code = geomFromTextFunction(*pInputGeomFromText, 1, *pOutputGeomFromText); + ASSERT_EQ(code, expectedCode); +} + +void callGeomFromTextWrapper1(int32_t inputType, void *strArray, TDRowValT valTypeArray[], int32_t rowNum, SScalarParam **pOutputGeomFromText) { + SScalarParam *pInputGeomFromText; + callGeomFromText(inputType, strArray, valTypeArray, rowNum, &pInputGeomFromText, pOutputGeomFromText, TSDB_CODE_SUCCESS); + destroyScalarParam(pInputGeomFromText, 1); +} + +void callGeomFromTextWrapper2(void *strArray, TDRowValT valTypeArray[], int32_t rowNum, SScalarParam **pOutputGeomFromText) { + callGeomFromTextWrapper1(TSDB_DATA_TYPE_VARCHAR, strArray, valTypeArray, rowNum, pOutputGeomFromText); +} + +void callGeomFromTextWrapper3(int32_t inputType, void *strArray, TDRowValT valTypeArray[], int32_t rowNum, int32_t expectedCode) { + SScalarParam *pInputGeomFromText; + SScalarParam *pOutputGeomFromText; + + callGeomFromText(inputType, strArray, valTypeArray, rowNum, &pInputGeomFromText, &pOutputGeomFromText, expectedCode); + + destroyScalarParam(pInputGeomFromText, 1); + destroyScalarParam(pOutputGeomFromText, 1); +} + +void callGeomFromTextWrapper4(void *strArray, TDRowValT valTypeArray[], int32_t rowNum, int32_t expectedCode) { + callGeomFromTextWrapper3(TSDB_DATA_TYPE_VARCHAR, strArray, valTypeArray, rowNum, expectedCode); +} + +void callGeomFromTextWrapper5(void *strArray, TDRowValT valTypeArray[], int32_t rowNum, SScalarParam *pOutputGeomFromText) { + SScalarParam *pInputGeomFromText; + makeOneScalarParam(&pInputGeomFromText, TSDB_DATA_TYPE_VARCHAR, strArray, valTypeArray, rowNum); + + setScalarParam(pOutputGeomFromText, TSDB_DATA_TYPE_GEOMETRY, 0, 0, rowNum); + + int32_t code = geomFromTextFunction(pInputGeomFromText, 1, pOutputGeomFromText); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + destroyScalarParam(pInputGeomFromText, 1); +} + +void callAsText(int32_t inputType, void *strArray, TDRowValT valTypeArray[], int32_t rowNum, + SScalarParam **pInputAsText, SScalarParam **pOutputAsText, + int32_t expectedCode) { + makeOneScalarParam(pInputAsText, inputType, strArray, valTypeArray, rowNum); + makeOneScalarParam(pOutputAsText, TSDB_DATA_TYPE_VARCHAR, 0, 0, rowNum); + + int32_t code = geomFromTextFunction(*pInputAsText, 1, *pOutputAsText); + ASSERT_EQ(code, expectedCode); +} + +void callAsTextWrapper1(int32_t inputType, void *strArray, TDRowValT valTypeArray[], int32_t rowNum, SScalarParam **pOutputAsText) { + SScalarParam *pInputAsText; + callAsText(inputType, strArray, valTypeArray, rowNum, &pInputAsText, pOutputAsText, TSDB_CODE_SUCCESS); + destroyScalarParam(pInputAsText, 1); +} + +void callAsTextWrapper2(int32_t inputType, void *strArray, TDRowValT valTypeArray[], int32_t rowNum, int32_t expectedCode) { + SScalarParam *pInputAsText; + SScalarParam *pOutputASText; + + callAsText(inputType, strArray, valTypeArray, rowNum, &pInputAsText, &pOutputASText, expectedCode); + + destroyScalarParam(pInputAsText, 1); + destroyScalarParam(pOutputASText, 1); +} + +void callMakePointAndCompareResult(int32_t type1, void *valueArray1, TDRowValT valTypeArray1[], bool isConstant1, + int32_t type2, void *valueArray2, TDRowValT valTypeArray2[], bool isConstant2, + SScalarParam *pExpectedResult, int32_t rowNum) { + int32_t rowNum1 = isConstant1 ? 1 : rowNum; + int32_t rowNum2 = isConstant2 ? 1 : rowNum; + + SScalarParam *pInputMakePoint = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + setScalarParam(pInputMakePoint, type1, valueArray1, valTypeArray1, rowNum1); + setScalarParam(pInputMakePoint + 1, type2, valueArray2, valTypeArray2, rowNum2); + + SScalarParam *pOutputMakePoint; + makeOneScalarParam(&pOutputMakePoint, TSDB_DATA_TYPE_GEOMETRY, 0, 0, rowNum); + + int32_t code = makePointFunction(pInputMakePoint, 2, pOutputMakePoint); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + ASSERT_EQ(pOutputMakePoint->columnData->info.type, TSDB_DATA_TYPE_GEOMETRY); + ASSERT_EQ(pExpectedResult->columnData->info.type, TSDB_DATA_TYPE_GEOMETRY); + + compareVarDataColumn(pOutputMakePoint->columnData, pExpectedResult->columnData, rowNum); + + destroyScalarParam(pInputMakePoint, 2); + destroyScalarParam(pOutputMakePoint, 1); +} + +#define MAKE_POINT_FIRST_COLUMN_VALUES {2, 3, -4} +#define MAKE_POINT_SECOND_COLUMN_VALUES {5, -6, -7} + +TEST(GeomIoFuncTest, makePointFunctionTwoColumns) { + const int32_t rowNum = 3; + SScalarParam *pExpectedResult; + TDRowValT valTypeArray[rowNum] = {TD_VTYPE_NORM, TD_VTYPE_NORM, TD_VTYPE_NORM}; + + // call GeomFromText() and generate pExpectedResult to compare later + char strArray[rowNum][TSDB_MAX_BINARY_LEN]; + STR_TO_VARSTR(strArray[0], "POINT(2.0 5.0)"); + STR_TO_VARSTR(strArray[1], "POINT(3.0 -6.0)"); + STR_TO_VARSTR(strArray[2], "POINT(-4.0 -7.0)"); + callGeomFromTextWrapper2(strArray, valTypeArray, rowNum, &pExpectedResult); + + // call MakePoint() with TINYINT and SMALLINT, and compare with result of GeomFromText() + int8_t tinyIntArray1[rowNum] = MAKE_POINT_FIRST_COLUMN_VALUES; + int16_t smallIntArray2[rowNum] = MAKE_POINT_SECOND_COLUMN_VALUES; + callMakePointAndCompareResult(TSDB_DATA_TYPE_TINYINT, tinyIntArray1, valTypeArray, false, + TSDB_DATA_TYPE_SMALLINT, smallIntArray2, valTypeArray, false, + pExpectedResult, rowNum); + + // call MakePoint() with INT and BIGINT, and compare with result of GeomFromText() + int32_t intArray1[rowNum] = MAKE_POINT_FIRST_COLUMN_VALUES; + int64_t bigIntArray2[rowNum] = MAKE_POINT_SECOND_COLUMN_VALUES; + callMakePointAndCompareResult(TSDB_DATA_TYPE_INT, intArray1, valTypeArray, false, + TSDB_DATA_TYPE_BIGINT, bigIntArray2, valTypeArray, false, + pExpectedResult, rowNum); + + // call MakePoint() with FLOAT and DOUBLE, and compare with result of GeomFromText() + float floatArray1[rowNum] = MAKE_POINT_FIRST_COLUMN_VALUES; + double doubleArray2[rowNum] = MAKE_POINT_SECOND_COLUMN_VALUES; + callMakePointAndCompareResult(TSDB_DATA_TYPE_FLOAT, floatArray1, valTypeArray, false, + TSDB_DATA_TYPE_DOUBLE, doubleArray2, valTypeArray, false, + pExpectedResult, rowNum); + + destroyScalarParam(pExpectedResult, 1); +} + +TEST(GeomIoFuncTest, makePointFunctionConstant) { + const int32_t rowNum = 3; + SScalarParam *pExpectedResult; + TDRowValT valTypeArray[rowNum] = {TD_VTYPE_NORM, TD_VTYPE_NORM, TD_VTYPE_NORM}; + + // 1. call GeomFromText() and generate pExpectedResult with first constant + char strArray[rowNum][TSDB_MAX_BINARY_LEN]; + STR_TO_VARSTR(strArray[0], "POINT(3.0 5.0)"); + STR_TO_VARSTR(strArray[1], "POINT(3.0 -6.0)"); + STR_TO_VARSTR(strArray[2], "POINT(3.0 -7.0)"); + callGeomFromTextWrapper2(strArray, valTypeArray, rowNum, &pExpectedResult); + + // call MakePoint() with TINYINT constant and BIGINT column, and compare with result of GeomFromText() + int8_t tinyIntConstant = 3; + int64_t bigIntArray[rowNum] = MAKE_POINT_SECOND_COLUMN_VALUES; + callMakePointAndCompareResult(TSDB_DATA_TYPE_TINYINT, &tinyIntConstant, valTypeArray, true, + TSDB_DATA_TYPE_BIGINT, bigIntArray, valTypeArray, false, + pExpectedResult, rowNum); + + destroyScalarParam(pExpectedResult, 1); + + // 2. call GeomFromText() and generate pExpectedResult with second constant + STR_TO_VARSTR(strArray[0], "POINT(2.0 3.0)"); + STR_TO_VARSTR(strArray[1], "POINT(3.0 3.0)"); + STR_TO_VARSTR(strArray[2], "POINT(-4.0 3.0)"); + callGeomFromTextWrapper2(strArray, valTypeArray, rowNum, &pExpectedResult); + + // call MakePoint() with INT column and FLOAT constant, and compare with result of GeomFromText() + int32_t intArray[rowNum] = MAKE_POINT_FIRST_COLUMN_VALUES; + float floatConstant = 3; + callMakePointAndCompareResult(TSDB_DATA_TYPE_INT, intArray, valTypeArray, false, + TSDB_DATA_TYPE_FLOAT, &floatConstant, valTypeArray, true, + pExpectedResult, rowNum); + + destroyScalarParam(pExpectedResult, 1); +} + +TEST(GeomIoFuncTest, makePointFunctionWithNull) { + const int32_t rowNum = 3; + SScalarParam *pExpectedResult; + TDRowValT valTypeNormArray[rowNum] = {TD_VTYPE_NORM, TD_VTYPE_NORM, TD_VTYPE_NORM}; + + // call GeomFromText() and generate pExpectedResult with all NULL values + char strArray[rowNum][TSDB_MAX_BINARY_LEN]; + TDRowValT valTypeNullArray[rowNum] = {TD_VTYPE_NULL, TD_VTYPE_NULL, TD_VTYPE_NULL}; + callGeomFromTextWrapper2(strArray, valTypeNullArray, rowNum, &pExpectedResult); + + // 1. call MakePoint() with NULL type and INT column, and compare all NULL results + int64_t intArray[rowNum] = MAKE_POINT_SECOND_COLUMN_VALUES; + callMakePointAndCompareResult(TSDB_DATA_TYPE_NULL, 0, 0, true, + TSDB_DATA_TYPE_INT, intArray, valTypeNormArray, false, + pExpectedResult, rowNum); + // swap params and compare + callMakePointAndCompareResult(TSDB_DATA_TYPE_INT, intArray, valTypeNormArray, false, + TSDB_DATA_TYPE_NULL, 0, 0, true, + pExpectedResult, rowNum); + + // call MakePoint() with SMALLINT NULL constant and BIGINT column, and compare all NULL results + int16_t smallIntConstant = 0; + int64_t bigIntArray[rowNum] = MAKE_POINT_SECOND_COLUMN_VALUES; + callMakePointAndCompareResult(TSDB_DATA_TYPE_SMALLINT, &smallIntConstant, valTypeNullArray, true, + TSDB_DATA_TYPE_BIGINT, bigIntArray, valTypeNormArray, false, + pExpectedResult, rowNum); + // swap params and compare + callMakePointAndCompareResult(TSDB_DATA_TYPE_BIGINT, bigIntArray, valTypeNormArray, false, + TSDB_DATA_TYPE_SMALLINT, &smallIntConstant, valTypeNullArray, true, + pExpectedResult, rowNum); + + destroyScalarParam(pExpectedResult, 1); + + // 2. call GeomFromText() and generate pExpectedResult with NULL value + STR_TO_VARSTR(strArray[0], "POINT(2.0 5.0)"); + STR_TO_VARSTR(strArray[2], "POINT(-4.0 -7.0)"); + TDRowValT valTypeWithNullArray[rowNum] = {TD_VTYPE_NORM, TD_VTYPE_NULL, TD_VTYPE_NORM}; + callGeomFromTextWrapper2(strArray, valTypeWithNullArray, rowNum, &pExpectedResult); + + // call MakePoint() with TINYINT column with NULL value and FLOAT column, and compare results with NULL value + int8_t tinyIntArray[rowNum] = MAKE_POINT_FIRST_COLUMN_VALUES; + float floatArray[rowNum] = MAKE_POINT_SECOND_COLUMN_VALUES; + callMakePointAndCompareResult(TSDB_DATA_TYPE_TINYINT, tinyIntArray, valTypeWithNullArray, false, + TSDB_DATA_TYPE_FLOAT, floatArray, valTypeNormArray, false, + pExpectedResult, rowNum); + + // call MakePoint() with SMALLINT column and DOUBLE column with NULL value, and compare results with NULL value + int16_t smallIntArray[rowNum] = MAKE_POINT_FIRST_COLUMN_VALUES; + double doubleArray[rowNum] = MAKE_POINT_SECOND_COLUMN_VALUES; + callMakePointAndCompareResult(TSDB_DATA_TYPE_SMALLINT, smallIntArray, valTypeNormArray, false, + TSDB_DATA_TYPE_DOUBLE, doubleArray, valTypeWithNullArray, false, + pExpectedResult, rowNum); + + destroyScalarParam(pExpectedResult, 1); +} + +TEST(GeomIoFuncTest, geomFromTextFunction) { + const int32_t rowNum = 4; + char strArray[rowNum][TSDB_MAX_BINARY_LEN]; + TDRowValT valTypeNormArray[rowNum] = {TD_VTYPE_NORM, TD_VTYPE_NORM, TD_VTYPE_NORM, TD_VTYPE_NORM}; + + // column input + // input of GeomFromText (with NULL value) and output of AsText should be same after calling GeomFromText() and AsText() + SScalarParam *pInputGeomFromText; + SScalarParam *pOutputGeomFromText; + SScalarParam *pOutputAsText; + + STR_TO_VARSTR(strArray[0], "POINT (2.000000 5.000000)"); + STR_TO_VARSTR(strArray[2], "LINESTRING (3.000000 -6.000000, -71.160837 42.259113)"); + STR_TO_VARSTR(strArray[3], "POLYGON ((-71.177658 42.390290, -71.177682 42.390370, -71.177606 42.390382, -71.177582 42.390303, -71.177658 42.390290))"); + TDRowValT valTypeWithNullArray[rowNum] = {TD_VTYPE_NORM, TD_VTYPE_NULL, TD_VTYPE_NORM, TD_VTYPE_NORM}; + callGeomFromText(TSDB_DATA_TYPE_VARCHAR, strArray, valTypeWithNullArray, rowNum, &pInputGeomFromText, &pOutputGeomFromText, TSDB_CODE_SUCCESS); + + makeOneScalarParam(&pOutputAsText, TSDB_DATA_TYPE_VARCHAR, 0, 0, rowNum); + int32_t code = asTextFunction(pOutputGeomFromText, 1, pOutputAsText); // pOutputGeomFromText is input for AsText() + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + compareVarDataColumn(pInputGeomFromText->columnData, pOutputAsText->columnData, rowNum); + + destroyScalarParam(pInputGeomFromText, 1); + destroyScalarParam(pOutputGeomFromText, 1); + destroyScalarParam(pOutputAsText, 1); + + // empty input + STR_TO_VARSTR(strArray[0], ""); + callGeomFromTextWrapper4(strArray, valTypeNormArray, 1, TSDB_CODE_SUCCESS); + + // NULL type input + callGeomFromTextWrapper1(TSDB_DATA_TYPE_NULL, 0, 0, 1, &pOutputGeomFromText); + ASSERT_EQ(colDataIsNull_s(pOutputGeomFromText->columnData, 0), true); + destroyScalarParam(pOutputGeomFromText, 1); + + // wrong type input [ToDo] make sure it is handled in geomFunc + int32_t intInput = 3; + callGeomFromTextWrapper3(TSDB_DATA_TYPE_INT, &intInput, valTypeNormArray, 1, TSDB_CODE_FUNC_FUNTION_PARA_VALUE); + + // wrong content input + STR_TO_VARSTR(strArray[0], "POIN(2 5)"); // lack of the last letter of 'POINT' + callGeomFromTextWrapper4(strArray, valTypeNormArray, 1, TSDB_CODE_FUNC_FUNTION_PARA_VALUE); + STR_TO_VARSTR(strArray[0], "LINESTRING(3 -6.1,-7.1 4.2,)"); // redundant comma at the end + callGeomFromTextWrapper4(strArray, valTypeNormArray, 1, TSDB_CODE_FUNC_FUNTION_PARA_VALUE); + STR_TO_VARSTR(strArray[0], "POLYGON((-71.1 42.3,-71.2 42.4,-71.3 42.5,-71.1 42.8))"); // the first point and last one are not same + callGeomFromTextWrapper4(strArray, valTypeNormArray, 1, TSDB_CODE_FUNC_FUNTION_PARA_VALUE); +} + +TEST(GeomIoFuncTest, asTextFunction) { + // column input has been tested in geomFromTextFunction + + TDRowValT valTypeArray[1] = {TD_VTYPE_NORM}; + + // empty input + char strInput[TSDB_MAX_BINARY_LEN]; + STR_TO_VARSTR(strInput, ""); + SScalarParam *pOutputAsText; + callAsTextWrapper1(TSDB_DATA_TYPE_GEOMETRY, strInput, valTypeArray, 1, &pOutputAsText); + ASSERT_EQ(colDataIsNull_s(pOutputAsText->columnData, 0), true); + destroyScalarParam(pOutputAsText, 1); + + // NULL type input + callAsTextWrapper1(TSDB_DATA_TYPE_NULL, 0, 0, 1, &pOutputAsText); + ASSERT_EQ(colDataIsNull_s(pOutputAsText->columnData, 0), true); + destroyScalarParam(pOutputAsText, 1); + + // wrong type input [ToDo] make sure it is handled in geomFunc + int32_t intInput = 3; + callAsTextWrapper2(TSDB_DATA_TYPE_INT, &intInput, valTypeArray, 1, TSDB_CODE_FUNC_FUNTION_PARA_VALUE); + + // wrong content input + STR_TO_VARSTR(strInput, "XXX"); + callAsTextWrapper2(TSDB_DATA_TYPE_GEOMETRY, strInput, valTypeArray, 1, TSDB_CODE_FUNC_FUNTION_PARA_VALUE); +} diff --git a/source/libs/geometry/test/geomRelationFuncTest.cpp b/source/libs/geometry/test/geomRelationFuncTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1084c941787168ef7edd8b6ecd114a77773245bd --- /dev/null +++ b/source/libs/geometry/test/geomRelationFuncTest.cpp @@ -0,0 +1,239 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "geomFuncTestUtil.h" + +void callGeomRelationFuncAndCompareResult(FScalarExecProcess geomRelationFunc, + SScalarParam *pInput, int32_t rowNum, + int32_t expectedCode, int8_t expectedResult[]) { + SScalarParam *pOutput; + makeOneScalarParam(&pOutput, TSDB_DATA_TYPE_BOOL, 0, 0, rowNum); + int32_t code = geomRelationFunc(pInput, 2, pOutput); + ASSERT_EQ(code, expectedCode); + + if (code == TSDB_CODE_SUCCESS) { + int8_t res = -1; + for (int32_t i = 0; i < rowNum; ++i) { + bool isNull1 = colDataIsNull_s(pOutput->columnData, i); + if (isNull1) { + res = -1; + } + else { + res = *(bool*)colDataGetData(pOutput->columnData, i); + } + + ASSERT_EQ(res, expectedResult[i]); + } + } + + destroyScalarParam(pOutput, 1); + destroyScalarParam(pInput, 2); +} + +/* +-- Use the following SQL to get expected results for all relation functions in PostgreSQL with PostGIS +WITH geom_str AS +(SELECT 'POINT(3.5 7.0)' AS g1, 'POINT(3.5 7.0)' AS g2 +UNION ALL +SELECT 'POINT(3.0 3.0)' AS g1, 'LINESTRING(1.0 1.0, 2.0 2.0, 5.0 6.0)' AS g2 +UNION ALL +SELECT 'POINT(3.0 6.0)' AS g1, 'POLYGON((3.0 6.0, 5.0 6.0, 5.0 8.0, 3.0 8.0, 3.0 6.0))' AS g2 +UNION ALL +SELECT 'LINESTRING(1.0 1.0, 2.0 2.0, 5.0 5.0)' AS g1, 'LINESTRING(1.0 4.0, 2.0 3.0, 5.0 0.0)' AS g2 +UNION ALL +SELECT 'LINESTRING(3.0 7.0, 4.0 7.0, 5.0 7.0)' AS g1, 'POLYGON((3.0 6.0, 5.0 6.0, 5.0 8.0, 3.0 8.0, 3.0 6.0))' AS g2 +UNION ALL +SELECT 'POLYGON((3.0 6.0, 5.0 6.0, 5.0 8.0, 3.0 8.0, 3.0 6.0))' AS g1, 'POLYGON((5.0 6.0, 7.0 6.0, 7.0 8.0, 5.0 8.0, 5.0 6.0))' AS g2 +) +SELECT ST_Intersects(g1, g2), ST_Equals(g1, g2), ST_Touches(g1, g2), ST_Covers(g1, g2), ST_Contains(g1, g2), ST_ContainsProperly(g1, g2) FROM geom_str +*/ +void geomRelationFuncTest(FScalarExecProcess geomRelationFunc, int8_t expectedResults[6][6]) { + const int32_t rowNum = 6; + + char strArray1[rowNum][TSDB_MAX_BINARY_LEN]; + STR_TO_VARSTR(strArray1[0], "POINT(3.5 7.0)"); + STR_TO_VARSTR(strArray1[1], "POINT(3.0 3.0)"); + STR_TO_VARSTR(strArray1[2], "POINT(3.0 6.0)"); + STR_TO_VARSTR(strArray1[3], "LINESTRING(1.0 1.0, 2.0 2.0, 5.0 5.0)"); + STR_TO_VARSTR(strArray1[4], "LINESTRING(3.0 7.0, 4.0 7.0, 5.0 7.0)"); + STR_TO_VARSTR(strArray1[5], "POLYGON((3.0 6.0, 5.0 6.0, 5.0 8.0, 3.0 8.0, 3.0 6.0))"); + TDRowValT valTypeArray1[rowNum] = {TD_VTYPE_NORM, TD_VTYPE_NORM, TD_VTYPE_NORM, TD_VTYPE_NORM, TD_VTYPE_NORM, TD_VTYPE_NORM}; + + char strArray2[rowNum][TSDB_MAX_BINARY_LEN]; + STR_TO_VARSTR(strArray2[0], "POINT(3.5 7.0)"); + STR_TO_VARSTR(strArray2[1], "LINESTRING(1.0 1.0, 2.0 2.0, 5.0 6.0)"); + STR_TO_VARSTR(strArray2[2], "POLYGON((3.0 6.0, 5.0 6.0, 5.0 8.0, 3.0 8.0, 3.0 6.0))"); + STR_TO_VARSTR(strArray2[3], "LINESTRING(1.0 4.0, 2.0 3.0, 5.0 0.0)"); + STR_TO_VARSTR(strArray2[4], "POLYGON((3.0 6.0, 5.0 6.0, 5.0 8.0, 3.0 8.0, 3.0 6.0))"); + STR_TO_VARSTR(strArray2[5], "POLYGON((5.0 6.0, 7.0 6.0, 7.0 8.0, 5.0 8.0, 5.0 6.0))"); + TDRowValT valTypeArray2[rowNum] = {TD_VTYPE_NORM, TD_VTYPE_NORM, TD_VTYPE_NORM, TD_VTYPE_NORM, TD_VTYPE_NORM, TD_VTYPE_NORM}; + + // two columns input + SScalarParam *pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + callGeomFromTextWrapper5(strArray1, valTypeArray1, rowNum, pInput); //pInput come from GeomFromText() + callGeomFromTextWrapper5(strArray2, valTypeArray2, rowNum, pInput + 1); + callGeomRelationFuncAndCompareResult(geomRelationFunc, pInput, rowNum, TSDB_CODE_SUCCESS, expectedResults[0]); + + // swap two columns + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + callGeomFromTextWrapper5(strArray2, valTypeArray2, rowNum, pInput); + callGeomFromTextWrapper5(strArray1, valTypeArray1, rowNum, pInput + 1); + callGeomRelationFuncAndCompareResult(geomRelationFunc, pInput, rowNum, TSDB_CODE_SUCCESS, expectedResults[1]); + + // constant and column input + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + callGeomFromTextWrapper5(strArray1, valTypeArray1, 1, pInput); + callGeomFromTextWrapper5(strArray2, valTypeArray2, rowNum, pInput + 1); + callGeomRelationFuncAndCompareResult(geomRelationFunc, pInput, rowNum, TSDB_CODE_SUCCESS, expectedResults[2]); + + // column and constant input + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + callGeomFromTextWrapper5(strArray1, valTypeArray1, rowNum, pInput); + callGeomFromTextWrapper5(strArray2, valTypeArray2, 1, pInput + 1); + callGeomRelationFuncAndCompareResult(geomRelationFunc, pInput, rowNum, TSDB_CODE_SUCCESS, expectedResults[3]); + + // two constants input + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + callGeomFromTextWrapper5(strArray1, valTypeArray1, 1, pInput); + callGeomFromTextWrapper5(strArray2, valTypeArray2, 1, pInput + 1); + callGeomRelationFuncAndCompareResult(geomRelationFunc, pInput, 1, TSDB_CODE_SUCCESS, expectedResults[4]); + + // two columns with NULL value input + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + valTypeArray1[2] = TD_VTYPE_NULL; + valTypeArray2[4] = TD_VTYPE_NULL; + callGeomFromTextWrapper5(strArray1, valTypeArray1, rowNum, pInput); + callGeomFromTextWrapper5(strArray2, valTypeArray2, rowNum, pInput + 1); + callGeomRelationFuncAndCompareResult(geomRelationFunc, pInput, rowNum, TSDB_CODE_SUCCESS, expectedResults[5]); + + // first NULL type input + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + setScalarParam(pInput, TSDB_DATA_TYPE_NULL, 0, 0, 1); + callGeomFromTextWrapper5(strArray2, valTypeArray2, rowNum, pInput + 1); + int8_t expectedResultNullType[rowNum] = {-1, -1, -1, -1, -1, -1}; + callGeomRelationFuncAndCompareResult(geomRelationFunc, pInput, rowNum, TSDB_CODE_SUCCESS, expectedResultNullType); + + // second NULL type input + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + callGeomFromTextWrapper5(strArray1, valTypeArray1, rowNum, pInput); + setScalarParam(pInput + 1, TSDB_DATA_TYPE_NULL, 0, 0, 1); + callGeomRelationFuncAndCompareResult(geomRelationFunc, pInput, rowNum, TSDB_CODE_SUCCESS, expectedResultNullType); + + // first empty content input + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + char strInput[TSDB_MAX_BINARY_LEN]; + STR_TO_VARSTR(strInput, ""); + setScalarParam(pInput, TSDB_DATA_TYPE_GEOMETRY, strInput, valTypeArray1, 1); + callGeomFromTextWrapper5(strArray2, valTypeArray2, rowNum, pInput + 1); + callGeomRelationFuncAndCompareResult(geomRelationFunc, pInput, rowNum, TSDB_CODE_SUCCESS, expectedResultNullType); + + // first wrong type input + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + int32_t intInput = 3; + setScalarParam(pInput, TSDB_DATA_TYPE_INT, &intInput, valTypeArray1, 1); + callGeomFromTextWrapper5(strArray2, valTypeArray2, rowNum, pInput + 1); + callGeomRelationFuncAndCompareResult(geomRelationFunc, pInput, rowNum, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, 0); + + // second wrong content input + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + STR_TO_VARSTR(strInput, "XXX"); + callGeomFromTextWrapper5(strArray1, valTypeArray1, rowNum, pInput); + setScalarParam(pInput + 1, TSDB_DATA_TYPE_GEOMETRY, strInput, valTypeArray2, 1); + callGeomRelationFuncAndCompareResult(geomRelationFunc, pInput, rowNum, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, 0); +} + +TEST(GeomRelationFuncTest, intersectsFunction) { + // 1: true, 0: false, -1: null + int8_t expectedResults[6][6] = { + {1, 0, 1, 1, 1, 1}, // two columns + {1, 0, 1, 1, 1, 1}, // two columns swpped + {1, 0, 1, 0, 1, 0}, // first constant + {1, 0, 0, 0, 1, 1}, // second constant + {1}, // two constant + {1, 0, -1, 1, -1, 1} // with Null value + }; + + geomRelationFuncTest(intersectsFunction, expectedResults); +} + +TEST(GeomRelationFuncTest, equalsFunction) { + // 1: true, 0: false, -1: null + int8_t expectedResults[6][6] = { + {1, 0, 0, 0, 0, 0}, // two columns + {1, 0, 0, 0, 0, 0}, // two columns swapped + {1, 0, 0, 0, 0, 0}, // first constant + {1, 0, 0, 0, 0, 0}, // second constant + {1}, // two constant + {1, 0, -1, 0, -1, 0} // with Null value + }; + + geomRelationFuncTest(equalsFunction, expectedResults); +} + +TEST(GeomRelationFuncTest, touchesFunction) { + // 1: true, 0: false, -1: null + int8_t expectedResults[6][6] = { + {0, 0, 1, 0, 0, 1}, // two columns + {0, 0, 1, 0, 0, 1}, // two columns swapped + {0, 0, 0, 0, 0, 0}, // first constant + {0, 0, 0, 0, 0, 0}, // second constant + {0}, // two constant + {0, 0, -1, 0, -1, 1} // with Null value + }; + + geomRelationFuncTest(touchesFunction, expectedResults); +} + +TEST(GeomRelationFuncTest, coversFunction) { + // 1: true, 0: false, -1: null + int8_t expectedResults[6][6] = { + {1, 0, 0, 0, 0, 0}, // two columns + {1, 0, 1, 0, 1, 0}, // two columns swapped + {1, 0, 0, 0, 0, 0}, // first constant + {1, 0, 0, 0, 1, 1}, // second constant + {1}, // two constant + {1, 0, -1, 0, -1, 0} // with Null value + }; + + geomRelationFuncTest(coversFunction, expectedResults); +} + +TEST(GeomRelationFuncTest, containsFunction) { + // 1: true, 0: false, -1: null + int8_t expectedResults[6][6] = { + {1, 0, 0, 0, 0, 0}, // two columns + {1, 0, 0, 0, 1, 0}, // two columns swapped + {1, 0, 0, 0, 0, 0}, // first constant + {1, 0, 0, 0, 1, 1}, // second constant + {1}, // two constant + {1, 0, -1, 0, -1, 0} // with Null value + }; + + geomRelationFuncTest(containsFunction, expectedResults); +} + +TEST(GeomRelationFuncTest, containsProperlyFunction) { + // 1: true, 0: false, -1: null + int8_t expectedResults[6][6] = { + {1, 0, 0, 0, 0, 0}, // two columns + {1, 0, 0, 0, 0, 0}, // two columns swapped + {1, 0, 0, 0, 0, 0}, // first constant + {1, 0, 0, 0, 1, 1}, // second constant + {1}, // two constant + {1, 0, -1, 0, -1, 0} // with Null value + }; + + geomRelationFuncTest(containsProperlyFunction, expectedResults); +} diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index 0c2ce37c4030dbb018eb7871f396be60aa0fc076..6f3f48610c866f02b1d02f88bfc22d040a88a575 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -12,7 +12,6 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common - PUBLIC vnode PUBLIC nodes PUBLIC scalar PUBLIC function diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 8b0e7125530fd557648d97106fc106132972c71b..a6fbc73332be3fa9d47bc232502bc696820d9869 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -738,7 +738,7 @@ static MemTable* idxInternalCacheCreate(int8_t type) { MemTable* tbl = taosMemoryCalloc(1, sizeof(MemTable)); idxMemRef(tbl); - // if (ttype == TSDB_DATA_TYPE_BINARY || ttype == TSDB_DATA_TYPE_NCHAR) { + // if (ttype == TSDB_DATA_TYPE_BINARY || ttype == TSDB_DATA_TYPE_NCHAR || ttype == TSDB_DATA_TYPE_GEOMETRY) { tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, ttype, MAX_INDEX_KEY_LEN, cmpFn, SL_ALLOW_DUP_KEY, idxCacheTermGet); //} return tbl; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index c2ac7f4478a6200d9408432e591132fed87a0208..a0310a9f0ebe130eb3aa7cfbbb169a41dedb04ca 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -76,7 +76,7 @@ char* idxInt2str(int64_t val, char* dst, int radix) { return dst - 1; } __compar_fn_t idxGetCompar(int8_t type) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) { return (__compar_fn_t)strcmp; } return getComparFunc(type, 0); @@ -107,7 +107,8 @@ static FORCE_INLINE TExeCond tCompareEqual(void* a, void* b, int8_t type) { return tCompare(func, QUERY_TERM, a, b, type); } TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) { - if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY) { + if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || + dtype == TSDB_DATA_TYPE_VARBINARY || dtype == TSDB_DATA_TYPE_GEOMETRY) { return tDoCompare(func, cmptype, a, b); } #if 1 @@ -354,7 +355,8 @@ int32_t idxConvertData(void* src, int8_t type, void** dst) { break; } - case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY + case TSDB_DATA_TYPE_GEOMETRY: { tlen = taosEncodeBinary(NULL, src, strlen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, strlen(src)); @@ -444,7 +446,8 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { *dst = (char*)*dst - tlen; break; } - case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY + case TSDB_DATA_TYPE_GEOMETRY: { tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 02ed0d2d05270511244b329c3e05c1b8a311035a..bfdcd2b030369033c9f350795107eab95fe8cedd 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "filter.h" #include "index.h" #include "indexComm.h" #include "indexInt.h" @@ -20,7 +21,6 @@ #include "querynodes.h" #include "scalar.h" #include "tdatablock.h" -#include "vnode.h" // clang-format off #define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) @@ -67,9 +67,8 @@ typedef union { typedef struct SIFParam { SHashObj *pFilter; - - SArray *result; - char *condValue; + SArray *result; + char *condValue; SIdxFltStatus status; uint8_t colValType; @@ -79,6 +78,7 @@ typedef struct SIFParam { char colName[TSDB_COL_NAME_LEN * 2 + 4]; SIndexMetaArg arg; + SMetaDataFilterAPI api; } SIFParam; typedef struct SIFCtx { @@ -86,6 +86,7 @@ typedef struct SIFCtx { SHashObj *pRes; /* element is SIFParam */ bool noExec; // true: just iterate condition tree, and add hint to executor plan SIndexMetaArg arg; + SMetaDataFilterAPI *pAPI; } SIFCtx; static FORCE_INLINE int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { @@ -288,6 +289,8 @@ static int32_t sifInitParamValByCol(SNode *r, SNode *l, SIFParam *param, SIFCtx } static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { param->status = SFLT_COARSE_INDEX; + param->api = *ctx->pAPI; + switch (nodeType(node)) { case QUERY_NODE_VALUE: { SValueNode *vn = (SValueNode *)node; @@ -364,6 +367,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } } + SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam)); if (NULL == paramList) { @@ -635,6 +639,10 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP ret = indexJsonSearch(arg->ivtIdx, mtm, output->result); indexMultiTermQueryDestroy(mtm); } else { + if (left->colValType == TSDB_DATA_TYPE_GEOMETRY || right->colValType == TSDB_DATA_TYPE_GEOMETRY) { + return TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR; + } + bool reverse = false, equal = false; FilterFunc filterFunc = sifGetFilterFunc(qtype, &reverse, &equal); @@ -659,7 +667,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP } else { if (sifSetFltParam(left, right, &typedata, ¶m) != 0) return -1; } - ret = metaFilterTableIds(arg->metaEx, ¶m, output->result); + ret = left->api.metaFilterTableIds(arg->metaEx, ¶m, output->result); } return ret; } @@ -972,8 +980,9 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { if (pNode == NULL || pDst == NULL) { return TSDB_CODE_QRY_INVALID_INPUT; } + int32_t code = 0; - SIFCtx ctx = {.code = 0, .noExec = false, .arg = pDst->arg}; + SIFCtx ctx = {.code = 0, .noExec = false, .arg = pDst->arg, .pAPI = &pDst->api}; ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { @@ -1006,13 +1015,13 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { return code; } -static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { +static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status, SMetaDataFilterAPI* pAPI) { int32_t code = TSDB_CODE_SUCCESS; if (pNode == NULL) { return TSDB_CODE_QRY_INVALID_INPUT; } - SIFCtx ctx = {.code = 0, .noExec = true}; + SIFCtx ctx = {.code = 0, .noExec = true, .pAPI = pAPI}; ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { indexError("index-filter failed to taosHashInit"); @@ -1044,8 +1053,8 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { return code; } -int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SIdxFltStatus *status) { - SIdxFltStatus st = idxGetFltStatus(pFilterNode); +int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SIdxFltStatus *status, SMetaDataFilterAPI* pAPI) { + SIdxFltStatus st = idxGetFltStatus(pFilterNode, pAPI); if (st == SFLT_NOT_INDEX) { *status = st; return 0; @@ -1054,7 +1063,7 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SFilterInfo *filter = NULL; SArray *output = taosArrayInit(8, sizeof(uint64_t)); - SIFParam param = {.arg = *metaArg, .result = output, .status = SFLT_NOT_INDEX}; + SIFParam param = {.arg = *metaArg, .result = output, .status = SFLT_NOT_INDEX, .api = *pAPI}; int32_t code = sifCalculate((SNode *)pFilterNode, ¶m); if (code != 0) { sifFreeParam(¶m); @@ -1071,13 +1080,13 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, return TSDB_CODE_SUCCESS; } -SIdxFltStatus idxGetFltStatus(SNode *pFilterNode) { +SIdxFltStatus idxGetFltStatus(SNode *pFilterNode, SMetaDataFilterAPI* pAPI) { SIdxFltStatus st = SFLT_NOT_INDEX; if (pFilterNode == NULL) { return SFLT_NOT_INDEX; } - if (sifGetFltHint((SNode *)pFilterNode, &st) != TSDB_CODE_SUCCESS) { + if (sifGetFltHint((SNode *)pFilterNode, &st, pAPI) != TSDB_CODE_SUCCESS) { st = SFLT_NOT_INDEX; } return st; diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 9e7ed5210425520f7b7c02a7f9b2edc6d98756ed..e18d0bbad3a99ef089a6c94111f3d148461d954a 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -29,7 +29,10 @@ typedef struct { char buf[0]; } SDataBlock; -static void deleteDataBlockFromLRU(const void* key, size_t keyLen, void* value) { taosMemoryFree(value); } +static void deleteDataBlockFromLRU(const void* key, size_t keyLen, void* value, void* ud) { + (void)ud; + taosMemoryFree(value); +} static FORCE_INLINE void idxGenLRUKey(char* buf, const char* path, int32_t blockId) { char* p = buf; @@ -136,7 +139,7 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of memcpy(buf + total, blk->buf + blkOffset, nread); LRUStatus s = taosLRUCacheInsert(ctx->lru, key, strlen(key), blk, cacheMemSize, deleteDataBlockFromLRU, NULL, - TAOS_LRU_PRIORITY_LOW); + TAOS_LRU_PRIORITY_LOW, NULL); if (s != TAOS_LRU_STATUS_OK) { return -1; } diff --git a/source/libs/index/src/indexJson.c b/source/libs/index/src/indexJson.c index 32b794cb71d6d1e179b4f43cf041eff3e6d300a9..1b4f847919034b5c1103ccf86802276107afb1b2 100644 --- a/source/libs/index/src/indexJson.c +++ b/source/libs/index/src/indexJson.c @@ -25,7 +25,7 @@ int indexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) { if (p->colType == TSDB_DATA_TYPE_BOOL) { p->colType = TSDB_DATA_TYPE_INT; } else if (p->colType == TSDB_DATA_TYPE_VARCHAR || p->colType == TSDB_DATA_TYPE_NCHAR || - p->colType == TSDB_DATA_TYPE_BINARY) { + p->colType == TSDB_DATA_TYPE_BINARY || p->colType == TSDB_DATA_TYPE_GEOMETRY) { // p->colType = TSDB_DATA_TYPE_NCHAR; } else { p->colType = TSDB_DATA_TYPE_DOUBLE; @@ -43,7 +43,7 @@ int indexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *res if (p->colType == TSDB_DATA_TYPE_BOOL) { p->colType = TSDB_DATA_TYPE_INT; } else if (p->colType == TSDB_DATA_TYPE_VARCHAR || p->colType == TSDB_DATA_TYPE_NCHAR || - p->colType == TSDB_DATA_TYPE_BINARY) { + p->colType == TSDB_DATA_TYPE_BINARY || p->colType == TSDB_DATA_TYPE_GEOMETRY) { // p->colType = TSDB_DATA_TYPE_NCHAR; } else { p->colType = TSDB_DATA_TYPE_DOUBLE; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index cdd1cc738693bf6dcc9c87b99b153370838a44be..9b6ba403184c433459d4131f56410c62750707be 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -545,7 +545,8 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { int8_t colType = tw->header.colType; colType = IDX_TYPE_GET_TYPE(colType); - if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { + if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR || + colType == TSDB_DATA_TYPE_GEOMETRY) { fn = tfileStrCompare; } else { fn = getComparFunc(colType, 0); diff --git a/source/libs/index/test/indexBench.cc b/source/libs/index/test/indexBench.cc index b828be0ffe97ee94c6b19e52c71d049ae023b66a..78c687029c7020fef8024b816bc85e5494a4ff72 100644 --- a/source/libs/index/test/indexBench.cc +++ b/source/libs/index/test/indexBench.cc @@ -87,7 +87,7 @@ SIndexTerm *indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colT int16_t sz = nColVal; memcpy(buf, (uint16_t *)&sz, 2); memcpy(buf + 2, colVal, nColVal); - if (colType == TSDB_DATA_TYPE_BINARY) { + if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_GEOMETRY) { return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf)); } else { return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal); diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 08bf84ff60fdc07393abf546630c67dd52f6abc1..612ce107c7bc0c6bbfea23c61dc653f39d52cd14 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -574,7 +574,7 @@ SIndexTerm* indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colT int16_t sz = nColVal; memcpy(buf, (uint16_t*)&sz, 2); memcpy(buf + 2, colVal, nColVal); - if (colType == TSDB_DATA_TYPE_BINARY) { + if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_GEOMETRY) { return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf)); } else { return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 8ae3fd41357da90b426eed6f2cd8b1c8c358d154..0e7698079952355fd9922f1cd219a255215cfbbd 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -24,7 +24,7 @@ SIndexTerm* indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colT int16_t sz = nColVal; memcpy(buf, (uint16_t*)&sz, 2); memcpy(buf + 2, colVal, nColVal); - if (colType == TSDB_DATA_TYPE_BINARY) { + if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_GEOMETRY) { return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf)); } else { return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal); diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 949e91198adcb69301fbc177b721289d6d736651..56cf0a2b51002f8878eda24d0aaf538e1a6a627f 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -468,9 +468,6 @@ static void monGenLogJson(SMonInfo *pMonitor) { return; } - SJson *pLogsJson = tjsonAddArrayToObject(pJson, "logs"); - if (pLogsJson == NULL) return; - SMonLogs *logs[6]; logs[0] = &pMonitor->log; logs[1] = &pMonitor->mmInfo.log; @@ -490,22 +487,6 @@ static void monGenLogJson(SMonInfo *pMonitor) { numOfInfoLogs += pLog->numOfInfoLogs; numOfDebugLogs += pLog->numOfDebugLogs; numOfTraceLogs += pLog->numOfTraceLogs; - - for (int32_t i = 0; i < taosArrayGetSize(pLog->logs); ++i) { - SJson *pLogJson = tjsonCreateObject(); - if (pLogJson == NULL) continue; - - SMonLogItem *pLogItem = taosArrayGet(pLog->logs, i); - - char buf[40] = {0}; - taosFormatUtcTime(buf, sizeof(buf), pLogItem->ts, TSDB_TIME_PRECISION_MILLI); - - tjsonAddStringToObject(pLogJson, "ts", buf); - tjsonAddStringToObject(pLogJson, "level", monLogLevelStr(pLogItem->level)); - tjsonAddStringToObject(pLogJson, "content", pLogItem->content); - - if (tjsonAddItemToArray(pLogsJson, pLogJson) != 0) tjsonDelete(pLogJson); - } } SJson *pSummaryJson = tjsonAddArrayToObject(pJson, "summary"); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 0f4e7bde638e7007383e90951f3dd05268971301..f5eacf0bd5d1c15bb7c773ee60caea7abc0dc0b5 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -110,6 +110,7 @@ static int32_t columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { COPY_SCALAR_FIELD(tableId); COPY_SCALAR_FIELD(tableType); COPY_SCALAR_FIELD(colId); + COPY_SCALAR_FIELD(projIdx); COPY_SCALAR_FIELD(colType); COPY_SCALAR_FIELD(hasIndex); COPY_CHAR_ARRAY_FIELD(dbName); @@ -157,7 +158,8 @@ static int32_t valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_VARBINARY: { + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_GEOMETRY: { int32_t len = pSrc->node.resType.bytes + 1; pDst->datum.p = taosMemoryCalloc(1, len); if (NULL == pDst->datum.p) { @@ -357,6 +359,9 @@ static int32_t logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { COPY_SCALAR_FIELD(requireDataOrder); COPY_SCALAR_FIELD(resultDataOrder); COPY_SCALAR_FIELD(groupAction); + COPY_SCALAR_FIELD(inputTsOrder); + COPY_SCALAR_FIELD(outputTsOrder); + COPY_SCALAR_FIELD(forceCreateNonBlockingOptr); return TSDB_CODE_SUCCESS; } @@ -393,6 +398,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { CLONE_NODE_LIST_FIELD(pTags); CLONE_NODE_FIELD(pSubtable); COPY_SCALAR_FIELD(igLastNull); + COPY_SCALAR_FIELD(groupOrderScan); return TSDB_CODE_SUCCESS; } @@ -401,8 +407,8 @@ static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pMergeCondition); CLONE_NODE_FIELD(pOnConditions); + CLONE_NODE_FIELD(pColEqualOnConditions); COPY_SCALAR_FIELD(isSingleTableJoin); - COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; } @@ -480,8 +486,6 @@ static int32_t logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* p COPY_SCALAR_FIELD(igExpired); COPY_SCALAR_FIELD(igCheckUpdate); COPY_SCALAR_FIELD(windowAlgo); - COPY_SCALAR_FIELD(inputTsOrder); - COPY_SCALAR_FIELD(outputTsOrder); return TSDB_CODE_SUCCESS; } @@ -493,7 +497,6 @@ static int32_t logicFillCopy(const SFillLogicNode* pSrc, SFillLogicNode* pDst) { CLONE_NODE_FIELD(pWStartTs); CLONE_NODE_FIELD(pValues); COPY_OBJECT_FIELD(timeRange, sizeof(STimeWindow)); - COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; } @@ -542,6 +545,9 @@ static int32_t physiNodeCopy(const SPhysiNode* pSrc, SPhysiNode* pDst) { CLONE_NODE_FIELD_EX(pOutputDataBlockDesc, SDataBlockDescNode*); CLONE_NODE_FIELD(pConditions); CLONE_NODE_LIST_FIELD(pChildren); + COPY_SCALAR_FIELD(inputTsOrder); + COPY_SCALAR_FIELD(outputTsOrder); + COPY_SCALAR_FIELD(forceCreateNonBlockingOptr); return TSDB_CODE_SUCCESS; } @@ -553,6 +559,7 @@ static int32_t physiScanCopy(const SScanPhysiNode* pSrc, SScanPhysiNode* pDst) { COPY_SCALAR_FIELD(suid); COPY_SCALAR_FIELD(tableType); COPY_OBJECT_FIELD(tableName, sizeof(SName)); + COPY_SCALAR_FIELD(groupOrderScan); return TSDB_CODE_SUCCESS; } @@ -588,7 +595,7 @@ static int32_t physiSysTableScanCopy(const SSystemTableScanPhysiNode* pSrc, SSys return TSDB_CODE_SUCCESS; } -static int32_t physiWindowCopy(const SWinodwPhysiNode* pSrc, SWinodwPhysiNode* pDst) { +static int32_t physiWindowCopy(const SWindowPhysiNode* pSrc, SWindowPhysiNode* pDst) { COPY_BASE_OBJECT_FIELD(node, physiNodeCopy); CLONE_NODE_LIST_FIELD(pExprs); CLONE_NODE_LIST_FIELD(pFuncs); @@ -667,7 +674,7 @@ static int32_t selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) { COPY_CHAR_ARRAY_FIELD(stmtName); COPY_SCALAR_FIELD(precision); COPY_SCALAR_FIELD(isEmptyResult); - COPY_SCALAR_FIELD(isTimeLineResult); + COPY_SCALAR_FIELD(timeLineResMode); COPY_SCALAR_FIELD(hasAggFuncs); COPY_SCALAR_FIELD(hasRepeatScanFuncs); return TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 136d1fc391b2c97c58fe70f3a4fcf358feea75e4..f25616065eb064d0160209320ecb744ba6ac23d8 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1428,6 +1428,7 @@ static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) { static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; static const char* jkJoinLogicPlanMergeCondition = "MergeConditions"; +static const char* jkJoinLogicPlanColEqualOnConditions = "ColumnEqualOnConditions"; static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj; @@ -1442,7 +1443,9 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinLogicPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions); + } return code; } @@ -1459,7 +1462,9 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions); } - + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinLogicPlanColEqualOnConditions, &pNode->pColEqualOnConditions); + } return code; } @@ -1554,6 +1559,7 @@ static const char* jkScanPhysiPlanTableId = "TableId"; static const char* jkScanPhysiPlanSTableId = "STableId"; static const char* jkScanPhysiPlanTableType = "TableType"; static const char* jkScanPhysiPlanTableName = "TableName"; +static const char* jkScanPhysiPlanGroupOrderScan = "GroupOrderScan"; static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { const STagScanPhysiNode* pNode = (const STagScanPhysiNode*)pObj; @@ -1577,6 +1583,9 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkScanPhysiPlanTableName, nameToJson, &pNode->tableName); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkScanPhysiPlanGroupOrderScan, pNode->groupOrderScan); + } return code; } @@ -1603,6 +1612,9 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, jkScanPhysiPlanTableName, jsonToName, &pNode->tableName); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkScanPhysiPlanGroupOrderScan, &pNode->groupOrderScan); + } return code; } @@ -1890,6 +1902,7 @@ static const char* jkJoinPhysiPlanInputTsOrder = "InputTsOrder"; static const char* jkJoinPhysiPlanMergeCondition = "MergeCondition"; static const char* jkJoinPhysiPlanOnConditions = "OnConditions"; static const char* jkJoinPhysiPlanTargets = "Targets"; +static const char* jkJoinPhysiPlanColEqualOnConditions = "ColumnEqualOnConditions"; static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj; @@ -1898,9 +1911,6 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanJoinType, pNode->joinType); } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanInputTsOrder, pNode->inputTsOrder); - } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkJoinPhysiPlanMergeCondition, nodeToJson, pNode->pMergeCondition); } @@ -1910,7 +1920,9 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinPhysiPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions); + } return code; } @@ -1921,9 +1933,6 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType, code); } - if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkJoinPhysiPlanInputTsOrder, pNode->inputTsOrder, code); - } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pOnConditions); } @@ -1933,7 +1942,9 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets); } - + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinPhysiPlanColEqualOnConditions, &pNode->pColEqualOnConditions); + } return code; } @@ -2096,6 +2107,7 @@ static int32_t jsonToPhysiMergeNode(const SJson* pJson, void* pObj) { static const char* jkSortPhysiPlanExprs = "Exprs"; static const char* jkSortPhysiPlanSortKeys = "SortKeys"; static const char* jkSortPhysiPlanTargets = "Targets"; +static const char* jkSortPhysiPlanMaxRows = "MaxRows"; static int32_t physiSortNodeToJson(const void* pObj, SJson* pJson) { const SSortPhysiNode* pNode = (const SSortPhysiNode*)pObj; @@ -2140,11 +2152,10 @@ static const char* jkWindowPhysiPlanWatermark = "Watermark"; static const char* jkWindowPhysiPlanDeleteMark = "DeleteMark"; static const char* jkWindowPhysiPlanIgnoreExpired = "IgnoreExpired"; static const char* jkWindowPhysiPlanInputTsOrder = "InputTsOrder"; -static const char* jkWindowPhysiPlanOutputTsOrder = "outputTsOrder"; static const char* jkWindowPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { - const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj; int32_t code = physicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { @@ -2171,12 +2182,6 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkWindowPhysiPlanIgnoreExpired, pNode->igExpired); } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkWindowPhysiPlanInputTsOrder, pNode->inputTsOrder); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkWindowPhysiPlanOutputTsOrder, pNode->outputTsOrder); - } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkWindowPhysiPlanMergeDataBlock, pNode->mergeDataBlock); } @@ -2185,7 +2190,7 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { } static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { - SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + SWindowPhysiNode* pNode = (SWindowPhysiNode*)pObj; int32_t code = jsonToPhysicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { @@ -2212,12 +2217,6 @@ static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkWindowPhysiPlanIgnoreExpired, &pNode->igExpired); } - if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkWindowPhysiPlanInputTsOrder, pNode->inputTsOrder, code); - } - if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkWindowPhysiPlanOutputTsOrder, pNode->outputTsOrder, code); - } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkWindowPhysiPlanMergeDataBlock, &pNode->mergeDataBlock); } @@ -2284,7 +2283,6 @@ static const char* jkFillPhysiPlanWStartTs = "WStartTs"; static const char* jkFillPhysiPlanValues = "Values"; static const char* jkFillPhysiPlanStartTime = "StartTime"; static const char* jkFillPhysiPlanEndTime = "EndTime"; -static const char* jkFillPhysiPlanInputTsOrder = "inputTsOrder"; static int32_t physiFillNodeToJson(const void* pObj, SJson* pJson) { const SFillPhysiNode* pNode = (const SFillPhysiNode*)pObj; @@ -2311,9 +2309,6 @@ static int32_t physiFillNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkFillPhysiPlanEndTime, pNode->timeRange.ekey); } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkFillPhysiPlanInputTsOrder, pNode->inputTsOrder); - } return code; } @@ -2343,9 +2338,6 @@ static int32_t jsonToPhysiFillNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkFillPhysiPlanEndTime, &pNode->timeRange.ekey); } - if (TSDB_CODE_SUCCESS == code) { - tjsonGetNumberValue(pJson, jkFillPhysiPlanInputTsOrder, pNode->inputTsOrder, code); - } return code; } @@ -3043,6 +3035,7 @@ static const char* jkColumnTableId = "TableId"; static const char* jkColumnTableType = "TableType"; static const char* jkColumnColId = "ColId"; static const char* jkColumnColType = "ColType"; +static const char* jkColumnProjId = "ProjId"; static const char* jkColumnDbName = "DbName"; static const char* jkColumnTableName = "TableName"; static const char* jkColumnTableAlias = "TableAlias"; @@ -3063,6 +3056,9 @@ static int32_t columnNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkColumnColId, pNode->colId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkColumnProjId, pNode->projIdx); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkColumnColType, pNode->colType); } @@ -3101,6 +3097,9 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetSmallIntValue(pJson, jkColumnColId, &pNode->colId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetSmallIntValue(pJson, jkColumnProjId, &pNode->projIdx); + } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkColumnColType, pNode->colType, code); } @@ -3176,6 +3175,7 @@ static int32_t datumToJson(const void* pObj, SJson* pJson) { } case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_GEOMETRY: code = tjsonAddStringToObject(pJson, jkValueDatum, varDataVal(pNode->datum.p)); break; case TSDB_DATA_TYPE_JSON: { @@ -3289,7 +3289,8 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) { break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_VARBINARY: { + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_GEOMETRY: { pNode->datum.p = taosMemoryCalloc(1, pNode->node.resType.bytes + 1); if (NULL == pNode->datum.p) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -5510,6 +5511,7 @@ static const char* jkDropDnodeStmtDnodeId = "DnodeId"; static const char* jkDropDnodeStmtFqdn = "Fqdn"; static const char* jkDropDnodeStmtPort = "Port"; static const char* jkDropDnodeStmtForce = "Force"; +static const char* jkDropDnodeStmtUnsafe = "Unsafe"; static int32_t dropDnodeStmtToJson(const void* pObj, SJson* pJson) { const SDropDnodeStmt* pNode = (const SDropDnodeStmt*)pObj; @@ -5524,6 +5526,9 @@ static int32_t dropDnodeStmtToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkDropDnodeStmtForce, pNode->force); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDropDnodeStmtUnsafe, pNode->unsafe); + } return code; } @@ -5541,6 +5546,9 @@ static int32_t jsonToDropDnodeStmt(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkDropDnodeStmtForce, &pNode->force); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDropDnodeStmtUnsafe, &pNode->unsafe); + } return code; } diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index 156744ef1dceb9e47860de780904c54997023f5c..f755b8cb8c013132239aa2dc19ad25c83bb0322e 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -82,9 +82,7 @@ static bool columnNodeEqual(const SColumnNode* a, const SColumnNode* b) { COMPARE_STRING_FIELD(dbName); COMPARE_STRING_FIELD(tableName); COMPARE_STRING_FIELD(colName); - if (0 == a->tableId) { - COMPARE_STRING_FIELD(tableAlias); - } + COMPARE_STRING_FIELD(tableAlias); return true; } @@ -109,6 +107,7 @@ static bool valueNodeEqual(const SValueNode* a, const SValueNode* b) { case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: COMPARE_VARDATA_FIELD(datum.p); break; case TSDB_DATA_TYPE_JSON: diff --git a/source/libs/nodes/src/nodesMatchFuncs.c b/source/libs/nodes/src/nodesMatchFuncs.c new file mode 100755 index 0000000000000000000000000000000000000000..401c7aad283c3b514349178f88332d329a18b005 --- /dev/null +++ b/source/libs/nodes/src/nodesMatchFuncs.c @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "querynodes.h" + +#define MATCH_SCALAR_FIELD(fldname) \ + do { \ + if (p->fldname != pSub->fldname) return false; \ + } while (0) + +#define MATCH_STRING(a, b) (((a) != NULL && (b) != NULL) ? (strcmp((a), (b)) == 0) : (a) == (b)) + +#define MATCH_VARDATA(a, b) \ + (((a) != NULL && (b) != NULL) \ + ? (varDataLen((a)) == varDataLen((b)) && memcmp(varDataVal((a)), varDataVal((b)), varDataLen((a))) == 0) \ + : (a) == (b)) + +#define MATCH_STRING_FIELD(fldname) \ + do { \ + if (!MATCH_STRING(p->fldname, pSub->fldname)) return false; \ + } while (0) + +#define MATCH_VARDATA_FIELD(fldname) \ + do { \ + if (!MATCH_VARDATA(p->fldname, pSub->fldname)) return false; \ + } while (0) + +#define MATCH_OBJECT_FIELD(fldname, matchFunc) \ + do { \ + if (!matchFunc(p->fldname, pSub->fldname)) return false; \ + } while (0) + +#define MATCH_NODE_FIELD(fldname) \ + do { \ + if (!nodesMatchNode(pSub->fldname, p->fldname)) return false; \ + } while (0) + +#define MATCH_NODE_LIST_FIELD(fldname) \ + do { \ + if (!nodesListMatch(p->fldname, pSub->fldname)) return false; \ + } while (0) + + +bool nodesListMatchExists(const SNodeList* pList, const SNode* pTarget) { + if (NULL == pList || NULL == pTarget) { + return false; + } + SNode* node = NULL; + bool exists = false; + FOREACH(node, pList) { + if (nodesMatchNode(node, pTarget)) { + exists = true; + break; + } + } + + return exists; +} + +bool nodesListMatch(const SNodeList* pList, const SNodeList* pSubList) { + if (pList == pSubList) { + return true; + } + + if (NULL == pList || NULL == pSubList) { + return false; + } + + if (pList->length != pSubList->length) { + return false; + } + + SNode* node = NULL; + bool match = true; + FOREACH(node, pList) { + if (!nodesListMatchExists(pSubList, node)) { + match = false; + break; + } + } + return match; +} + +static bool columnNodeMatch(const SColumnNode* pSub, const SColumnNode* p) { + if (0 == strcmp(p->colName, pSub->node.aliasName)) { + return true; + } + return false; +} + +static bool valueNodeMatch(const SValueNode* pSub, const SValueNode* p) { + return nodesEqualNode((SNode*)pSub, (SNode*)p); +} + +static bool operatorNodeMatch(const SOperatorNode* pSub, const SOperatorNode* p) { + MATCH_SCALAR_FIELD(opType); + MATCH_NODE_FIELD(pLeft); + MATCH_NODE_FIELD(pRight); + return true; +} + +static bool logicConditionNodeMatch(const SLogicConditionNode* pSub, const SLogicConditionNode* p) { + MATCH_SCALAR_FIELD(condType); + MATCH_NODE_LIST_FIELD(pParameterList); + return true; +} + +static bool functionNodeMatch(const SFunctionNode* pSub, const SFunctionNode* p) { + MATCH_SCALAR_FIELD(funcId); + MATCH_STRING_FIELD(functionName); + MATCH_NODE_LIST_FIELD(pParameterList); + return true; +} + +static bool whenThenNodeMatch(const SWhenThenNode* pSub, const SWhenThenNode* p) { + MATCH_NODE_FIELD(pWhen); + MATCH_NODE_FIELD(pThen); + return true; +} + +static bool caseWhenNodeMatch(const SCaseWhenNode* pSub, const SCaseWhenNode* p) { + MATCH_NODE_FIELD(pCase); + MATCH_NODE_FIELD(pElse); + MATCH_NODE_LIST_FIELD(pWhenThenList); + return true; +} + +bool nodesMatchNode(const SNode* pSub, const SNode* p) { + if (pSub == p) { + return true; + } + + if (NULL == pSub || NULL == p) { + return false; + } + + if (nodeType(pSub) != nodeType(p)) { + return false; + } + + switch (nodeType(p)) { + case QUERY_NODE_COLUMN: + return columnNodeMatch((const SColumnNode*)pSub, (const SColumnNode*)p); + case QUERY_NODE_VALUE: + return valueNodeMatch((const SValueNode*)pSub, (const SValueNode*)p); + case QUERY_NODE_OPERATOR: + return operatorNodeMatch((const SOperatorNode*)pSub, (const SOperatorNode*)p); + case QUERY_NODE_LOGIC_CONDITION: + return logicConditionNodeMatch((const SLogicConditionNode*)pSub, (const SLogicConditionNode*)p); + case QUERY_NODE_FUNCTION: + return functionNodeMatch((const SFunctionNode*)pSub, (const SFunctionNode*)p); + case QUERY_NODE_WHEN_THEN: + return whenThenNodeMatch((const SWhenThenNode*)pSub, (const SWhenThenNode*)p); + case QUERY_NODE_CASE_WHEN: + return caseWhenNodeMatch((const SCaseWhenNode*)pSub, (const SCaseWhenNode*)p); + case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_TEMP_TABLE: + case QUERY_NODE_JOIN_TABLE: + case QUERY_NODE_GROUPING_SET: + case QUERY_NODE_ORDER_BY_EXPR: + case QUERY_NODE_LIMIT: + return false; + default: + break; + } + + return false; +} diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index c06eb62771b1c96065c3ef3d9f0c3917d9297843..20e829766dafe75aa2372aa1e4147ac9b856ee71 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -825,6 +825,7 @@ static int32_t datumToMsg(const void* pObj, STlvEncoder* pEncoder) { case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: code = tlvEncodeBinary(pEncoder, VALUE_CODE_DATUM, pNode->datum.p, varDataTLen(pNode->datum.p)); break; case TSDB_DATA_TYPE_JSON: @@ -923,7 +924,8 @@ static int32_t msgToDatum(STlv* pTlv, void* pObj) { break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_VARBINARY: { + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_GEOMETRY: { if (pTlv->len > pNode->node.resType.bytes + VARSTR_HEADER_SIZE) { code = TSDB_CODE_FAILED; break; @@ -1849,7 +1851,10 @@ enum { PHY_NODE_CODE_CONDITIONS, PHY_NODE_CODE_CHILDREN, PHY_NODE_CODE_LIMIT, - PHY_NODE_CODE_SLIMIT + PHY_NODE_CODE_SLIMIT, + PHY_NODE_CODE_INPUT_TS_ORDER, + PHY_NODE_CODE_OUTPUT_TS_ORDER, + PHY_NODE_CODE_FORCE_NONBLOCKING_OPTR }; static int32_t physiNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -1868,6 +1873,15 @@ static int32_t physiNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeObj(pEncoder, PHY_NODE_CODE_SLIMIT, nodeToMsg, pNode->pSlimit); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_NODE_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_NODE_CODE_OUTPUT_TS_ORDER, pNode->outputTsOrder); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_NODE_CODE_FORCE_NONBLOCKING_OPTR, pNode->forceCreateNonBlockingOptr); + } return code; } @@ -1894,6 +1908,14 @@ static int32_t msgToPhysiNode(STlvDecoder* pDecoder, void* pObj) { case PHY_NODE_CODE_SLIMIT: code = msgToNodeFromTlv(pTlv, (void**)&pNode->pSlimit); break; + case PHY_NODE_CODE_INPUT_TS_ORDER: + code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); + break; + case PHY_NODE_CODE_OUTPUT_TS_ORDER: + code = tlvDecodeEnum(pTlv, &pNode->outputTsOrder, sizeof(pNode->outputTsOrder)); + break; + case PHY_NODE_CODE_FORCE_NONBLOCKING_OPTR: + code = tlvDecodeBool(pTlv, &pNode->forceCreateNonBlockingOptr); default: break; } @@ -1909,7 +1931,8 @@ enum { PHY_SCAN_CODE_BASE_UID, PHY_SCAN_CODE_BASE_SUID, PHY_SCAN_CODE_BASE_TABLE_TYPE, - PHY_SCAN_CODE_BASE_TABLE_NAME + PHY_SCAN_CODE_BASE_TABLE_NAME, + PHY_SCAN_CODE_BASE_GROUP_ORDER_SCAN }; static int32_t physiScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -1934,6 +1957,9 @@ static int32_t physiScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeObj(pEncoder, PHY_SCAN_CODE_BASE_TABLE_NAME, nameToMsg, &pNode->tableName); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_SCAN_CODE_BASE_GROUP_ORDER_SCAN, pNode->groupOrderScan); + } return code; } @@ -1966,6 +1992,9 @@ static int32_t msgToPhysiScanNode(STlvDecoder* pDecoder, void* pObj) { case PHY_SCAN_CODE_BASE_TABLE_NAME: code = tlvDecodeObjFromTlv(pTlv, msgToName, &pNode->tableName); break; + case PHY_SCAN_CODE_BASE_GROUP_ORDER_SCAN: + code = tlvDecodeBool(pTlv, &pNode->groupOrderScan); + break; default: break; } @@ -2317,7 +2346,8 @@ enum { PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION, PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS, PHY_SORT_MERGE_JOIN_CODE_TARGETS, - PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER + PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, + PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS }; static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -2337,9 +2367,8 @@ static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TARGETS, nodeListToMsg, pNode->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pColEqualOnConditions); } - return code; } @@ -2365,8 +2394,8 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) { case PHY_SORT_MERGE_JOIN_CODE_TARGETS: code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets); break; - case PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER: - code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); + case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pColEqualOnConditions); break; default: break; @@ -2640,7 +2669,7 @@ enum { }; static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { - const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj; int32_t code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); if (TSDB_CODE_SUCCESS == code) { @@ -2667,12 +2696,6 @@ static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeI8(pEncoder, PHY_WINDOW_CODE_IG_EXPIRED, pNode->igExpired); } - if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeEnum(pEncoder, PHY_WINDOW_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); - } - if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeEnum(pEncoder, PHY_WINDOW_CODE_OUTPUT_TS_ORDER, pNode->outputTsOrder); - } if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeBool(pEncoder, PHY_WINDOW_CODE_MERGE_DATA_BLOCK, pNode->mergeDataBlock); } @@ -2681,7 +2704,7 @@ static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { } static int32_t msgToPhysiWindowNode(STlvDecoder* pDecoder, void* pObj) { - SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + SWindowPhysiNode* pNode = (SWindowPhysiNode*)pObj; int32_t code = TSDB_CODE_SUCCESS; STlv* pTlv = NULL; @@ -2714,12 +2737,6 @@ static int32_t msgToPhysiWindowNode(STlvDecoder* pDecoder, void* pObj) { case PHY_WINDOW_CODE_IG_EXPIRED: code = tlvDecodeI8(pTlv, &pNode->igExpired); break; - case PHY_WINDOW_CODE_INPUT_TS_ORDER: - code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); - break; - case PHY_WINDOW_CODE_OUTPUT_TS_ORDER: - code = tlvDecodeEnum(pTlv, &pNode->outputTsOrder, sizeof(pNode->outputTsOrder)); - break; case PHY_WINDOW_CODE_MERGE_DATA_BLOCK: code = tlvDecodeBool(pTlv, &pNode->mergeDataBlock); break; @@ -2838,9 +2855,6 @@ static int32_t physiFillNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeObj(pEncoder, PHY_FILL_CODE_TIME_RANGE, timeWindowToMsg, &pNode->timeRange); } - if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeEnum(pEncoder, PHY_FILL_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); - } return code; } @@ -2873,9 +2887,6 @@ static int32_t msgToPhysiFillNode(STlvDecoder* pDecoder, void* pObj) { case PHY_FILL_CODE_TIME_RANGE: code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, (void**)&pNode->timeRange); break; - case PHY_FILL_CODE_INPUT_TS_ORDER: - code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); - break; default: break; } diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index 0181da92a9e7eeb5387a68ed18b0b20470979101..b57bba0cc9bd510576e611a5f0416ef3d21982d4 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -120,9 +120,9 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { } if (colNode->tableAlias[0]) { - *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->colName); + *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->node.userAlias); } else { - *len += snprintf(buf + *len, bufSize - *len, "%s", colNode->colName); + *len += snprintf(buf + *len, bufSize - *len, "%s", colNode->node.userAlias); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index ce575ede8a5dd2020c6af39394a72d6fa39cc348..b3623a4b0a0a2707d4e62cb3a98bd2a00f85fec6 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -214,6 +214,18 @@ void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContex (void)walkExprs(pList, TRAVERSAL_POSTORDER, walker, pContext); } +static void checkParamIsFunc(SFunctionNode *pFunc) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (numOfParams > 1) { + for (int32_t i = 0; i < numOfParams; ++i) { + SNode* pPara = nodesListGetNode(pFunc->pParameterList, i); + if (nodeType(pPara) == QUERY_NODE_FUNCTION) { + ((SFunctionNode *)pPara)->node.asParam = true; + } + } + } +} + static EDealRes rewriteExprs(SNodeList* pNodeList, ETraversalOrder order, FNodeRewriter rewriter, void* pContext); static EDealRes rewriteExpr(SNode** pRawNode, ETraversalOrder order, FNodeRewriter rewriter, void* pContext) { @@ -248,9 +260,12 @@ static EDealRes rewriteExpr(SNode** pRawNode, ETraversalOrder order, FNodeRewrit case QUERY_NODE_LOGIC_CONDITION: res = rewriteExprs(((SLogicConditionNode*)pNode)->pParameterList, order, rewriter, pContext); break; - case QUERY_NODE_FUNCTION: - res = rewriteExprs(((SFunctionNode*)pNode)->pParameterList, order, rewriter, pContext); + case QUERY_NODE_FUNCTION: { + SFunctionNode* pFunc = (SFunctionNode*)pNode; + checkParamIsFunc(pFunc); + res = rewriteExprs(pFunc->pParameterList, order, rewriter, pContext); break; + } case QUERY_NODE_REAL_TABLE: case QUERY_NODE_TEMP_TABLE: break; // todo diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 13c5a34084d256e0a7d4c7341b4f3e53a8e32892..c8197721fb2803efa6d4ab40a8ef4dd1f20a46bb 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -87,7 +87,7 @@ static void* nodesCalloc(int32_t num, int32_t size) { return (char*)p + 1; } -static void nodesFree(void* p) { +void nodesFree(void* p) { char* ptr = (char*)p - 1; if (0 == *ptr) { taosMemoryFree(ptr); @@ -608,7 +608,7 @@ static void destroyPhysiNode(SPhysiNode* pNode) { nodesDestroyNode(pNode->pSlimit); } -static void destroyWinodwPhysiNode(SWinodwPhysiNode* pNode) { +static void destroyWinodwPhysiNode(SWindowPhysiNode* pNode) { destroyPhysiNode((SPhysiNode*)pNode); nodesDestroyList(pNode->pExprs); nodesDestroyList(pNode->pFuncs); @@ -907,6 +907,10 @@ void nodesDestroyNode(SNode* pNode) { SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pNode; nodesDestroyNode((SNode*)pStmt->pOptions); nodesDestroyList(pStmt->pCols); + if (pStmt->pReq) { + tFreeSMCreateSmaReq(pStmt->pReq); + taosMemoryFreeClear(pStmt->pReq); + } break; } case QUERY_NODE_DROP_INDEX_STMT: // no pointer field @@ -921,6 +925,7 @@ void nodesDestroyNode(SNode* pNode) { break; case QUERY_NODE_CREATE_TOPIC_STMT: nodesDestroyNode(((SCreateTopicStmt*)pNode)->pQuery); + nodesDestroyNode(((SCreateTopicStmt*)pNode)->pWhere); break; case QUERY_NODE_DROP_TOPIC_STMT: // no pointer field case QUERY_NODE_DROP_CGROUP_STMT: // no pointer field @@ -952,6 +957,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pStmt->pQuery); nodesDestroyList(pStmt->pTags); nodesDestroyNode(pStmt->pSubtable); + tFreeSCMCreateStreamReq(pStmt->pReq); + taosMemoryFreeClear(pStmt->pReq); break; } case QUERY_NODE_DROP_STREAM_STMT: // no pointer field @@ -1050,7 +1057,9 @@ void nodesDestroyNode(SNode* pNode) { } case QUERY_NODE_QUERY: { SQuery* pQuery = (SQuery*)pNode; + nodesDestroyNode(pQuery->pPrevRoot); nodesDestroyNode(pQuery->pRoot); + nodesDestroyNode(pQuery->pPostRoot); taosMemoryFreeClear(pQuery->pResSchema); if (NULL != pQuery->pCmdMsg) { taosMemoryFreeClear(pQuery->pCmdMsg->pMsg); @@ -1088,6 +1097,7 @@ void nodesDestroyNode(SNode* pNode) { destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyNode(pLogicNode->pMergeCondition); nodesDestroyNode(pLogicNode->pOnConditions); + nodesDestroyNode(pLogicNode->pColEqualOnConditions); break; } case QUERY_NODE_LOGIC_PLAN_AGG: { @@ -1220,6 +1230,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pMergeCondition); nodesDestroyNode(pPhyNode->pOnConditions); nodesDestroyList(pPhyNode->pTargets); + nodesDestroyNode(pPhyNode->pColEqualOnConditions); break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { @@ -1257,7 +1268,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: - destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: { @@ -1273,19 +1284,19 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: - destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: { SStateWinodwPhysiNode* pPhyNode = (SStateWinodwPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode); nodesDestroyNode(pPhyNode->pStateKey); break; } case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: { SEventWinodwPhysiNode* pPhyNode = (SEventWinodwPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode); nodesDestroyNode(pPhyNode->pStartCond); nodesDestroyNode(pPhyNode->pEndCond); break; @@ -1575,6 +1586,7 @@ void* nodesGetValueFromNode(SValueNode* pNode) { case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_GEOMETRY: return (void*)pNode->datum.p; default: break; @@ -1637,6 +1649,7 @@ int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value) { case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_GEOMETRY: pNode->datum.p = (char*)value; break; default: @@ -1694,7 +1707,8 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) { } case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_VARBINARY: { + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_GEOMETRY: { int32_t bufSize = varDataLen(pNode->datum.p) + 2 + 1; void* buf = taosMemoryMalloc(bufSize); if (NULL == buf) { @@ -1947,9 +1961,9 @@ static uint32_t funcNodeHash(const char* pKey, uint32_t len) { } static int32_t funcNodeEqual(const void* pLeft, const void* pRight, size_t len) { - if (0 != strcmp((*(const SExprNode**)pLeft)->aliasName, (*(const SExprNode**)pRight)->aliasName)) { - return 1; - } + // if (0 != strcmp((*(const SExprNode**)pLeft)->aliasName, (*(const SExprNode**)pRight)->aliasName)) { + // return 1; + // } return nodesEqualNode(*(const SNode**)pLeft, *(const SNode**)pRight) ? 0 : 1; } @@ -2070,7 +2084,7 @@ char* nodesGetNameFromColumnNode(SNode* pNode) { return "NULL"; } - return ((SColumnNode*)pNode)->colName; + return ((SColumnNode*)pNode)->node.userAlias; } int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots) { @@ -2130,6 +2144,7 @@ void nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_GEOMETRY: pVal->pz = taosMemoryMalloc(pVal->nLen + 1); memcpy(pVal->pz, pNode->datum.p, pVal->nLen); pVal->pz[pVal->nLen] = 0; diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 860250df85b928bac1111c961033928a16506b8d..41553918e1f1dd664dbe6cc910050eeb40e10e47 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( parser - PRIVATE os util nodes catalog function scalar transport qcom + PRIVATE os util nodes catalog function scalar geometry transport qcom ) if(${BUILD_TEST}) diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 1d1a522d0125df847ae3921f3dd586ab030ae47b..78422bf746ebcb72c495a2475b8dc26872e7bacc 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -127,6 +127,7 @@ SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode); SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd); +SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint); SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen); SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse); @@ -192,7 +193,7 @@ SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const ST SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t alterType, const SToken* pVal); SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName); SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort); -SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool force); +SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool force, bool unsafe); SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue); SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName); SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SNode* pIndexName, @@ -205,11 +206,11 @@ SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, cons SNode* createRestoreComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId); SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pQuery); SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SToken* pSubDbName, - bool withMeta); + int8_t withMeta); SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable, - bool withMeta); + int8_t withMeta, SNode* pWhere); SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pTopicName); -SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId, SToken* pTopicName); +SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pCGroupId, SToken* pTopicName); SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue); SNode* createDefaultExplainOptions(SAstCreateContext* pCxt); SNode* setExplainVerbose(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal); diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 66aec272d787df6fc4e04248d4833bda78b98a65..69253e62e23a0e3970c072f4db9de9c568c4d5e0 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -34,6 +34,8 @@ int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); +int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow); +int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 850571eea1533541318c4dc57c60b14311fcc5a8..d96bb9bba430a4443a17f92be81fd777c62388bd 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -84,6 +84,7 @@ int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta); STableComInfo getTableInfo(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta); +int32_t getTableTypeFromTableNode(SNode *pTable); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName); @@ -115,6 +116,13 @@ int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput); int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes); void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request); +SNode* createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable); + +/** + * @brief return a - b with overflow check + * @retval val range between [INT64_MIN, INT64_MAX] + */ +int64_t int64SafeSub(int64_t a, int64_t b); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index b682cdd4fd838252044b50b0ca7023725dbd1aad..6c3f589159b8db9903a9fefee9dbb6ead26f96d0 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -126,8 +126,10 @@ with_opt(A) ::= WITH search_condition(B). /************************************************ create/drop/alter/restore dnode *********************************************/ cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); } cmd ::= CREATE DNODE dnode_endpoint(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B); } -cmd ::= DROP DNODE NK_INTEGER(A) force_opt(B). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A, B); } -cmd ::= DROP DNODE dnode_endpoint(A) force_opt(B). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A, B); } +cmd ::= DROP DNODE NK_INTEGER(A) force_opt(B). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A, B, false); } +cmd ::= DROP DNODE dnode_endpoint(A) force_opt(B). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A, B, false); } +cmd ::= DROP DNODE NK_INTEGER(A) unsafe_opt(B). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A, false, B); } +cmd ::= DROP DNODE dnode_endpoint(A) unsafe_opt(B). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A, false, B); } cmd ::= ALTER DNODE NK_INTEGER(A) NK_STRING(B). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &A, &B, NULL); } cmd ::= ALTER DNODE NK_INTEGER(A) NK_STRING(B) NK_STRING(C). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &A, &B, &C); } cmd ::= ALTER ALL DNODES NK_STRING(A). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &A, NULL); } @@ -145,6 +147,10 @@ dnode_endpoint(A) ::= NK_IPTOKEN(B). force_opt(A) ::= . { A = false; } force_opt(A) ::= FORCE. { A = true; } +%type unsafe_opt { bool } +%destructor unsafe_opt { } +unsafe_opt(A) ::= UNSAFE. { A = true; } + /************************************************ alter local *********************************************************/ cmd ::= ALTER LOCAL NK_STRING(A). { pCxt->pRootNode = createAlterLocalStmt(pCxt, &A, NULL); } cmd ::= ALTER LOCAL NK_STRING(A) NK_STRING(B). { pCxt->pRootNode = createAlterLocalStmt(pCxt, &A, &B); } @@ -376,6 +382,7 @@ type_name(A) ::= VARCHAR NK_LP NK_INTEGER(B) NK_RP. type_name(A) ::= MEDIUMBLOB. { A = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } type_name(A) ::= BLOB. { A = createDataType(TSDB_DATA_TYPE_BLOB); } type_name(A) ::= VARBINARY NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &B); } +type_name(A) ::= GEOMETRY NK_LP NK_INTEGER(B) NK_RP. { A = createVarLenDataType(TSDB_DATA_TYPE_GEOMETRY, &B); } type_name(A) ::= DECIMAL. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); } @@ -440,6 +447,7 @@ cmd ::= SHOW MNODES. cmd ::= SHOW QNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT); } cmd ::= SHOW FUNCTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } cmd ::= SHOW INDEXES FROM table_name_cond(A) from_db_opt(B). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, B, A, OP_TYPE_EQUAL); } +cmd ::= SHOW INDEXES FROM db_name(B) NK_DOT table_name(A). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, createIdentifierValueNode(pCxt, &B), createIdentifierValueNode(pCxt, &A), OP_TYPE_EQUAL); } cmd ::= SHOW STREAMS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } cmd ::= SHOW ACCOUNTS. { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } cmd ::= SHOW APPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT); } @@ -464,7 +472,9 @@ cmd ::= SHOW TABLE DISTRIBUTED full_table_name(A). cmd ::= SHOW CONSUMERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } cmd ::= SHOW SUBSCRIPTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } cmd ::= SHOW TAGS FROM table_name_cond(A) from_db_opt(B). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, B, A, OP_TYPE_EQUAL); } +cmd ::= SHOW TAGS FROM db_name(B) NK_DOT table_name(A). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, createIdentifierValueNode(pCxt, &B), createIdentifierValueNode(pCxt, &A), OP_TYPE_EQUAL); } cmd ::= SHOW TABLE TAGS tag_list_opt(C) FROM table_name_cond(A) from_db_opt(B). { pCxt->pRootNode = createShowTableTagsStmt(pCxt, A, B, C); } +cmd ::= SHOW TABLE TAGS tag_list_opt(C) FROM db_name(B) NK_DOT table_name(A). { pCxt->pRootNode = createShowTableTagsStmt(pCxt, createIdentifierValueNode(pCxt, &A), createIdentifierValueNode(pCxt, &B), C); } cmd ::= SHOW VNODES NK_INTEGER(A). { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A), NULL); } cmd ::= SHOW VNODES NK_STRING(A). { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &A)); } // show alive @@ -531,14 +541,18 @@ sma_stream_opt(A) ::= sma_stream_opt(B) MAX_DELAY duration_literal(C). sma_stream_opt(A) ::= sma_stream_opt(B) DELETE_MARK duration_literal(C). { ((SStreamOptions*)B)->pDeleteMark = releaseRawExprNode(pCxt, C); A = B; } /************************************************ create/drop topic ***************************************************/ +%type with_meta { int32_t } +%destructor with_meta { } +with_meta(A) ::= AS. { A = 0; } +with_meta(A) ::= WITH META AS. { A = 1; } +with_meta(A) ::= ONLY META AS. { A = 2; } + cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_or_subquery(C). { pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, A, &B, C); } -cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS DATABASE db_name(C). { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, false); } -cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) - WITH META AS DATABASE db_name(C). { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, true); } -cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) - AS STABLE full_table_name(C). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, false); } -cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) - WITH META AS STABLE full_table_name(C). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, true); } +cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) with_meta(D) + DATABASE db_name(C). { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, D); } +cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) with_meta(E) + STABLE full_table_name(C) where_clause_opt(D). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, E, D); } + cmd ::= DROP TOPIC exists_opt(A) topic_name(B). { pCxt->pRootNode = createDropTopicStmt(pCxt, A, &B); } cmd ::= DROP CONSUMER GROUP exists_opt(A) cgroup_name(B) ON topic_name(C). { pCxt->pRootNode = createDropCGroupStmt(pCxt, A, &B, &C); } @@ -1088,6 +1102,8 @@ having_clause_opt(A) ::= HAVING search_condition(B). range_opt(A) ::= . { A = NULL; } range_opt(A) ::= RANGE NK_LP expr_or_subquery(B) NK_COMMA expr_or_subquery(C) NK_RP. { A = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)); } +range_opt(A) ::= + RANGE NK_LP expr_or_subquery(B) NK_RP. { A = createInterpTimePoint(pCxt, releaseRawExprNode(pCxt, B)); } every_opt(A) ::= . { A = NULL; } every_opt(A) ::= EVERY NK_LP duration_literal(B) NK_RP. { A = releaseRawExprNode(pCxt, B); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index eeccf18c7ba8b7babdf2dbd3098b531b3642eaea..f85218c50a767ca41acef02618251b2e0749bceb 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -210,6 +210,15 @@ static bool checkTopicName(SAstCreateContext* pCxt, SToken* pTopicName) { return true; } +static bool checkCGroupName(SAstCreateContext* pCxt, SToken* pCGroup) { + trimEscape(pCGroup); + if (pCGroup->n >= TSDB_CGROUP_LEN) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pCGroup->z); + return false; + } + return true; +} + static bool checkStreamName(SAstCreateContext* pCxt, SToken* pStreamName) { trimEscape(pStreamName); if (pStreamName->n >= TSDB_STREAM_NAME_LEN) { @@ -259,8 +268,19 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { strcpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName); } else { int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); - strncpy(pExpr->aliasName, pRawExpr->p, len); - pExpr->aliasName[len] = '\0'; + + // See TS-3398. + // Len of pRawExpr->p could be larger than len of aliasName[TSDB_COL_NAME_LEN]. + // If aliasName is truncated, hash value of aliasName could be the same. + T_MD5_CTX ctx; + tMD5Init(&ctx); + tMD5Update(&ctx, (uint8_t*)pRawExpr->p, pRawExpr->n); + tMD5Final(&ctx); + char* p = pExpr->aliasName; + for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) { + sprintf(p, "%02x", ctx.digest[i]); + p += 2; + } strncpy(pExpr->userAlias, pRawExpr->p, len); pExpr->userAlias[len] = '\0'; } @@ -498,7 +518,7 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d CHECK_OUT_OF_MEM(func); strcpy(func->functionName, "cast"); func->node.resType = dt; - if (TSDB_DATA_TYPE_VARCHAR == dt.type) { + if (TSDB_DATA_TYPE_VARCHAR == dt.type || TSDB_DATA_TYPE_GEOMETRY == dt.type) { func->node.resType.bytes = func->node.resType.bytes + VARSTR_HEADER_SIZE; } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { func->node.resType.bytes = func->node.resType.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; @@ -684,6 +704,11 @@ SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd return createBetweenAnd(pCxt, createPrimaryKeyCol(pCxt, NULL), pStart, pEnd); } +SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint) { + CHECK_PARSER_STATUS(pCxt); + return createOperatorNode(pCxt, OP_TYPE_EQUAL, createPrimaryKeyCol(pCxt, NULL), pPoint); +} + SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen) { CHECK_PARSER_STATUS(pCxt); SWhenThenNode* pWhenThen = (SWhenThenNode*)nodesMakeNode(QUERY_NODE_WHEN_THEN); @@ -822,16 +847,9 @@ SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill) { SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable) { CHECK_PARSER_STATUS(pCxt); - SSelectStmt* select = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + SNode* select = createSelectStmtImpl(isDistinct, pProjectionList, pTable); CHECK_OUT_OF_MEM(select); - select->isDistinct = isDistinct; - select->pProjectionList = pProjectionList; - select->pFromTable = pTable; - sprintf(select->stmtName, "%p", select); - select->isTimeLineResult = true; - select->onlyHasKeepOrderFunc = true; - select->timeRange = TSWINDOW_INITIALIZER; - return (SNode*)select; + return select; } static void setSubquery(SNode* pStmt) { @@ -1576,7 +1594,7 @@ SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const return (SNode*)pStmt; } -SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool force) { +SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool force, bool unsafe) { CHECK_PARSER_STATUS(pCxt); SDropDnodeStmt* pStmt = (SDropDnodeStmt*)nodesMakeNode(QUERY_NODE_DROP_DNODE_STMT); CHECK_OUT_OF_MEM(pStmt); @@ -1589,6 +1607,7 @@ SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool f } } pStmt->force = force; + pStmt->unsafe = unsafe; return (SNode*)pStmt; } @@ -1696,7 +1715,7 @@ SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, } SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SToken* pSubDbName, - bool withMeta) { + int8_t withMeta) { CHECK_PARSER_STATUS(pCxt); if (!checkTopicName(pCxt, pTopicName) || !checkDbName(pCxt, pSubDbName, true)) { return NULL; @@ -1711,7 +1730,7 @@ SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, ST } SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable, - bool withMeta) { + int8_t withMeta, SNode* pWhere) { CHECK_PARSER_STATUS(pCxt); if (!checkTopicName(pCxt, pTopicName)) { return NULL; @@ -1721,6 +1740,8 @@ SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); pStmt->ignoreExists = ignoreExists; pStmt->withMeta = withMeta; + pStmt->pWhere = pWhere; + strcpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName); strcpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName); nodesDestroyNode(pRealTable); @@ -1739,12 +1760,15 @@ SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken return (SNode*)pStmt; } -SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId, +SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pCGroupId, SToken* pTopicName) { CHECK_PARSER_STATUS(pCxt); if (!checkTopicName(pCxt, pTopicName)) { return NULL; } + if (!checkCGroupName(pCxt, pCGroupId)) { + return NULL; + } SDropCGroupStmt* pStmt = (SDropCGroupStmt*)nodesMakeNode(QUERY_NODE_DROP_CGROUP_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->ignoreNotExists = ignoreNotExists; diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 5db1f5dbdbca70bed529373c3bb2caa76d74efd2..fdec9cba79aa8dd07fdf3706c99ab74023664397 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -355,6 +355,11 @@ static int32_t collectMetaKeyFromCreateTopic(SCollectMetaKeyCxt* pCxt, SCreateTo if (NULL != pStmt->pQuery) { return collectMetaKeyFromQuery(pCxt, pStmt->pQuery); } + if (NULL != pStmt->pWhere) { + int32_t code = collectMetaKeyFromRealTableImpl(pCxt, pStmt->subDbName, pStmt->subSTbName, + AUTH_TYPE_READ); + return code; + } return TSDB_CODE_SUCCESS; } @@ -379,6 +384,10 @@ static int32_t collectMetaKeyFromCreateStream(SCollectMetaKeyCxt* pCxt, SCreateS if (TSDB_CODE_SUCCESS == code) { code = collectMetaKeyFromQuery(pCxt, pStmt->pQuery); } + if (TSDB_CODE_SUCCESS == code && pStmt->pOptions->fillHistory) { + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + code = reserveDbCfgForLastRow(pCxt, pSelect->pFromTable); + } return code; } @@ -581,7 +590,7 @@ static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShow code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_READ, + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ, pCxt->pMetaCache); } return code; diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 1586d8128ba8d425575526ecf5567e688408af42..9b2ac662c84443c8efa39b308af51d65a941706a 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -28,6 +28,10 @@ typedef struct SSelectAuthCxt { SSelectStmt* pSelect; } SSelectAuthCxt; +typedef struct SAuthRewriteCxt { + STableNode* pTarget; +} SAuthRewriteCxt; + static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt); static void setUserAuthInfo(SParseContext* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, @@ -90,12 +94,26 @@ static int32_t mergeStableTagCond(SNode** pWhere, SNode* pTagCond) { return code; } -static int32_t appendStableTagCond(SNode** pWhere, SNode* pTagCond) { +EDealRes rewriteAuthTable(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + SColumnNode* pCol = (SColumnNode*)pNode; + SAuthRewriteCxt* pCxt = (SAuthRewriteCxt*)pContext; + strcpy(pCol->tableName, pCxt->pTarget->tableName); + strcpy(pCol->tableAlias, pCxt->pTarget->tableAlias); + } + + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteAppendStableTagCond(SNode** pWhere, SNode* pTagCond, STableNode* pTable) { SNode* pTagCondCopy = nodesCloneNode(pTagCond); if (NULL == pTagCondCopy) { return TSDB_CODE_OUT_OF_MEMORY; } + SAuthRewriteCxt cxt = {.pTarget = pTable}; + nodesWalkExpr(pTagCondCopy, rewriteAuthTable, &cxt); + if (NULL == *pWhere) { *pWhere = pTagCondCopy; return TSDB_CODE_SUCCESS; @@ -117,7 +135,7 @@ static EDealRes authSelectImpl(SNode* pNode, void* pContext) { STableNode* pTable = (STableNode*)pNode; pAuthCxt->errCode = checkAuth(pAuthCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_READ, &pTagCond); if (TSDB_CODE_SUCCESS == pAuthCxt->errCode && NULL != pTagCond) { - pAuthCxt->errCode = appendStableTagCond(&pCxt->pSelect->pWhere, pTagCond); + pAuthCxt->errCode = rewriteAppendStableTagCond(&pCxt->pSelect->pWhere, pTagCond, pTable); } return TSDB_CODE_SUCCESS == pAuthCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } else if (QUERY_NODE_TEMP_TABLE == nodeType(pNode)) { @@ -152,7 +170,7 @@ static int32_t authDelete(SAuthCxt* pCxt, SDeleteStmt* pDelete) { STableNode* pTable = (STableNode*)pDelete->pFromTable; int32_t code = checkAuth(pCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_WRITE, &pTagCond); if (TSDB_CODE_SUCCESS == code && NULL != pTagCond) { - code = appendStableTagCond(&pDelete->pWhere, pTagCond); + code = rewriteAppendStableTagCond(&pDelete->pWhere, pTagCond, pTable); } return code; } @@ -175,7 +193,7 @@ static int32_t authShowTables(SAuthCxt* pCxt, SShowStmt* pStmt) { static int32_t authShowCreateTable(SAuthCxt* pCxt, SShowCreateTableStmt* pStmt) { SNode* pTagCond = NULL; // todo check tag condition for subtable - return checkAuth(pCxt, pStmt->dbName, NULL, AUTH_TYPE_READ, &pTagCond); + return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ, &pTagCond); } static int32_t authCreateTable(SAuthCxt* pCxt, SCreateTableStmt* pStmt) { diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 01b62a90511fd933239f806160ca6760299b7f95..c7219c47882212bf38a0ff8da556b02c7b586ab3 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -311,6 +311,9 @@ static int32_t calcConstDelete(SCalcConstContext* pCxt, SDeleteStmt* pDelete) { if (TSDB_CODE_SUCCESS == code) { code = calcConstStmtCondition(pCxt, &pDelete->pWhere, &pDelete->deleteZeroRows); } + if (code == TSDB_CODE_SUCCESS && pDelete->timeRange.skey > pDelete->timeRange.ekey) { + pDelete->deleteZeroRows = true; + } return code; } @@ -369,18 +372,33 @@ static bool notRefByOrderBy(SColumnNode* pCol, SNodeList* pOrderByList) { return !cxt.hasThisCol; } +static bool isDistinctSubQuery(SNode* pNode) { + if (NULL == pNode) { + return false; + } + switch (nodeType(pNode)) { + case QUERY_NODE_SELECT_STMT: + return ((SSelectStmt*)pNode)->isDistinct; + case QUERY_NODE_SET_OPERATOR: + return isDistinctSubQuery((((SSetOperator*)pNode)->pLeft)) || isDistinctSubQuery((((SSetOperator*)pNode)->pLeft)); + default: + break; + } + return false; +} + static bool isSetUselessCol(SSetOperator* pSetOp, int32_t index, SExprNode* pProj) { if (!isUselessCol(pProj)) { return false; } SNodeList* pLeftProjs = getChildProjection(pSetOp->pLeft); - if (!isUselessCol((SExprNode*)nodesListGetNode(pLeftProjs, index))) { + if (!isUselessCol((SExprNode*)nodesListGetNode(pLeftProjs, index)) || isDistinctSubQuery(pSetOp->pLeft)) { return false; } SNodeList* pRightProjs = getChildProjection(pSetOp->pRight); - if (!isUselessCol((SExprNode*)nodesListGetNode(pRightProjs, index))) { + if (!isUselessCol((SExprNode*)nodesListGetNode(pRightProjs, index)) || isDistinctSubQuery(pSetOp->pLeft)) { return false; } @@ -465,6 +483,9 @@ static bool isEmptyResultQuery(SNode* pStmt) { } break; } + case QUERY_NODE_DELETE_STMT: + isEmptyResult = ((SDeleteStmt*)pStmt)->deleteZeroRows; + break; default: break; } diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index dbf5931e8a32c2c5088bdd6d63598c273a25e96d..78b05b6df57d10d5f7d54669353d8906556a72d4 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -127,14 +127,15 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem if(kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || kv->type != pTagSchema->type){ code = TSDB_CODE_SML_INVALID_DATA; - uError("SML smlBuildCol error col not same %s", pTagSchema->name); + uError("SML smlBuildTagRow error col not same %s", pTagSchema->name); goto end; } taosArrayPush(*tagName, pTagSchema->name); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; // strcpy(val.colName, pTagSchema->name); - if (pTagSchema->type == TSDB_DATA_TYPE_BINARY) { + if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || + pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) { val.pData = (uint8_t*)kv->value; val.nData = kv->length; } else if (pTagSchema->type == TSDB_DATA_TYPE_NCHAR) { @@ -209,7 +210,7 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 SSmlKv* kv = (SSmlKv*)data; if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){ ret = TSDB_CODE_SML_INVALID_DATA; - uError("SML smlBuildCol error col not same %s", pColSchema->name); + uInfo("SML smlBuildCol error col not same %s", pColSchema->name); goto end; } if (kv->type == TSDB_DATA_TYPE_NCHAR) { @@ -236,7 +237,7 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 } pVal->value.pData = pUcs4; pVal->value.nData = len; - } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + } else if (kv->type == TSDB_DATA_TYPE_BINARY || kv->type == TSDB_DATA_TYPE_GEOMETRY) { pVal->value.nData = kv->length; pVal->value.pData = (uint8_t*)kv->value; } else { @@ -363,7 +364,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc } pVal->value.pData = pUcs4; pVal->value.nData = len; - } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + } else if (kv->type == TSDB_DATA_TYPE_BINARY || kv->type == TSDB_DATA_TYPE_GEOMETRY) { pVal->value.nData = kv->length; pVal->value.pData = (uint8_t*)kv->value; } else { diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 02de9f227d1f4a8afdf8de83e71b1aabd0e78069..8d35674949801893500f63d1a6e7a9e94683f47a 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -18,6 +18,7 @@ #include "scalar.h" #include "tglobal.h" #include "ttime.h" +#include "geosWrapper.h" #define NEXT_TOKEN_WITH_PREV(pSql, token) \ do { \ @@ -305,11 +306,32 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t return TSDB_CODE_SUCCESS; } +// need to call geosFreeBuffer(*output) later +static int parseGeometry(SToken *pToken, unsigned char **output, size_t *size) { + int32_t code = TSDB_CODE_FAILED; + + //[ToDo] support to parse WKB as well as WKT + if (pToken->type == TK_NK_STRING) { + code = initCtxGeomFromText(); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doGeomFromText(pToken->z, output, size); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + return code; +} + static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val, SMsgBuf* pMsgBuf) { int64_t iv; uint64_t uv; char* endptr = NULL; + int32_t code = TSDB_CODE_SUCCESS; if (isNullValue(pSchema->type, pToken)) { if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { @@ -456,6 +478,30 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, break; } + case TSDB_DATA_TYPE_GEOMETRY: { + unsigned char* output = NULL; + size_t size = 0; + + code = parseGeometry(pToken, &output, &size); + if (code != TSDB_CODE_SUCCESS) { + code = buildSyntaxErrMsg(pMsgBuf, getThreadLocalGeosCtx()->errMsg, pToken->z); + } else if (size + VARSTR_HEADER_SIZE > pSchema->bytes) { + // Too long values will raise the invalid sql error message + code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + } else { + val->pData = taosMemoryMalloc(size); + if (NULL == val->pData) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + memcpy(val->pData, output, size); + val->nData = size; + } + } + + geosFreeBuffer(output); + break; + } + case TSDB_DATA_TYPE_NCHAR: { int32_t output = 0; void* p = taosMemoryCalloc(1, pSchema->bytes - VARSTR_HEADER_SIZE); @@ -486,7 +532,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } } - return TSDB_CODE_SUCCESS; + return code; } // input pStmt->pSql: [(tag1_name, ...)] TAGS (tag1_value, ...) ... @@ -1350,6 +1396,37 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, pVal->value.nData = pToken->n; break; } + case TSDB_DATA_TYPE_GEOMETRY: { + int32_t code = TSDB_CODE_FAILED; + unsigned char *output = NULL; + size_t size = 0; + + code = parseGeometry(pToken, &output, &size); + if (code != TSDB_CODE_SUCCESS) { + code = buildSyntaxErrMsg(&pCxt->msg, getThreadLocalGeosCtx()->errMsg, pToken->z); + } + // Too long values will raise the invalid sql error message + else if (size + VARSTR_HEADER_SIZE > pSchema->bytes) { + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + } + else { + pVal->value.pData = taosMemoryMalloc(size); + if (NULL == pVal->value.pData) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + else { + memcpy(pVal->value.pData, output, size); + pVal->value.nData = size; + } + } + + geosFreeBuffer(output); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + break; + } case TSDB_DATA_TYPE_TIMESTAMP: { if (parseTime(pSql, pToken, timePrec, &pVal->value.val, &pCxt->msg) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); @@ -1552,7 +1629,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt (*pNumOfRows)++; } - if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxMemUsedByInsert * 1024 * 1024) { + if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxInsertBatchRows) { pStmt->fileProcessing = true; break; } @@ -1561,6 +1638,8 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt } taosMemoryFree(pLine); + parserDebug("0x%" PRIx64 " %d rows have been parsed", pCxt->pComCxt->requestId, *pNumOfRows); + if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) && (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) && !pStmt->fileProcessing) { code = buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 922a0f45ff38fba30fc4b766a3b6427026b49b36..bcbea6cd2e67b803f364dddeb22c7af6e4d902d8 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -128,7 +128,8 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch } else { STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; // strcpy(val.colName, pTagSchema->name); - if (pTagSchema->type == TSDB_DATA_TYPE_BINARY) { + if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || + pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) { val.pData = (uint8_t*)bind[c].buffer; val.nData = colLen; } else if (pTagSchema->type == TSDB_DATA_TYPE_NCHAR) { @@ -266,7 +267,10 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in pBind = bind + c; } - tColDataAddValueByBind(pCol, pBind); + code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1); + if (code) { + goto _return; + } } qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum); @@ -309,7 +313,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu pBind = bind; } - tColDataAddValueByBind(pCol, pBind); + tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1); qDebug("stmt col %d bind %d rows data", colIdx, rowNum); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index f921094752f337796b66c24c27db42fd8e54d5eb..de7d154db6a8ecf604fa2197ad09e323ae185a43 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -272,6 +272,41 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat return code; } +static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) { + int32_t code = TSDB_CODE_SUCCESS; + SSubmitTbData* pTmp = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == pTmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pTmp->flags = pSrc->flags; + pTmp->suid = pSrc->suid; + pTmp->uid = pSrc->uid; + pTmp->sver = pSrc->sver; + pTmp->pCreateTbReq = NULL; + if (pTmp->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + pTmp->aCol = taosArrayInit(128, sizeof(SColData)); + if (NULL == pTmp->aCol) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pTmp); + } + } else { + pTmp->aRowP = taosArrayInit(128, POINTER_BYTES); + if (NULL == pTmp->aRowP) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pTmp); + } + } + } + + taosMemoryFree(pSrc); + if (TSDB_CODE_SUCCESS == code) { + *pDst = pTmp; + } + + return code; +} + + static void resetColValues(SArray* pValues) { int32_t num = taosArrayGetSize(pValues); for (int32_t i = 0; i < num; ++i) { @@ -381,7 +416,7 @@ static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCx } } taosArrayPush(pVgCxt->pData->aSubmitTbData, pTableCxt->pData); - taosMemoryFreeClear(pTableCxt->pData); + rebuildTableData(pTableCxt->pData, &pTableCxt->pData); qDebug("add tableDataCxt uid:%" PRId64 " to vgId:%d", pTableCxt->pMeta->uid, pVgCxt->vgId); diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 5c1f4bf98c693f41f02fc9f6d9c14170988311d0..ca7ac1a0b666b556f45b91b3b0dad18e8499561f 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -103,6 +103,7 @@ static SKeyword keywordTable[] = { {"FORCE", TK_FORCE}, {"FUNCTION", TK_FUNCTION}, {"FUNCTIONS", TK_FUNCTIONS}, + {"GEOMETRY", TK_GEOMETRY}, {"GRANT", TK_GRANT}, {"GRANTS", TK_GRANTS}, {"GROUP", TK_GROUP}, @@ -139,6 +140,7 @@ static SKeyword keywordTable[] = { {"MAX_SPEED", TK_MAX_SPEED}, {"MERGE", TK_MERGE}, {"META", TK_META}, + {"ONLY", TK_ONLY}, {"MINROWS", TK_MINROWS}, {"MINUS", TK_MINUS}, {"MNODE", TK_MNODE}, @@ -241,6 +243,7 @@ static SKeyword keywordTable[] = { {"TSERIES", TK_TSERIES}, {"TTL", TK_TTL}, {"UNION", TK_UNION}, + {"UNSAFE", TK_UNSAFE}, {"UNSIGNED", TK_UNSIGNED}, {"UNTREATED", TK_UNTREATED}, {"UPDATE", TK_UPDATE}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c5fe3a1f73314eff38f33ebe683cf0692f0c6efa..38118c03f85f268f539d5707add4549afea8064e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -53,8 +53,17 @@ typedef struct STranslateContext { bool createStream; bool stableQuery; bool showRewrite; + SNode* pPrevRoot; + SNode* pPostRoot; } STranslateContext; +typedef struct SBuildTopicContext { + bool colExists; + bool colNotFound; + STableMeta* pMeta; + SNodeList* pTags; +} SBuildTopicContext; + typedef struct SFullDatabaseName { char fullDbName[TSDB_DB_FNAME_LEN]; } SFullDatabaseName; @@ -269,6 +278,10 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = { static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode); static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode); static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal); +static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt); +static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta, SNode** pQuery); +static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery); +static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery); static bool afterGroupBy(ESqlClause clause) { return clause > SQL_CLAUSE_GROUP_BY; } @@ -496,6 +509,10 @@ static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int } static int32_t getDBCfg(STranslateContext* pCxt, const char* pDbName, SDbCfgInfo* pInfo) { + if (IS_SYS_DBNAME(pDbName)) { + return TSDB_CODE_SUCCESS; + } + SParseContext* pParCxt = pCxt->pParseCxt; SName name; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pDbName, strlen(pDbName)); @@ -700,6 +717,10 @@ static bool isWindowPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsWindowPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); } +static bool isInterpFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsInterpFunc(((SFunctionNode*)pNode)->funcId)); +} + static bool isInterpPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsInterpPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); } @@ -751,18 +772,40 @@ static SNodeList* getProjectList(const SNode* pNode) { static bool isTimeLineQuery(SNode* pStmt) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { - return ((SSelectStmt*)pStmt)->isTimeLineResult; + return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineResMode) || (TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineResMode); + } else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) { + return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode; } else { return false; } } static bool isGlobalTimeLineQuery(SNode* pStmt) { - if (!isTimeLineQuery(pStmt)) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + return TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineResMode; + } else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) { + return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode; + } else { return false; } - SSelectStmt* pSelect = (SSelectStmt*)pStmt; - return NULL == pSelect->pPartitionByList || NULL != pSelect->pOrderByList; +} + +static bool isTimeLineAlignedQuery(SNode* pStmt) { + SSelectStmt *pSelect = (SSelectStmt *)pStmt; + if (isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + return true; + } + if (!isTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + return false; + } + if (QUERY_NODE_SELECT_STMT != nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + return false; + } + SSelectStmt *pSub = (SSelectStmt *)((STempTableNode*)pSelect->pFromTable)->pSubquery; + if (nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) { + return true; + } + return false; } static bool isPrimaryKeyImpl(SNode* pExpr) { @@ -778,7 +821,19 @@ static bool isPrimaryKeyImpl(SNode* pExpr) { FUNCTION_TYPE_IROWTS == pFunc->funcType) { return true; } - } + } else if (QUERY_NODE_OPERATOR == nodeType(pExpr)) { + SOperatorNode* pOper = (SOperatorNode*)pExpr; + if (OP_TYPE_ADD != pOper->opType && OP_TYPE_SUB != pOper->opType) { + return false; + } + if (!isPrimaryKeyImpl(pOper->pLeft)) { + return false; + } + if (QUERY_NODE_VALUE != nodeType(pOper->pRight)) { + return false; + } + return true; + } return false; } @@ -827,7 +882,7 @@ static void setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColum strcpy(pCol->node.aliasName, pCol->colName); } if ('\0' == pCol->node.userAlias[0]) { - strcpy(pCol->node.userAlias, pCol->colName); + strcpy(pCol->node.userAlias, pExpr->userAlias); } pCol->node.resType = pExpr->resType; } @@ -839,6 +894,7 @@ static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* p (igTags ? 0 : ((TSDB_SUPER_TABLE == pMeta->tableType) ? pMeta->tableInfo.numOfTags : 0)); for (int32_t i = 0; i < nums; ++i) { if (invisibleColumn(pCxt->pParseCxt->enableSysInfo, pMeta->tableType, pMeta->schema[i].flags)) { + pCxt->pParseCxt->hasInvisibleCol = true; continue; } SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); @@ -877,6 +933,10 @@ static int32_t findAndSetColumn(STranslateContext* pCxt, SColumnNode** pColRef, if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; if (isInternalPrimaryKey(pCol)) { + if (TSDB_SYSTEM_TABLE == pMeta->tableType) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pCol->colName); + } + setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema, -1, pCol); *pFound = true; return TSDB_CODE_SUCCESS; @@ -1168,7 +1228,8 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, break; } case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_VARBINARY: { + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_GEOMETRY: { if (strict && (pVal->node.resType.bytes > targetDt.bytes - VARSTR_HEADER_SIZE)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } @@ -1241,7 +1302,7 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD } static int32_t calcTypeBytes(SDataType dt) { - if (TSDB_DATA_TYPE_BINARY == dt.type) { + if (TSDB_DATA_TYPE_BINARY == dt.type || TSDB_DATA_TYPE_GEOMETRY == dt.type) { return dt.bytes + VARSTR_HEADER_SIZE; } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { return dt.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; @@ -1344,13 +1405,33 @@ static bool isCountStar(SFunctionNode* pFunc) { return (QUERY_NODE_COLUMN == nodeType(pPara) && 0 == strcmp(((SColumnNode*)pPara)->colName, "*")); } +static int32_t rewriteCountStarAsCount1(STranslateContext* pCxt, SFunctionNode* pCount) { + int32_t code = TSDB_CODE_SUCCESS; + SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == pVal) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pVal->node.resType.type = TSDB_DATA_TYPE_INT; + pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; + const int32_t val = 1; + nodesSetValueNodeValue(pVal, (void*)&val); + pVal->translate = true; + nodesListErase(pCount->pParameterList, nodesListGetCell(pCount->pParameterList, 0)); + code = nodesListAppend(pCount->pParameterList, (SNode*)pVal); + return code; +} + // count(*) is rewritten as count(ts) for scannning optimization static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) { SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pCount->pParameterList, 0); STableNode* pTable = NULL; int32_t code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable); - if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { - setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol); + if (TSDB_CODE_SUCCESS == code) { + if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol); + } else { + code = rewriteCountStarAsCount1(pCxt, pCount); + } } return code; } @@ -1523,14 +1604,15 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; SNode* pTable = pSelect->pFromTable; - if ((NULL != pTable && QUERY_NODE_REAL_TABLE != nodeType(pTable))) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE, - "%s is only supported in single table query", pFunc->functionName); - } - if (pSelect->hasAggFuncs || pSelect->hasMultiRowsFunc || pSelect->hasIndefiniteRowsFunc) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } + + if (pSelect->hasInterpFunc && (FUNC_RETURN_ROWS_INDEFINITE == pSelect->returnRows || pSelect->returnRows != fmGetFuncReturnRows(pFunc))) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, + "%s ignoring null value options cannot be used when applying to multiple columns", pFunc->functionName); + } + if (NULL != pSelect->pWindow || NULL != pSelect->pGroupByList) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "%s function is not supported in window query or group query", pFunc->functionName); @@ -1566,7 +1648,7 @@ static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFu } SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; if (NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && - !isTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery) && !isTimeLineAlignedQuery(pCxt->pCurrStmt)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "%s function requires valid time series input", pFunc->functionName); } @@ -1631,6 +1713,20 @@ static int32_t translateForbidStreamFunc(STranslateContext* pCxt, SFunctionNode* return TSDB_CODE_SUCCESS; } +static int32_t translateForbidSysTableFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsForbidSysTableFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + + SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; + SNode* pTable = pSelect->pFromTable; + if (NULL != pTable && QUERY_NODE_REAL_TABLE == nodeType(pTable) && + TSDB_SYSTEM_TABLE == ((SRealTableNode*)pTable)->pMeta->tableType) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC, pFunc->functionName); + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsRepeatScanFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1699,7 +1795,7 @@ static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFu "%s(*) is only supported in SELECTed list", pFunc->functionName); } } - if (tsKeepColumnName && 1 == LIST_LENGTH(pFunc->pParameterList) && !pFunc->node.asAlias) { + if (tsKeepColumnName && 1 == LIST_LENGTH(pFunc->pParameterList) && !pFunc->node.asAlias && !pFunc->node.asParam) { strcpy(pFunc->node.userAlias, ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->userAlias); strcpy(pFunc->node.aliasName, pFunc->node.userAlias); } @@ -1731,7 +1827,10 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { if (fmIsIndefiniteRowsFunc(pFunc->funcId)) { pSelect->hasIndefiniteRowsFunc = true; pSelect->returnRows = fmGetFuncReturnRows(pFunc); + } else if (fmIsInterpFunc(pFunc->funcId)) { + pSelect->returnRows = fmGetFuncReturnRows(pFunc); } + pSelect->hasMultiRowsFunc = pSelect->hasMultiRowsFunc ? true : fmIsMultiRowsFunc(pFunc->funcId); if (fmIsSelectFunc(pFunc->funcId)) { pSelect->hasSelectFunc = true; @@ -1758,6 +1857,7 @@ static int32_t rewriteFuncToValue(STranslateContext* pCxt, char* pLiteral, SNode return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pVal->node.aliasName, ((SExprNode*)*pNode)->aliasName); + strcpy(pVal->node.userAlias, ((SExprNode*)*pNode)->userAlias); pVal->node.resType = ((SExprNode*)*pNode)->resType; if (NULL == pLiteral) { pVal->isNull = true; @@ -1858,6 +1958,9 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SFunctionNode* p if (TSDB_CODE_SUCCESS == code) { code = translateForbidStreamFunc(pCxt, pFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = translateForbidSysTableFunc(pCxt, pFunc); + } if (TSDB_CODE_SUCCESS == code) { code = translateRepeatScanFunc(pCxt, pFunc); } @@ -2205,7 +2308,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { } } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { - if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) { + if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc || (isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) { return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); } else { return rewriteColToSelectValFunc(pCxt, pNode); @@ -2276,7 +2379,7 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) return TSDB_CODE_SUCCESS; } if (!pSelect->onlyHasKeepOrderFunc) { - pSelect->isTimeLineResult = false; + pSelect->timeLineResMode = TIME_LINE_NONE; } CheckAggColCoexistCxt cxt = {.pTranslateCxt = pCxt, .existCol = false}; nodesRewriteExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); @@ -2296,7 +2399,7 @@ static int32_t checkHavingGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) int32_t code = TSDB_CODE_SUCCESS; if (NULL == getGroupByList(pCxt) && NULL == pSelect->pPartitionByList && NULL == pSelect->pWindow) { return code; - } + } if (NULL != pSelect->pHaving) { code = checkExprForGroupBy(pCxt, &pSelect->pHaving); } @@ -2307,7 +2410,7 @@ static int32_t checkHavingGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pOrderByList) { code = checkExprListForGroupBy(pCxt, pSelect, pSelect->pOrderByList); } -*/ +*/ return code; } @@ -2615,7 +2718,7 @@ static int32_t replaceTbName(STranslateContext* pCxt, SSelectStmt* pSelect) { SNode** pNode = NULL; SRewriteTbNameContext pRewriteCxt = {0}; pRewriteCxt.pTbName = pTable->table.tableName; - + nodesRewriteExprPostOrder(&pSelect->pWhere, doTranslateTbName, &pRewriteCxt); return pRewriteCxt.errCode; @@ -2623,9 +2726,9 @@ static int32_t replaceTbName(STranslateContext* pCxt, SSelectStmt* pSelect) { static int32_t checkJoinTable(STranslateContext* pCxt, SJoinTableNode* pJoinTable) { if ((QUERY_NODE_TEMP_TABLE == nodeType(pJoinTable->pLeft) && - !isTimeLineQuery(((STempTableNode*)pJoinTable->pLeft)->pSubquery)) || + !isGlobalTimeLineQuery(((STempTableNode*)pJoinTable->pLeft)->pSubquery)) || (QUERY_NODE_TEMP_TABLE == nodeType(pJoinTable->pRight) && - !isTimeLineQuery(((STempTableNode*)pJoinTable->pRight)->pSubquery))) { + !isGlobalTimeLineQuery(((STempTableNode*)pJoinTable->pRight)->pSubquery))) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SUPPORT_JOIN, "Join requires valid time series input"); } @@ -2660,7 +2763,7 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { } if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType) { if (isSelectStmt(pCxt->pCurrStmt)) { - ((SSelectStmt*)pCxt->pCurrStmt)->isTimeLineResult = false; + ((SSelectStmt*)pCxt->pCurrStmt)->timeLineResMode = TIME_LINE_NONE; } else if (isDeleteStmt(pCxt->pCurrStmt)) { code = TSDB_CODE_TSC_INVALID_OPERATION; break; @@ -2756,6 +2859,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { } else { len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->aliasName); strncpy(pFunc->node.aliasName, buf, TMIN(len, sizeof(pFunc->node.aliasName) - 1)); + len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->userAlias); strncpy(pFunc->node.userAlias, buf, TMIN(len, sizeof(pFunc->node.userAlias) - 1)); } @@ -2842,14 +2946,14 @@ static int32_t createMultiResFuncsFromStar(STranslateContext* pCxt, SFunctionNod static int32_t createTags(STranslateContext* pCxt, SNodeList** pOutput) { if (QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_PC, - "The _TAGS pseudo column can only be used for subtable and supertable queries"); + "The _TAGS pseudo column can only be used for child table and super table queries"); } SRealTableNode* pTable = (SRealTableNode*)(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable); const STableMeta* pMeta = pTable->pMeta; if (TSDB_SUPER_TABLE != pMeta->tableType && TSDB_CHILD_TABLE != pMeta->tableType) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_PC, - "The _TAGS pseudo column can only be used for subtable and supertable queries"); + "The _TAGS pseudo column can only be used for child table and super table queries"); } SSchema* pTagsSchema = getTableTagSchema(pMeta); @@ -2916,6 +3020,7 @@ static int32_t getPositionValue(const SValueNode* pVal) { case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_GEOMETRY: return -1; case TSDB_DATA_TYPE_BOOL: return (pVal->datum.b ? 1 : 0); @@ -2993,7 +3098,7 @@ static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { } static EDealRes needFillImpl(SNode* pNode, void* pContext) { - if (isAggFunc(pNode) && FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pNode)->funcType) { + if ((isAggFunc(pNode) || isInterpFunc(pNode)) && FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pNode)->funcType) { *(bool*)pContext = true; return DEAL_RES_END; } @@ -3008,16 +3113,16 @@ static bool needFill(SNode* pNode) { static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList* pValues, int32_t index) { SListCell* pCell = nodesListGetCell(pValues, index); - if (dataTypeEqual(&dt, &((SExprNode*)pCell->pNode)->resType)) { + if (dataTypeEqual(&dt, &((SExprNode*)pCell->pNode)->resType) && (QUERY_NODE_VALUE == nodeType(pCell->pNode))) { return TSDB_CODE_SUCCESS; } - SNode* pCaseFunc = NULL; - int32_t code = createCastFunc(pCxt, pCell->pNode, dt, &pCaseFunc); + SNode* pCastFunc = NULL; + int32_t code = createCastFunc(pCxt, pCell->pNode, dt, &pCastFunc); if (TSDB_CODE_SUCCESS == code) { - code = scalarCalculateConstants(pCaseFunc, &pCell->pNode); + code = scalarCalculateConstants(pCastFunc, &pCell->pNode); } if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE != nodeType(pCell->pNode)) { - code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value is just a constant"); + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value can only accept constant"); } else if (TSDB_CODE_SUCCESS != code) { code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch"); } @@ -3041,6 +3146,7 @@ static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeL if (TSDB_CODE_SUCCESS != code) { return code; } + ++fillNo; } } @@ -3109,12 +3215,16 @@ static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect } if (TSDB_CODE_SUCCESS == code) { code = checkExprListForGroupBy(pCxt, pSelect, pSelect->pProjectionList); - } + } if (TSDB_CODE_SUCCESS == code) { code = translateFillValues(pCxt, pSelect); } if (NULL == pSelect->pProjectionList || 0 >= pSelect->pProjectionList->length) { - code = TSDB_CODE_PAR_INVALID_SELECTED_EXPR; + if (pCxt->pParseCxt->hasInvisibleCol) { + code = TSDB_CODE_PAR_PERMISSION_DENIED; + } else { + code = TSDB_CODE_PAR_INVALID_SELECTED_EXPR; + } } return code; } @@ -3137,7 +3247,7 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST); } pCxt->currClause = SQL_CLAUSE_GROUP_BY; - pSelect->isTimeLineResult = false; + pSelect->timeLineResMode = TIME_LINE_NONE; return translateExprList(pCxt, pSelect->pGroupByList); } @@ -3198,23 +3308,25 @@ static int32_t checkFill(STranslateContext* pCxt, SFillNode* pFill, SValueNode* if (NULL == pInterval) { return TSDB_CODE_SUCCESS; } - - int64_t timeRange = TABS(pFill->timeRange.skey - pFill->timeRange.ekey); + int64_t timeRange = 0; int64_t intervalRange = 0; - if (TIME_IS_VAR_DURATION(pInterval->unit)) { - int64_t f = 1; - if (pInterval->unit == 'n') { - f = 30LL * MILLISECOND_PER_DAY; - } else if (pInterval->unit == 'y') { - f = 365LL * MILLISECOND_PER_DAY; - } - intervalRange = pInterval->datum.i * f; - } else { - intervalRange = pInterval->datum.i; - } - - if ((timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); + if (!pCxt->createStream) { + int64_t res = int64SafeSub(pFill->timeRange.skey, pFill->timeRange.ekey); + timeRange = res < 0 ? res == INT64_MIN ? INT64_MAX : -res : res; + if (IS_CALENDAR_TIME_DURATION(pInterval->unit)) { + int64_t f = 1; + if (pInterval->unit == 'n') { + f = 30LL * MILLISECOND_PER_DAY; + } else if (pInterval->unit == 'y') { + f = 365LL * MILLISECOND_PER_DAY; + } + intervalRange = pInterval->datum.i * f; + } else { + intervalRange = pInterval->datum.i; + } + if ((timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); + } } return TSDB_CODE_SUCCESS; @@ -3282,7 +3394,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; SValueNode* pInter = (SValueNode*)pInterval->pInterval; - bool valInter = TIME_IS_VAR_DURATION(pInter->unit); + bool valInter = IS_CALENDAR_TIME_DURATION(pInter->unit); if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime, getPrecisionStr(precision)); @@ -3296,7 +3408,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* if (pInter->unit == 'n' && pOffset->unit == 'y') { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT); } - bool fixed = !TIME_IS_VAR_DURATION(pOffset->unit) && !valInter; + bool fixed = !IS_CALENDAR_TIME_DURATION(pOffset->unit) && !valInter; if ((fixed && pOffset->datum.i >= pInter->datum.i) || (!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >= getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { @@ -3312,10 +3424,11 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* const static int32_t INTERVAL_SLIDING_FACTOR = 100; SValueNode* pSliding = (SValueNode*)pInterval->pSliding; - if (TIME_IS_VAR_DURATION(pSliding->unit)) { + if (IS_CALENDAR_TIME_DURATION(pSliding->unit)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT); } - if ((pSliding->datum.i < convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, precision)) || + if ((pSliding->datum.i < + convertTimeFromPrecisionToUnit(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, pSliding->unit)) || (pInter->datum.i / pSliding->datum.i > INTERVAL_SLIDING_FACTOR)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL); } @@ -3431,6 +3544,10 @@ static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; } + if (pSelect->pFromTable->type == QUERY_NODE_REAL_TABLE && + ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType == TSDB_SYSTEM_TABLE) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED, "WINDOW"); + } pCxt->currClause = SQL_CLAUSE_WINDOW; int32_t code = translateExpr(pCxt, &pSelect->pWindow); if (TSDB_CODE_SUCCESS == code) { @@ -3460,6 +3577,22 @@ static int32_t createDefaultFillNode(STranslateContext* pCxt, SNode** pOutput) { return TSDB_CODE_SUCCESS; } +static int32_t createDefaultEveryNode(STranslateContext* pCxt, SNode** pOutput) { + SValueNode* pEvery = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == pEvery) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pEvery->node.resType.type = TSDB_DATA_TYPE_BIGINT; + pEvery->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + pEvery->isDuration = true; + pEvery->literal = taosStrdup("1s"); + + + *pOutput = (SNode*)pEvery; + return TSDB_CODE_SUCCESS; +} + static int32_t checkEvery(STranslateContext* pCxt, SValueNode* pInterval) { int32_t len = strlen(pInterval->literal); @@ -3475,7 +3608,12 @@ static int32_t checkEvery(STranslateContext* pCxt, SValueNode* pInterval) { static int32_t translateInterpEvery(STranslateContext* pCxt, SNode** pEvery) { int32_t code = TSDB_CODE_SUCCESS; - code = checkEvery(pCxt, (SValueNode*)(*pEvery)); + if (NULL == *pEvery) { + code = createDefaultEveryNode(pCxt, pEvery); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkEvery(pCxt, (SValueNode*)(*pEvery)); + } if (TSDB_CODE_SUCCESS == code) { code = translateExpr(pCxt, pEvery); } @@ -3504,6 +3642,9 @@ static int32_t translateInterpFill(STranslateContext* pCxt, SSelectStmt* pSelect if (TSDB_CODE_SUCCESS == code) { code = checkFill(pCxt, (SFillNode*)pSelect->pFill, (SValueNode*)pSelect->pEvery, true); } + if (TSDB_CODE_SUCCESS == code) { + code = checkFillValues(pCxt, (SFillNode*)pSelect->pFill, pSelect->pProjectionList); + } return code; } @@ -3521,8 +3662,12 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { } if (NULL == pSelect->pRange || NULL == pSelect->pEvery || NULL == pSelect->pFill) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, - "Missing RANGE clause, EVERY clause or FILL clause"); + if (pSelect->pRange != NULL && QUERY_NODE_OPERATOR == nodeType(pSelect->pRange) && pSelect->pEvery == NULL) { + // single point interp every can be omitted + } else { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, + "Missing RANGE clause, EVERY clause or FILL clause"); + } } int32_t code = translateExpr(pCxt, &pSelect->pRange); @@ -3537,7 +3682,18 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->currClause = SQL_CLAUSE_PARTITION_BY; - int32_t code = translateExprList(pCxt, pSelect->pPartitionByList); + int32_t code = TSDB_CODE_SUCCESS; + + if (pSelect->pPartitionByList) { + int8_t typeType = getTableTypeFromTableNode(pSelect->pFromTable); + SNode* pPar = nodesListGetNode(pSelect->pPartitionByList, 0); + if (!((TSDB_NORMAL_TABLE == typeType || TSDB_CHILD_TABLE == typeType) && + 1 == pSelect->pPartitionByList->length && (QUERY_NODE_FUNCTION == nodeType(pPar) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPar)->funcType))) { + pSelect->timeLineResMode = TIME_LINE_MULTI; + } + + code = translateExprList(pCxt, pSelect->pPartitionByList); + } if (TSDB_CODE_SUCCESS == code) { code = translateExprList(pCxt, pSelect->pTags); } @@ -3666,9 +3822,9 @@ static void resetResultTimeline(SSelectStmt* pSelect) { if ((QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && isPrimaryKey((STempTableNode*)pSelect->pFromTable, pOrder)) || (QUERY_NODE_TEMP_TABLE != nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder))) { - pSelect->isTimeLineResult = true; + pSelect->timeLineResMode = TIME_LINE_GLOBAL; } else { - pSelect->isTimeLineResult = false; + pSelect->timeLineResMode = TIME_LINE_NONE; } } @@ -3738,7 +3894,7 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect if (TSDB_CODE_SUCCESS == code) { code = replaceTbName(pCxt, pSelect); } - + return code; } @@ -3798,8 +3954,13 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS pLeftExpr = pLeftFuncExpr; } snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName); - if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, - createSetOperProject(pSetOperator->stmtName, pLeft))) { + SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft); + if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight) + && ((SColumnNode*)pLeft)->colId == PRIMARYKEY_TIMESTAMP_COL_ID + && ((SColumnNode*)pRight)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + ((SColumnNode*)pProj)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } + if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, pProj)) { return TSDB_CODE_OUT_OF_MEMORY; } } @@ -3811,6 +3972,10 @@ static uint8_t calcSetOperatorPrecision(SSetOperator* pSetOperator) { } static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pSetOperator) { + if (NULL == pSetOperator->pOrderByList || pSetOperator->pOrderByList->length <= 0) { + return TSDB_CODE_SUCCESS; + } + bool other; int32_t code = translateOrderByPosition(pCxt, pSetOperator->pProjectionList, pSetOperator->pOrderByList, &other); if (TSDB_CODE_SUCCESS == code) { @@ -3823,6 +3988,14 @@ static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pS if (TSDB_CODE_SUCCESS == code) { code = replaceOrderByAlias(pCxt, pSetOperator->pProjectionList, pSetOperator->pOrderByList); } + if (TSDB_CODE_SUCCESS == code) { + SNode* pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, 0))->pExpr; + if (isPrimaryKeyImpl(pOrder)) { + pSetOperator->timeLineResMode = TIME_LINE_GLOBAL; + } else { + pSetOperator->timeLineResMode = TIME_LINE_NONE; + } + } return code; } @@ -4611,7 +4784,8 @@ static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SN } if (TSDB_CODE_SUCCESS == code) { if ((TSDB_DATA_TYPE_VARCHAR == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN) || - (TSDB_DATA_TYPE_NCHAR == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN)) { + (TSDB_DATA_TYPE_NCHAR == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN) || + (TSDB_DATA_TYPE_GEOMETRY == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN)) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } } @@ -4662,7 +4836,8 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in } if (TSDB_CODE_SUCCESS == code) { if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_BINARY_LEN) || - (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_NCHAR_LEN)) { + (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_NCHAR_LEN) || + (TSDB_DATA_TYPE_GEOMETRY == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_GEOMETRY_LEN)) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } } @@ -4839,6 +5014,14 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt if (TSDB_CODE_SUCCESS == code) { code = checkTableSchema(pCxt, pStmt); } + if (TSDB_CODE_SUCCESS == code) { + if (createStable && pStmt->pOptions->ttl != 0) { + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TABLE_OPTION, + "Only supported for create non-super table in databases " + "configured with the 'TTL' option"); + } + } + return code; } @@ -4873,6 +5056,7 @@ static int32_t buildTableForSampleAst(SSampleAstInfo* pInfo, SNode** pOutput) { } snprintf(pTable->table.dbName, sizeof(pTable->table.dbName), "%s", pInfo->pDbName); snprintf(pTable->table.tableName, sizeof(pTable->table.tableName), "%s", pInfo->pTableName); + snprintf(pTable->table.tableAlias, sizeof(pTable->table.tableAlias), "%s", pInfo->pTableName); TSWAP(pTable->pMeta, pInfo->pRollupTableMeta); *pOutput = (SNode*)pTable; return TSDB_CODE_SUCCESS; @@ -5517,6 +5701,7 @@ static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt strcpy(dropReq.fqdn, pStmt->fqdn); dropReq.port = pStmt->port; dropReq.force = pStmt->force; + dropReq.unsafe = pStmt->unsafe; return buildCmdMsg(pCxt, TDMT_MND_DROP_DNODE, (FSerializeFunc)tSerializeSDropDnodeReq, &dropReq); } @@ -5646,6 +5831,15 @@ static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStm if (TSDB_CODE_SUCCESS == code) { code = getSmaIndexAst(pCxt, pStmt, &pReq->ast, &pReq->astLen, &pReq->expr, &pReq->exprLen); } + if (TSDB_CODE_SUCCESS == code) { + STableMeta* pMetaCache = NULL; + code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + pStmt->pOptions->tsPrecision = pMetaCache->tableInfo.precision; + code = createLastTsSelectStmt(pStmt->dbName, pStmt->tableName, pMetaCache, &pStmt->pPrevQuery); + } + taosMemoryFreeClear(pMetaCache); + } return code; } @@ -5671,15 +5865,60 @@ static int32_t checkCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pS } static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { - SMCreateSmaReq createSmaReq = {0}; int32_t code = checkCreateSmaIndex(pCxt, pStmt); + pStmt->pReq = taosMemoryCalloc(1, sizeof(SMCreateSmaReq)); + if (pStmt->pReq == NULL) code = TSDB_CODE_OUT_OF_MEMORY; + if (TSDB_CODE_SUCCESS == code) { + code = buildCreateSmaReq(pCxt, pStmt, pStmt->pReq); + } + TSWAP(pCxt->pPrevRoot, pStmt->pPrevQuery); + return code; +} + +int32_t createIntervalFromCreateSmaIndexStmt(SCreateIndexStmt* pStmt, SInterval* pInterval) { + pInterval->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i; + pInterval->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit; + pInterval->offset = NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0; + pInterval->sliding = NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pInterval->interval; + pInterval->slidingUnit = NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pInterval->intervalUnit; + pInterval->precision = pStmt->pOptions->tsPrecision; + return TSDB_CODE_SUCCESS; +} + +int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void ** pResRow) { + int32_t code = TSDB_CODE_SUCCESS; + SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pQuery->pRoot; + int64_t lastTs = 0; + SInterval interval = {0}; + STranslateContext pCxt = {0}; + code = initTranslateContext(pParseCxt, NULL, &pCxt); if (TSDB_CODE_SUCCESS == code) { - code = buildCreateSmaReq(pCxt, pStmt, &createSmaReq); + code = createIntervalFromCreateSmaIndexStmt(pStmt, &interval); + } + if (TSDB_CODE_SUCCESS == code) { + if (pResRow && pResRow[0]) { + lastTs = *(int64_t*)pResRow[0]; + } else if (interval.interval > 0) { + lastTs = convertTimePrecision(taosGetTimestampMs(), TSDB_TIME_PRECISION_MILLI, interval.precision); + } else { + lastTs = taosGetTimestampMs(); + } + } + if (TSDB_CODE_SUCCESS == code) { + if (interval.interval > 0) { + pStmt->pReq->lastTs = taosTimeTruncate(lastTs, &interval); + } else { + pStmt->pReq->lastTs = lastTs; + } + code = buildCmdMsg(&pCxt, TDMT_MND_CREATE_SMA, (FSerializeFunc)tSerializeSMCreateSmaReq, pStmt->pReq); } if (TSDB_CODE_SUCCESS == code) { - code = buildCmdMsg(pCxt, TDMT_MND_CREATE_SMA, (FSerializeFunc)tSerializeSMCreateSmaReq, &createSmaReq); + code = setQuery(&pCxt, pQuery); } - tFreeSMCreateSmaReq(&createSmaReq); + setRefreshMate(&pCxt, pQuery); + destroyTranslateContext(&pCxt); + tFreeSMCreateSmaReq(pStmt->pReq); + taosMemoryFreeClear(pStmt->pReq); return code; } @@ -5814,6 +6053,9 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); tNameGetFullDbName(&name, pReq->subDbName); tNameExtractFullName(&name, pReq->subStbName); + if(pStmt->pQuery != NULL) { + code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); + } } else if ('\0' != pStmt->subDbName[0]) { pReq->subType = TOPIC_SUB_TYPE__DB; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->subDbName, strlen(pStmt->subDbName)); @@ -5836,12 +6078,111 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS return code; } +static int32_t addTagList(SNodeList** ppList, SNode* pNode) { + if (NULL == *ppList) { + *ppList = nodesMakeList(); + } + + nodesListStrictAppend(*ppList, pNode); + + return TSDB_CODE_SUCCESS; +} + +static EDealRes checkColumnTagsInCond(SNode* pNode, void* pContext) { + SBuildTopicContext* pCxt = (SBuildTopicContext*)pContext; + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + ETableColumnType type; + getColumnTypeFromMeta(pCxt->pMeta, ((SColumnNode*)pNode)->colName, &type); + if (type == TCOL_TYPE_COLUMN) { + pCxt->colExists = true; + return DEAL_RES_ERROR; + } else if (type == TCOL_TYPE_TAG) { + addTagList(&pCxt->pTags, nodesCloneNode(pNode)); + } else { + pCxt->colNotFound = true; + return DEAL_RES_ERROR; + } + } else if (QUERY_NODE_FUNCTION == nodeType(pNode)) { + SFunctionNode* pFunc = (SFunctionNode*)pNode; + if (0 == strcasecmp(pFunc->functionName, "tbname")) { + addTagList(&pCxt->pTags, nodesCloneNode(pNode)); + } + } + + return DEAL_RES_CONTINUE; +} + +static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* pStmt, STableMeta* pMeta, SNodeList** ppProjection) { + SBuildTopicContext colCxt = {.colExists = false, .colNotFound = false, .pMeta = pMeta, .pTags = NULL}; + nodesWalkExprPostOrder(pStmt->pWhere, checkColumnTagsInCond, &colCxt); + if (colCxt.colNotFound) { + nodesDestroyList(colCxt.pTags); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Invalid column name"); + } else if (colCxt.colExists) { + nodesDestroyList(colCxt.pTags); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Columns are forbidden in where clause"); + } + if (NULL == colCxt.pTags) { // put one column to select +// for (int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) { + SSchema* column = &pMeta->schema[0]; + SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == col) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(col->colName, column->name); + strcpy(col->node.aliasName, col->colName); + strcpy(col->node.userAlias, col->colName); + addTagList(&colCxt.pTags, (SNode*)col); +// } + } + + *ppProjection = colCxt.pTags; + return TSDB_CODE_SUCCESS; +} + +static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt, SNode** pSelect) { + SParseContext* pParCxt = pCxt->pParseCxt; + SRequestConnInfo connInfo = {.pTrans = pParCxt->pTransporter, + .requestId = pParCxt->requestId, + .requestObjRefId = pParCxt->requestRid, + .mgmtEps = pParCxt->mgmtEpSet}; + SName name; + STableMeta* pMeta = NULL; + int32_t code = getTableMetaImpl(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta); + if (code) { + taosMemoryFree(pMeta); + return code; + } + if (TSDB_SUPER_TABLE != pMeta->tableType) { + taosMemoryFree(pMeta); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only supertable table can be used"); + } + + SNodeList* pProjection = NULL; + code = checkCollectTopicTags(pCxt, pStmt, pMeta, &pProjection); + if (TSDB_CODE_SUCCESS == code) { + SRealTableNode* realTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); + strcpy(realTable->table.dbName, pStmt->subDbName); + strcpy(realTable->table.tableName, pStmt->subSTbName); + strcpy(realTable->table.tableAlias, pStmt->subSTbName); + *pSelect = createSelectStmtImpl(true, pProjection, (SNode*)realTable); + ((SSelectStmt*)*pSelect)->pWhere = nodesCloneNode(pStmt->pWhere); + pCxt->pParseCxt->topicQuery = true; + code = translateQuery(pCxt, *pSelect); + } + + taosMemoryFree(pMeta); + return code; +} + static int32_t checkCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) { - if (NULL == pStmt->pQuery) { + if (NULL == pStmt->pQuery && NULL == pStmt->pWhere) { return TSDB_CODE_SUCCESS; } - if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) { + if (pStmt->pWhere) { + return buildQueryForTableTopic(pCxt, pStmt, &pStmt->pQuery); + } else if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; if (!pSelect->isDistinct && (NULL != pSelect->pFromTable && QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable)) && @@ -5870,9 +6211,7 @@ static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* p static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt) { SMDropTopicReq dropReq = {0}; - SName name; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName)); - tNameGetFullDbName(&name, dropReq.name); + snprintf(dropReq.name, sizeof(dropReq.name), "%d.%s", pCxt->pParseCxt->acctId, pStmt->topicName); dropReq.igNotExists = pStmt->ignoreNotExists; return buildCmdMsg(pCxt, TDMT_MND_TMQ_DROP_TOPIC, (FSerializeFunc)tSerializeSMDropTopicReq, &dropReq); @@ -5898,7 +6237,7 @@ static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pSt char* p = strchr(pStmt->config, ' '); if (NULL != p) { *p = 0; - strcpy(pStmt->value, p + 1); + tstrncpy(pStmt->value, p + 1, sizeof(pStmt->value)); } return TSDB_CODE_SUCCESS; } @@ -6207,7 +6546,7 @@ static int32_t subtableExprHasColumnOrPseudoColumn(SNode* pNode) { static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || - !pSelect->isTimeLineResult || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || + !isTimeLineQuery(pStmt->pQuery) || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || crossTableWithUdaf(pSelect) || isEventWindowQuery(pSelect) || hasJsonTypeProjection(pSelect)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); } @@ -6257,7 +6596,10 @@ typedef struct SProjColPos { } SProjColPos; static int32_t projColPosCompar(const void* l, const void* r) { - return ((SProjColPos*)l)->colId > ((SProjColPos*)r)->colId; + if (((SProjColPos*)l)->colId < ((SProjColPos*)r)->colId) { + return -1; + } + return ((SProjColPos*)l)->colId == ((SProjColPos*)r)->colId ? 0 : 1; } static void projColPosDelete(void* p) { nodesDestroyNode(((SProjColPos*)p)->pProj); } @@ -6563,6 +6905,54 @@ static int32_t translateStreamTargetTable(STranslateContext* pCxt, SCreateStream return code; } +static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta, SNode** pQuery) { + SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == col) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + strcpy(col->tableAlias, pTable); + strcpy(col->colName, pMeta->schema[0].name); + SNodeList* pParamterList = nodesMakeList(); + if (NULL == pParamterList) { + nodesDestroyNode((SNode *)col); + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = nodesListStrictAppend(pParamterList, (SNode *)col); + if (code) { + nodesDestroyNode((SNode *)col); + nodesDestroyList(pParamterList); + return code; + } + + SNode* pFunc = (SNode*)createFunction("last", pParamterList); + if (NULL == pFunc) { + nodesDestroyList(pParamterList); + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNodeList* pProjectionList = nodesMakeList(); + if (NULL == pProjectionList) { + nodesDestroyList(pParamterList); + return TSDB_CODE_OUT_OF_MEMORY; + } + code = nodesListStrictAppend(pProjectionList, pFunc); + if (code) { + nodesDestroyNode(pFunc); + nodesDestroyList(pProjectionList); + return code; + } + + code = createSimpleSelectStmtFromProjList(pDb, pTable, pProjectionList, (SSelectStmt **)pQuery); + if (code) { + nodesDestroyList(pProjectionList); + return code; + } + + return code; +} + static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) { pCxt->createStream = true; STableMeta* pMeta = NULL; @@ -6589,6 +6979,18 @@ static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt getSourceDatabase(pStmt->pQuery, pCxt->pParseCxt->acctId, pReq->sourceDB); code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); } + if (TSDB_CODE_SUCCESS == code && pStmt->pOptions->fillHistory) { + SRealTableNode* pTable = (SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable); + code = createLastTsSelectStmt(pTable->table.dbName, pTable->table.tableName, pTable->pMeta, &pStmt->pPrevQuery); +/* + if (TSDB_CODE_SUCCESS == code) { + STranslateContext cxt = {0}; + int32_t code = initTranslateContext(pCxt->pParseCxt, pCxt->pMetaCache, &cxt); + code = translateQuery(&cxt, pStmt->pPrevQuery); + destroyTranslateContext(&cxt); + } +*/ + } taosMemoryFree(pMeta); return code; } @@ -6655,13 +7057,86 @@ static int32_t translateCreateStream(STranslateContext* pCxt, SCreateStreamStmt* code = buildCreateStreamReq(pCxt, pStmt, &createReq); } if (TSDB_CODE_SUCCESS == code) { - code = buildCmdMsg(pCxt, TDMT_MND_CREATE_STREAM, (FSerializeFunc)tSerializeSCMCreateStreamReq, &createReq); + if (NULL == pStmt->pPrevQuery) { + code = buildCmdMsg(pCxt, TDMT_MND_CREATE_STREAM, (FSerializeFunc)tSerializeSCMCreateStreamReq, &createReq); + } else { + pStmt->pReq = taosMemoryMalloc(sizeof(createReq)); + if (NULL == pStmt->pReq) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + memcpy(pStmt->pReq, &createReq, sizeof(createReq)); + memset(&createReq, 0, sizeof(createReq)); + TSWAP(pCxt->pPrevRoot, pStmt->pPrevQuery); + } + } } tFreeSCMCreateStreamReq(&createReq); return code; } +static int32_t buildIntervalForCreateStream(SCreateStreamStmt* pStmt, SInterval* pInterval) { + int32_t code = TSDB_CODE_SUCCESS; + if (QUERY_NODE_SELECT_STMT != nodeType(pStmt->pQuery)) { + return code; + } + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + if (NULL == pSelect->pWindow || QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow)) { + return code; + } + + SIntervalWindowNode* pWindow = (SIntervalWindowNode*)pSelect->pWindow; + pInterval->interval = ((SValueNode*)pWindow->pInterval)->datum.i; + pInterval->intervalUnit = ((SValueNode*)pWindow->pInterval)->unit; + pInterval->offset = (NULL != pWindow->pOffset ? ((SValueNode*)pWindow->pOffset)->datum.i : 0); + pInterval->sliding = (NULL != pWindow->pSliding ? ((SValueNode*)pWindow->pSliding)->datum.i : pInterval->interval); + pInterval->slidingUnit = + (NULL != pWindow->pSliding ? ((SValueNode*)pWindow->pSliding)->unit : pInterval->intervalUnit); + pInterval->precision = ((SColumnNode*)pWindow->pCol)->node.resType.precision; + + return code; +} + +int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow) { + SCreateStreamStmt* pStmt = (SCreateStreamStmt*)pQuery->pRoot; + STranslateContext cxt = {0}; + SInterval interval = {0}; + int64_t lastTs = 0; + + int32_t code = initTranslateContext(pParseCxt, NULL, &cxt); + if (TSDB_CODE_SUCCESS == code) { + code = buildIntervalForCreateStream(pStmt, &interval); + } + if (TSDB_CODE_SUCCESS == code) { + if (pResRow && pResRow[0]) { + lastTs = *(int64_t*)pResRow[0]; + } else if (interval.interval > 0) { + lastTs = convertTimePrecision(taosGetTimestampMs(), TSDB_TIME_PRECISION_MILLI, interval.precision); + } else { + lastTs = taosGetTimestampMs(); + } + } + if (TSDB_CODE_SUCCESS == code) { + if (interval.interval > 0) { + pStmt->pReq->lastTs = taosTimeTruncate(lastTs, &interval); + } else { + pStmt->pReq->lastTs = lastTs; + } + code = buildCmdMsg(&cxt, TDMT_MND_CREATE_STREAM, (FSerializeFunc)tSerializeSCMCreateStreamReq, pStmt->pReq); + } + if (TSDB_CODE_SUCCESS == code) { + code = setQuery(&cxt, pQuery); + } + setRefreshMate(&cxt, pQuery); + destroyTranslateContext(&cxt); + + tFreeSCMCreateStreamReq(pStmt->pReq); + taosMemoryFreeClear(pStmt->pReq); + + return code; +} + + static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pStmt) { SMDropStreamReq dropReq = {0}; SName name; @@ -7260,7 +7735,7 @@ static int32_t extractShowCreateTableResultSchema(int32_t* numOfCols, SSchema** } static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pSchema) { - *numOfCols = 2; + *numOfCols = 3; *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); if (NULL == (*pSchema)) { return TSDB_CODE_OUT_OF_MEMORY; @@ -7274,6 +7749,10 @@ static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pS (*pSchema)[1].bytes = TSDB_CONFIG_VALUE_LEN; strcpy((*pSchema)[1].name, "value"); + (*pSchema)[2].type = TSDB_DATA_TYPE_BINARY; + (*pSchema)[2].bytes = TSDB_CONFIG_SCOPE_LEN; + strcpy((*pSchema)[2].name, "scope"); + return TSDB_CODE_SUCCESS; } @@ -7342,8 +7821,7 @@ static SNodeList* createProjectCols(int32_t ncols, const char* const pCols[]) { return pProjections; } -static int32_t createSimpleSelectStmt(const char* pDb, const char* pTable, int32_t numOfProjs, - const char* const pProjCol[], SSelectStmt** pStmt) { +static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt) { SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); if (NULL == pSelect) { return TSDB_CODE_OUT_OF_MEMORY; @@ -7359,27 +7837,38 @@ static int32_t createSimpleSelectStmt(const char* pDb, const char* pTable, int32 snprintf(pRealTable->table.tableName, sizeof(pRealTable->table.tableName), "%s", pTable); snprintf(pRealTable->table.tableAlias, sizeof(pRealTable->table.tableAlias), "%s", pTable); pSelect->pFromTable = (SNode*)pRealTable; + pSelect->pProjectionList = pProjectionList; + + *pStmt = pSelect; + + return TSDB_CODE_SUCCESS; +} + +static int32_t createSimpleSelectStmtFromCols(const char* pDb, const char* pTable, int32_t numOfProjs, + const char* const pProjCol[], SSelectStmt** pStmt) { + SNodeList* pProjectionList = NULL; if (numOfProjs >= 0) { - pSelect->pProjectionList = createProjectCols(numOfProjs, pProjCol); - if (NULL == pSelect->pProjectionList) { - nodesDestroyNode((SNode*)pSelect); + pProjectionList = createProjectCols(numOfProjs, pProjCol); + if (NULL == pProjectionList) { return TSDB_CODE_OUT_OF_MEMORY; } } - *pStmt = pSelect; + return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt); +} - return TSDB_CODE_SUCCESS; +static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt) { + return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt); } static int32_t createSelectStmtForShow(ENodeType showType, SSelectStmt** pStmt) { const SSysTableShowAdapter* pShow = &sysTableShowAdapter[showType - SYSTABLE_SHOW_TYPE_OFFSET]; - return createSimpleSelectStmt(pShow->pDbName, pShow->pTableName, pShow->numOfShowCols, pShow->pShowCols, pStmt); + return createSimpleSelectStmtFromCols(pShow->pDbName, pShow->pTableName, pShow->numOfShowCols, pShow->pShowCols, pStmt); } static int32_t createSelectStmtForShowTableDist(SShowTableDistributedStmt* pStmt, SSelectStmt** pOutput) { - return createSimpleSelectStmt(pStmt->dbName, pStmt->tableName, 0, NULL, pOutput); + return createSimpleSelectStmtFromCols(pStmt->dbName, pStmt->tableName, 0, NULL, pOutput); } static int32_t createOperatorNode(EOperatorType opType, const char* pColName, SNode* pRight, SNode** pOp) { @@ -7513,7 +8002,7 @@ static int32_t createShowTableTagsProjections(SNodeList** pProjections, SNodeLis static int32_t rewriteShowStableTags(STranslateContext* pCxt, SQuery* pQuery) { SShowTableTagsStmt* pShow = (SShowTableTagsStmt*)pQuery->pRoot; SSelectStmt* pSelect = NULL; - int32_t code = createSimpleSelectStmt(((SValueNode*)pShow->pDbName)->literal, ((SValueNode*)pShow->pTbName)->literal, + int32_t code = createSimpleSelectStmtFromCols(((SValueNode*)pShow->pDbName)->literal, ((SValueNode*)pShow->pTbName)->literal, -1, NULL, &pSelect); if (TSDB_CODE_SUCCESS == code) { code = createShowTableTagsProjections(&pSelect->pProjectionList, &pShow->pTags); @@ -7999,6 +8488,9 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { isJson = true; code = buildJsonTagVal(pCxt, pTagSchema, pVal, pTagArray, ppTag); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode((SNode*)pVal); + } taosArrayPush(tagName, pTagSchema->name); } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL && !pVal->isNull) { char* tmpVal = nodesGetValueFromNode(pVal); @@ -8319,13 +8811,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS SArray* pTagVals = taosArrayInit(1, sizeof(STagVal)); int32_t code = TSDB_CODE_SUCCESS; STag* pTag = NULL; - do { - code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf); - if (TSDB_CODE_SUCCESS != code) { - break; - } - } while (0); - + code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf); taosArrayDestroy(pTagVals); if (code != TSDB_CODE_SUCCESS) { return code; @@ -8841,6 +9327,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { } break; default: + pQuery->haveResultSet = false; pQuery->execMode = QUERY_EXEC_MODE_RPC; if (NULL != pCxt->pCmdMsg) { TSWAP(pQuery->pCmdMsg, pCxt->pCmdMsg); @@ -8875,6 +9362,10 @@ int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMe if (TSDB_CODE_SUCCESS == code) { code = translateQuery(&cxt, pQuery->pRoot); } + if (TSDB_CODE_SUCCESS == code && (cxt.pPrevRoot || cxt.pPostRoot)) { + pQuery->pPrevRoot = cxt.pPrevRoot; + pQuery->pPostRoot = cxt.pPostRoot; + } if (TSDB_CODE_SUCCESS == code) { code = setQuery(&cxt, pQuery); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 14da6f8aabcfba0cfdc3a71a731b1c34acdb32d5..1c292b1ec4a0377fe72efde544692e4b228243ae 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -138,7 +138,7 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY: return "Primary timestamp column cannot be dropped"; case TSDB_CODE_PAR_INVALID_MODIFY_COL: - return "Only binary/nchar column length could be modified, and the length can only be increased, not decreased"; + return "Only binary/nchar/geometry column length could be modified, and the length can only be increased, not decreased"; case TSDB_CODE_PAR_INVALID_TBNAME: return "Invalid tbname pseudo column"; case TSDB_CODE_PAR_INVALID_FUNCTION_NAME: @@ -164,12 +164,18 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "%s function is not supported in fill query"; case TSDB_CODE_PAR_INVALID_WINDOW_PC: return "_WSTART, _WEND and _WDURATION can only be used in window query"; + case TSDB_CODE_PAR_INVALID_TAGS_PC: + return "Tags can only applied to super table and child table"; case TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC: return "%s function is not supported in time window query"; case TSDB_CODE_PAR_STREAM_NOT_ALLOWED_FUNC: return "%s function is not supported in stream query"; case TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC: return "%s function is not supported in group query"; + case TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC: + return "%s function is not supported in system table query"; + case TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED: + return "%s is not supported in system table query"; case TSDB_CODE_PAR_INVALID_INTERP_CLAUSE: return "Invalid usage of RANGE clause, EVERY clause or FILL clause"; case TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN: @@ -249,6 +255,17 @@ int32_t getNumOfTags(const STableMeta* pTableMeta) { return getTableInfo(pTableM STableComInfo getTableInfo(const STableMeta* pTableMeta) { return pTableMeta->tableInfo; } +int32_t getTableTypeFromTableNode(SNode *pTable) { + if (NULL == pTable) { + return -1; + } + if (QUERY_NODE_REAL_TABLE != nodeType(pTable)) { + return -1; + } + return ((SRealTableNode *)pTable)->pMeta->tableType; +} + + STableMeta* tableMetaDup(const STableMeta* pTableMeta) { int32_t numOfFields = TABLE_TOTAL_COL_NUM(pTableMeta); if (numOfFields > TSDB_MAX_COLUMNS || numOfFields < TSDB_MIN_COLUMNS) { @@ -305,6 +322,7 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) { dst[j] = src[k]; j++; } + if (j >= dlen) j = dlen - 1; dst[j] = '\0'; return j; } @@ -415,7 +433,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi end: taosHashCleanup(keyHash); if (retCode == TSDB_CODE_SUCCESS) { - tTagNew(pTagVals, 1, true, ppTag); + retCode = tTagNew(pTagVals, 1, true, ppTag); } for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { STagVal* p = (STagVal*)taosArrayGet(pTagVals, i); @@ -498,7 +516,7 @@ int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, char* pStr) { - return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL != pTable && '\0' == pTable[0]) ? NULL : pTable, + return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL == pTable || '\0' == pTable[0]) ? "``" : pTable, type); } @@ -524,6 +542,9 @@ static void getStringFromAuthStr(const char* pStart, char* pStr, char** pNext) { strncpy(pStr, pStart, p - pStart); *pNext = ++p; } + if (*pStart == '`' && *(pStart + 1) == '`') { + *pStr = 0; + } } static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUserAuth) { @@ -532,7 +553,11 @@ static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUser pUserAuth->tbName.acctId = getIntegerFromAuthStr(p, &p); getStringFromAuthStr(p, pUserAuth->tbName.dbname, &p); getStringFromAuthStr(p, pUserAuth->tbName.tname, &p); - pUserAuth->tbName.type = TSDB_TABLE_NAME_T; + if (pUserAuth->tbName.tname[0]) { + pUserAuth->tbName.type = TSDB_TABLE_NAME_T; + } else { + pUserAuth->tbName.type = TSDB_DB_NAME_T; + } pUserAuth->type = getIntegerFromAuthStr(p, &p); } @@ -666,6 +691,22 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog return code; } + +SNode* createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable) { + SSelectStmt* select = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + if (NULL == select) { + return NULL; + } + select->isDistinct = isDistinct; + select->pProjectionList = pProjectionList; + select->pFromTable = pTable; + sprintf(select->stmtName, "%p", select); + select->timeLineResMode = select->isDistinct ? TIME_LINE_NONE : TIME_LINE_GLOBAL; + select->onlyHasKeepOrderFunc = true; + select->timeRange = TSWINDOW_INITIALIZER; + return (SNode*)select; +} + static int32_t putMetaDataToHash(const char* pKey, int32_t len, const SArray* pData, int32_t index, SHashObj** pHash) { if (NULL == *pHash) { *pHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); @@ -1101,3 +1142,18 @@ void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { taosHashCleanup(pMetaCache->pTableIndex); taosHashCleanup(pMetaCache->pTableCfg); } + +int64_t int64SafeSub(int64_t a, int64_t b) { + int64_t res = (uint64_t)a - (uint64_t)b; + + if (a >= 0 && b < 0) { + if ((uint64_t)res > (uint64_t)INT64_MAX) { + // overflow + res = INT64_MAX; + } + } else if (a < 0 && b > 0 && res >= 0) { + // underflow + res = INT64_MIN; + } + return res; +} diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 47482db7400f98c839c112d93e503b20d25f8073..10fda8741b073c2fb8a7fa80e21afbc3680b3ede 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -99,6 +99,7 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { switch (pParam->buffer_type) { case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_GEOMETRY: pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); if (NULL == pVal->datum.p) { return TSDB_CODE_OUT_OF_MEMORY; @@ -203,7 +204,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata const struct SMetaData* pMetaData, SQuery* pQuery) { SParseMetaCache metaCache = {0}; int32_t code = nodesAcquireAllocator(pCxt->allocatorId); - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && pCatalogReq) { code = putMetaDataToCache(pCatalogReq, pMetaData, &metaCache); } if (TSDB_CODE_SUCCESS == code) { @@ -220,6 +221,21 @@ int32_t qContinueParseSql(SParseContext* pCxt, struct SCatalogReq* pCatalogReq, return parseInsertSql(pCxt, &pQuery, pCatalogReq, pMetaData); } +int32_t qContinueParsePostQuery(SParseContext* pCxt, SQuery* pQuery, void** pResRow) { + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(pQuery->pRoot)) { + case QUERY_NODE_CREATE_STREAM_STMT: + code = translatePostCreateStream(pCxt, pQuery, pResRow); + break; + case QUERY_NODE_CREATE_INDEX_STMT: + code = translatePostCreateSmaIndex(pCxt, pQuery, pResRow); + default: + break; + } + + return code; +} + void qDestroyParseContext(SParseContext* pCxt) { if (NULL == pCxt) { return; diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index bb0b0400353e763e6cf1d0e8da55328fe62ddefd..a912fb4e71c8aeb0b78298915540fbd52029a597 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -25,7 +25,6 @@ #include #include /************ Begin %include sections from the grammar ************************/ -#line 11 "sql.y" #include #include @@ -42,7 +41,6 @@ #include "parAst.h" #define YYSTACKDEPTH 0 -#line 46 "sql.c" /**************** End of %include directives **********************************/ /* These constants specify the various numeric values for terminal symbols ** in a format understandable to "makeheaders". This section is blank unless @@ -106,27 +104,27 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 482 +#define YYNOCODE 487 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - SNodeList* yy72; - SNode* yy164; - EJoinType yy196; - bool yy441; - EFillMode yy446; - SToken yy497; - ENullOrder yy517; - EOrder yy550; - int32_t yy560; - int8_t yy563; - int64_t yy693; - SDataType yy700; - SAlterOption yy761; - EOperatorType yy796; - STokenPair yy953; + EJoinType yy140; + SDataType yy310; + STokenPair yy347; + EOperatorType yy354; + SAlterOption yy365; + SToken yy371; + ENullOrder yy399; + int32_t yy416; + SNode* yy452; + int8_t yy475; + bool yy667; + EOrder yy690; + int64_t yy729; + SNodeList* yy812; + EFillMode yy844; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -142,18 +140,18 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 787 -#define YYNRULE 591 -#define YYNRULE_WITH_ACTION 591 -#define YYNTOKEN 335 -#define YY_MAX_SHIFT 786 -#define YY_MIN_SHIFTREDUCE 1162 -#define YY_MAX_SHIFTREDUCE 1752 -#define YY_ERROR_ACTION 1753 -#define YY_ACCEPT_ACTION 1754 -#define YY_NO_ACTION 1755 -#define YY_MIN_REDUCE 1756 -#define YY_MAX_REDUCE 2346 +#define YYNSTATE 800 +#define YYNRULE 600 +#define YYNRULE_WITH_ACTION 600 +#define YYNTOKEN 338 +#define YY_MAX_SHIFT 799 +#define YY_MIN_SHIFTREDUCE 1180 +#define YY_MAX_SHIFTREDUCE 1779 +#define YY_ERROR_ACTION 1780 +#define YY_ACCEPT_ACTION 1781 +#define YY_NO_ACTION 1782 +#define YY_MIN_REDUCE 1783 +#define YY_MAX_REDUCE 2382 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -220,842 +218,821 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2980) +#define YY_ACTTAB_COUNT (2858) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 2158, 402, 680, 1930, 38, 303, 182, 2091, 643, 162, - /* 10 */ 694, 2317, 48, 46, 1680, 1757, 380, 1932, 1969, 216, - /* 20 */ 399, 133, 1529, 529, 1980, 1799, 642, 186, 564, 655, - /* 30 */ 140, 2318, 644, 1610, 1829, 1527, 123, 2176, 107, 122, - /* 40 */ 121, 120, 119, 118, 117, 116, 115, 114, 1555, 2126, - /* 50 */ 1907, 696, 41, 40, 251, 141, 47, 45, 44, 43, - /* 60 */ 42, 1605, 531, 1922, 1557, 41, 40, 19, 528, 47, - /* 70 */ 45, 44, 43, 42, 1535, 1905, 1554, 47, 45, 44, - /* 80 */ 43, 42, 142, 2157, 526, 2217, 2193, 527, 1792, 339, - /* 90 */ 2159, 700, 2161, 2162, 695, 693, 690, 681, 2211, 783, - /* 100 */ 167, 30, 15, 760, 759, 758, 757, 411, 1871, 756, - /* 110 */ 755, 144, 750, 749, 748, 747, 746, 745, 744, 157, - /* 120 */ 740, 739, 738, 410, 409, 735, 734, 733, 175, 174, - /* 130 */ 657, 184, 2254, 2255, 1350, 138, 2259, 123, 1612, 1613, - /* 140 */ 122, 121, 120, 119, 118, 117, 116, 115, 114, 1341, - /* 150 */ 722, 721, 720, 1345, 719, 1347, 1348, 718, 715, 679, - /* 160 */ 1356, 712, 1358, 1359, 709, 706, 1384, 1385, 1585, 1595, - /* 170 */ 680, 1930, 41, 40, 1611, 1614, 47, 45, 44, 43, - /* 180 */ 42, 730, 155, 154, 727, 726, 725, 152, 1530, 133, - /* 190 */ 1528, 62, 666, 408, 407, 568, 569, 41, 40, 567, - /* 200 */ 661, 47, 45, 44, 43, 42, 730, 155, 154, 727, - /* 210 */ 726, 725, 152, 665, 262, 534, 1536, 1554, 527, 1792, - /* 220 */ 1533, 1534, 543, 1584, 1587, 1588, 1589, 1590, 1591, 1592, - /* 230 */ 1593, 1594, 692, 688, 1603, 1604, 1606, 1607, 1608, 1609, - /* 240 */ 2, 48, 46, 541, 2158, 2041, 349, 618, 1552, 399, - /* 250 */ 2317, 1529, 14, 13, 658, 482, 358, 2048, 496, 638, - /* 260 */ 679, 495, 1610, 52, 1527, 2323, 186, 582, 581, 580, - /* 270 */ 2318, 644, 2046, 667, 572, 137, 576, 465, 2136, 497, - /* 280 */ 575, 2176, 234, 467, 1195, 574, 579, 374, 373, 1756, - /* 290 */ 1605, 573, 1921, 2126, 1982, 696, 19, 1982, 172, 1677, - /* 300 */ 1639, 379, 2140, 1535, 389, 560, 556, 552, 548, 1980, - /* 310 */ 231, 179, 1980, 132, 131, 130, 129, 128, 127, 126, - /* 320 */ 125, 124, 168, 1197, 1768, 1200, 1201, 2157, 783, 367, - /* 330 */ 2193, 15, 2032, 110, 2159, 700, 2161, 2162, 695, 2142, - /* 340 */ 690, 455, 51, 2261, 1749, 183, 189, 2246, 1982, 690, - /* 350 */ 88, 395, 2242, 229, 1653, 394, 1640, 404, 445, 732, - /* 360 */ 1975, 1977, 444, 1980, 188, 1439, 1440, 1612, 1613, 2258, - /* 370 */ 493, 1539, 2272, 487, 486, 485, 484, 481, 480, 479, - /* 380 */ 478, 477, 473, 472, 471, 470, 348, 462, 461, 460, - /* 390 */ 218, 457, 456, 365, 529, 679, 1799, 1585, 1595, 2012, - /* 400 */ 1555, 655, 140, 1611, 1614, 618, 41, 40, 2317, 2322, - /* 410 */ 47, 45, 44, 43, 42, 1982, 371, 1530, 189, 1528, - /* 420 */ 228, 222, 364, 2323, 186, 227, 1556, 539, 2318, 644, - /* 430 */ 1980, 1754, 2048, 37, 397, 1634, 1635, 1636, 1637, 1638, - /* 440 */ 1642, 1643, 1644, 1645, 392, 220, 1748, 2045, 667, 1533, - /* 450 */ 1534, 637, 1584, 1587, 1588, 1589, 1590, 1591, 1592, 1593, - /* 460 */ 1594, 692, 688, 1603, 1604, 1606, 1607, 1608, 1609, 2, - /* 470 */ 12, 48, 46, 87, 87, 393, 666, 51, 2176, 399, - /* 480 */ 451, 1529, 1779, 165, 372, 666, 370, 369, 2158, 566, - /* 490 */ 368, 1932, 1610, 643, 1527, 1708, 2317, 189, 697, 1925, - /* 500 */ 1926, 1719, 414, 185, 2254, 2255, 413, 138, 2259, 1742, - /* 510 */ 568, 642, 186, 2158, 567, 1982, 2318, 644, 1456, 1457, - /* 520 */ 1605, 680, 1930, 697, 345, 2176, 19, 664, 489, 2041, - /* 530 */ 1981, 2126, 636, 1535, 655, 140, 675, 2126, 2041, 696, - /* 540 */ 191, 633, 630, 629, 1706, 1707, 1709, 1710, 1711, 618, - /* 550 */ 2176, 1778, 2317, 1676, 1455, 1458, 469, 66, 783, 523, - /* 560 */ 101, 15, 2126, 1553, 696, 468, 521, 2323, 186, 517, - /* 570 */ 513, 2157, 2318, 644, 2193, 1832, 284, 169, 2159, 700, - /* 580 */ 2161, 2162, 695, 12, 690, 1923, 41, 40, 207, 206, - /* 590 */ 47, 45, 44, 43, 42, 596, 2157, 1612, 1613, 2193, - /* 600 */ 2126, 605, 110, 2159, 700, 2161, 2162, 695, 594, 690, - /* 610 */ 592, 488, 143, 1684, 150, 2217, 2246, 619, 2283, 1554, - /* 620 */ 395, 2242, 179, 1554, 294, 295, 1529, 1585, 1595, 293, - /* 630 */ 639, 634, 627, 1611, 1614, 2261, 187, 2254, 2255, 1527, - /* 640 */ 138, 2259, 366, 2031, 582, 581, 580, 1530, 165, 1528, - /* 650 */ 249, 572, 137, 576, 248, 62, 1933, 575, 1777, 723, - /* 660 */ 1620, 2257, 574, 579, 374, 373, 1554, 2158, 573, 730, - /* 670 */ 155, 154, 727, 726, 725, 152, 260, 658, 1535, 1533, - /* 680 */ 1534, 1696, 1584, 1587, 1588, 1589, 1590, 1591, 1592, 1593, - /* 690 */ 1594, 692, 688, 1603, 1604, 1606, 1607, 1608, 1609, 2, - /* 700 */ 48, 46, 1615, 783, 2176, 62, 1535, 2126, 399, 1294, - /* 710 */ 1529, 1218, 62, 1217, 90, 2158, 2126, 353, 696, 91, - /* 720 */ 378, 1610, 598, 1527, 34, 697, 233, 1801, 680, 1930, - /* 730 */ 41, 40, 680, 1930, 47, 45, 44, 43, 42, 9, - /* 740 */ 655, 140, 1776, 2261, 1219, 285, 742, 57, 1296, 1605, - /* 750 */ 2157, 449, 2176, 2193, 680, 1930, 110, 2159, 700, 2161, - /* 760 */ 2162, 695, 1535, 690, 2126, 2119, 696, 1586, 183, 2256, - /* 770 */ 2246, 36, 1775, 450, 395, 2242, 1538, 41, 40, 408, - /* 780 */ 407, 47, 45, 44, 43, 42, 62, 783, 93, 1543, - /* 790 */ 49, 2126, 1530, 1906, 1528, 2273, 153, 443, 2157, 442, - /* 800 */ 1610, 2193, 1536, 2158, 110, 2159, 700, 2161, 2162, 695, - /* 810 */ 189, 690, 618, 697, 1586, 2317, 2337, 570, 2246, 680, - /* 820 */ 1930, 2126, 395, 2242, 1533, 1534, 1612, 1613, 1605, 441, - /* 830 */ 2323, 186, 680, 1930, 1919, 2318, 644, 402, 459, 1292, - /* 840 */ 2176, 1535, 282, 2254, 654, 165, 134, 653, 12, 2317, - /* 850 */ 10, 474, 2126, 1932, 696, 56, 1585, 1595, 1774, 732, - /* 860 */ 189, 1773, 1611, 1614, 642, 186, 686, 189, 1982, 2318, - /* 870 */ 644, 680, 1930, 578, 577, 403, 1530, 2136, 1528, 2322, - /* 880 */ 1915, 2322, 2317, 1980, 2317, 1772, 2157, 680, 1930, 2193, - /* 890 */ 475, 2145, 170, 2159, 700, 2161, 2162, 695, 2321, 690, - /* 900 */ 2321, 2140, 2318, 2320, 2318, 2319, 542, 2126, 1533, 1534, - /* 910 */ 2126, 1584, 1587, 1588, 1589, 1590, 1591, 1592, 1593, 1594, - /* 920 */ 692, 688, 1603, 1604, 1606, 1607, 1608, 1609, 2, 48, - /* 930 */ 46, 1541, 680, 1930, 2126, 2158, 1771, 399, 2142, 1529, - /* 940 */ 1305, 189, 2321, 645, 2338, 697, 405, 2280, 690, 1917, - /* 950 */ 1610, 1927, 1527, 1304, 165, 1544, 1673, 1539, 1770, 41, - /* 960 */ 40, 1767, 1932, 47, 45, 44, 43, 42, 317, 41, - /* 970 */ 40, 1959, 2176, 47, 45, 44, 43, 42, 1605, 1554, - /* 980 */ 680, 1930, 1976, 1977, 2126, 2126, 696, 1547, 1549, 680, - /* 990 */ 1930, 1535, 680, 1930, 680, 1930, 1309, 2109, 2120, 252, - /* 1000 */ 688, 1603, 1604, 1606, 1607, 1608, 1609, 2126, 614, 1308, - /* 1010 */ 2126, 659, 2027, 663, 1556, 1913, 783, 498, 2157, 49, - /* 1020 */ 1218, 2193, 1217, 2158, 110, 2159, 700, 2161, 2162, 695, - /* 1030 */ 250, 690, 587, 697, 724, 2293, 2337, 1973, 2246, 680, - /* 1040 */ 1930, 728, 395, 2242, 1973, 618, 1641, 597, 2317, 1766, - /* 1050 */ 203, 680, 1930, 1219, 433, 1612, 1613, 199, 298, 1765, - /* 1060 */ 2176, 247, 1764, 2323, 186, 680, 1930, 1557, 2318, 644, - /* 1070 */ 677, 1763, 2126, 500, 696, 680, 1930, 590, 680, 1930, - /* 1080 */ 1557, 435, 431, 584, 678, 1585, 1595, 754, 752, 246, - /* 1090 */ 84, 1611, 1614, 83, 304, 1200, 1201, 406, 2126, 1500, - /* 1100 */ 1501, 44, 43, 42, 1203, 1530, 2157, 1528, 2126, 2193, - /* 1110 */ 1553, 2126, 110, 2159, 700, 2161, 2162, 695, 1762, 690, - /* 1120 */ 2126, 164, 603, 35, 2337, 1761, 2246, 1586, 1760, 70, - /* 1130 */ 395, 2242, 69, 1646, 1759, 2266, 1673, 1533, 1534, 2027, - /* 1140 */ 1584, 1587, 1588, 1589, 1590, 1591, 1592, 1593, 1594, 692, - /* 1150 */ 688, 1603, 1604, 1606, 1607, 1608, 1609, 2, 48, 46, - /* 1160 */ 2027, 2158, 2136, 682, 284, 2218, 399, 2126, 1529, 618, - /* 1170 */ 2112, 697, 2317, 625, 2126, 743, 2144, 2126, 1892, 1610, - /* 1180 */ 684, 1527, 2218, 2126, 201, 74, 2140, 2323, 186, 452, - /* 1190 */ 617, 729, 2318, 644, 1973, 54, 1908, 3, 2176, 146, - /* 1200 */ 239, 135, 453, 237, 1816, 205, 241, 1605, 153, 240, - /* 1210 */ 2126, 428, 696, 571, 1808, 243, 646, 200, 242, 421, - /* 1220 */ 1535, 1806, 245, 2142, 396, 244, 583, 600, 153, 599, - /* 1230 */ 261, 50, 50, 690, 82, 1290, 585, 266, 153, 647, - /* 1240 */ 148, 1751, 1752, 588, 2157, 783, 50, 2193, 15, 687, - /* 1250 */ 110, 2159, 700, 2161, 2162, 695, 650, 690, 2147, 291, - /* 1260 */ 71, 1802, 2337, 1537, 2246, 106, 151, 1495, 395, 2242, - /* 1270 */ 153, 14, 13, 64, 50, 103, 50, 736, 737, 704, - /* 1280 */ 151, 153, 136, 151, 1612, 1613, 1934, 1498, 1251, 256, - /* 1290 */ 1705, 1704, 691, 259, 1872, 1769, 268, 662, 2286, 1270, - /* 1300 */ 1268, 279, 631, 166, 232, 1453, 273, 1870, 323, 1869, - /* 1310 */ 2177, 778, 55, 2149, 1585, 1595, 412, 2036, 296, 672, - /* 1320 */ 1611, 1614, 321, 73, 1793, 300, 72, 1252, 1798, 1335, - /* 1330 */ 2276, 1970, 1647, 1596, 1530, 316, 1528, 346, 1362, 1366, - /* 1340 */ 1373, 1371, 156, 656, 278, 281, 1, 214, 508, 506, - /* 1350 */ 503, 5, 415, 420, 362, 2158, 1560, 437, 436, 194, - /* 1360 */ 193, 196, 439, 1476, 204, 697, 1533, 1534, 311, 1584, - /* 1370 */ 1587, 1588, 1589, 1590, 1591, 1592, 1593, 1594, 692, 688, - /* 1380 */ 1603, 1604, 1606, 1607, 1608, 1609, 2, 62, 1557, 454, - /* 1390 */ 2037, 491, 2176, 458, 463, 1552, 476, 483, 2029, 490, - /* 1400 */ 492, 501, 1631, 502, 2126, 499, 696, 504, 209, 211, - /* 1410 */ 208, 505, 507, 509, 1558, 524, 535, 4, 1540, 532, - /* 1420 */ 525, 533, 219, 1555, 2158, 109, 536, 1559, 221, 537, - /* 1430 */ 1561, 538, 540, 224, 697, 544, 2311, 226, 2157, 85, - /* 1440 */ 86, 2193, 1221, 561, 110, 2159, 700, 2161, 2162, 695, - /* 1450 */ 230, 690, 563, 562, 352, 112, 2221, 2100, 2246, 602, - /* 1460 */ 604, 2176, 395, 2242, 565, 81, 80, 448, 1920, 89, - /* 1470 */ 198, 236, 1916, 2126, 648, 696, 149, 238, 312, 158, - /* 1480 */ 159, 1918, 1914, 440, 438, 160, 2158, 253, 161, 608, - /* 1490 */ 607, 257, 1483, 651, 347, 609, 697, 429, 2265, 615, - /* 1500 */ 427, 423, 419, 416, 441, 2097, 2096, 2157, 2277, 612, - /* 1510 */ 2193, 255, 2158, 110, 2159, 700, 2161, 2162, 695, 2287, - /* 1520 */ 690, 632, 697, 2176, 622, 2337, 2292, 2246, 670, 613, - /* 1530 */ 264, 395, 2242, 2291, 628, 2126, 384, 696, 267, 2268, - /* 1540 */ 8, 635, 189, 641, 623, 385, 621, 620, 277, 2176, - /* 1550 */ 2340, 649, 652, 139, 1673, 1556, 2262, 660, 388, 286, - /* 1560 */ 274, 2126, 1562, 696, 313, 2042, 96, 668, 669, 2157, - /* 1570 */ 2056, 314, 2193, 2055, 276, 110, 2159, 700, 2161, 2162, - /* 1580 */ 695, 673, 690, 275, 2054, 272, 2158, 2337, 173, 2246, - /* 1590 */ 391, 674, 61, 395, 2242, 2157, 697, 98, 2193, 2316, - /* 1600 */ 100, 110, 2159, 700, 2161, 2162, 695, 280, 690, 102, - /* 1610 */ 2227, 315, 702, 2219, 1974, 2246, 1893, 1931, 779, 395, - /* 1620 */ 2242, 318, 780, 2176, 782, 327, 53, 341, 307, 331, - /* 1630 */ 320, 2118, 322, 2117, 342, 2126, 2116, 696, 78, 2113, - /* 1640 */ 417, 418, 1520, 1521, 354, 355, 192, 2158, 2111, 422, - /* 1650 */ 424, 425, 426, 2110, 363, 2108, 430, 697, 434, 432, - /* 1660 */ 2106, 1511, 2087, 195, 2086, 197, 1479, 79, 1478, 2157, - /* 1670 */ 2107, 2068, 2193, 2158, 2067, 110, 2159, 700, 2161, 2162, - /* 1680 */ 695, 2066, 690, 697, 2176, 446, 447, 683, 2065, 2246, - /* 1690 */ 2064, 1430, 2020, 395, 2242, 2019, 2126, 2017, 696, 145, - /* 1700 */ 2016, 2015, 2018, 2014, 2013, 2011, 2010, 2009, 202, 2158, - /* 1710 */ 2176, 464, 2008, 466, 2022, 2007, 2006, 2005, 2004, 697, - /* 1720 */ 147, 1992, 2126, 2003, 696, 2002, 2001, 2000, 1999, 1998, - /* 1730 */ 2157, 1997, 1996, 2193, 1995, 1994, 111, 2159, 700, 2161, - /* 1740 */ 2162, 695, 1993, 690, 1991, 1990, 2176, 2021, 1989, 1988, - /* 1750 */ 2246, 1987, 1432, 1986, 2245, 2242, 2157, 1985, 2126, 2193, - /* 1760 */ 696, 494, 111, 2159, 700, 2161, 2162, 695, 1984, 690, - /* 1770 */ 1983, 1835, 1306, 2158, 1310, 1302, 2246, 1834, 1833, 1831, - /* 1780 */ 685, 2242, 225, 697, 350, 351, 1828, 511, 1827, 1820, - /* 1790 */ 510, 1810, 698, 514, 210, 2193, 518, 2158, 111, 2159, - /* 1800 */ 700, 2161, 2162, 695, 212, 690, 512, 697, 515, 519, - /* 1810 */ 2176, 516, 2246, 520, 1788, 213, 357, 2242, 215, 522, - /* 1820 */ 76, 1202, 2126, 1787, 696, 2085, 77, 2146, 2075, 180, - /* 1830 */ 2063, 2062, 2040, 217, 2176, 223, 1909, 181, 1830, 530, - /* 1840 */ 1826, 1244, 547, 545, 546, 1824, 2126, 549, 696, 550, - /* 1850 */ 551, 1822, 553, 554, 555, 1819, 2157, 557, 558, 2193, - /* 1860 */ 559, 2158, 111, 2159, 700, 2161, 2162, 695, 1805, 690, - /* 1870 */ 1804, 697, 1784, 1911, 63, 235, 2246, 1378, 1377, 1910, - /* 1880 */ 2157, 2243, 606, 2193, 1293, 1291, 169, 2159, 700, 2161, - /* 1890 */ 2162, 695, 751, 690, 1289, 1288, 1287, 1286, 2176, 1285, - /* 1900 */ 786, 1282, 753, 382, 1281, 1280, 1279, 1817, 375, 1809, - /* 1910 */ 2126, 376, 696, 1807, 310, 377, 586, 589, 1783, 2158, - /* 1920 */ 591, 1782, 593, 1781, 595, 113, 1505, 2284, 29, 697, - /* 1930 */ 178, 1507, 1504, 2084, 58, 2158, 1509, 776, 772, 768, - /* 1940 */ 764, 67, 308, 1485, 2157, 697, 1487, 2193, 2074, 610, - /* 1950 */ 340, 2159, 700, 2161, 2162, 695, 2176, 690, 2061, 2059, - /* 1960 */ 1489, 383, 6, 2322, 611, 31, 163, 624, 2126, 258, - /* 1970 */ 696, 381, 2176, 20, 17, 616, 7, 1721, 263, 21, - /* 1980 */ 22, 265, 108, 1703, 2126, 301, 696, 1695, 626, 271, - /* 1990 */ 171, 270, 65, 2147, 269, 33, 2158, 32, 24, 1736, - /* 2000 */ 92, 1735, 2157, 1741, 1742, 2193, 697, 23, 340, 2159, - /* 2010 */ 700, 2161, 2162, 695, 386, 690, 1740, 676, 2157, 1739, - /* 2020 */ 387, 2193, 2158, 18, 333, 2159, 700, 2161, 2162, 695, - /* 2030 */ 283, 690, 697, 2176, 1670, 1669, 60, 176, 59, 2060, - /* 2040 */ 2058, 2057, 2039, 94, 95, 2126, 289, 696, 671, 2038, - /* 2050 */ 97, 25, 288, 290, 302, 1701, 103, 287, 2158, 2176, - /* 2060 */ 26, 11, 177, 292, 390, 297, 13, 640, 694, 68, - /* 2070 */ 299, 2126, 99, 696, 1622, 1621, 1545, 254, 2196, 2157, - /* 2080 */ 1600, 1598, 2193, 1597, 689, 170, 2159, 700, 2161, 2162, - /* 2090 */ 695, 190, 690, 39, 2158, 2176, 16, 27, 1577, 1569, - /* 2100 */ 28, 701, 703, 1363, 697, 2157, 401, 2126, 2193, 696, - /* 2110 */ 705, 340, 2159, 700, 2161, 2162, 695, 1360, 690, 707, - /* 2120 */ 2158, 708, 710, 1632, 1357, 711, 699, 1351, 713, 714, - /* 2130 */ 697, 2176, 1349, 716, 717, 104, 398, 2339, 305, 105, - /* 2140 */ 1372, 2157, 1355, 2126, 2193, 696, 1354, 339, 2159, 700, - /* 2150 */ 2161, 2162, 695, 75, 690, 1353, 2212, 2176, 1352, 1368, - /* 2160 */ 1242, 1274, 400, 731, 1273, 1272, 1271, 1269, 1267, 2126, - /* 2170 */ 1266, 696, 306, 1300, 1265, 741, 1263, 2157, 1262, 1261, - /* 2180 */ 2193, 1260, 1259, 340, 2159, 700, 2161, 2162, 695, 1258, - /* 2190 */ 690, 1257, 1297, 1295, 1248, 2158, 1254, 1253, 1250, 1249, - /* 2200 */ 1247, 1825, 761, 2157, 762, 697, 2193, 763, 1823, 340, - /* 2210 */ 2159, 700, 2161, 2162, 695, 2158, 690, 765, 766, 767, - /* 2220 */ 1821, 1818, 769, 770, 771, 697, 773, 774, 775, 1803, - /* 2230 */ 777, 1192, 2176, 1780, 309, 781, 1755, 1531, 319, 784, - /* 2240 */ 1755, 1755, 785, 1755, 2126, 1755, 696, 1755, 1755, 1755, - /* 2250 */ 1755, 1755, 2176, 1755, 1755, 1755, 1755, 1755, 1755, 1755, - /* 2260 */ 1755, 1755, 1755, 1755, 2126, 1755, 696, 1755, 1755, 1755, - /* 2270 */ 1755, 1755, 1755, 1755, 1755, 1755, 1755, 1755, 601, 1755, - /* 2280 */ 1755, 2193, 1755, 2158, 335, 2159, 700, 2161, 2162, 695, - /* 2290 */ 1755, 690, 1755, 697, 1755, 1755, 1755, 1755, 2157, 1755, - /* 2300 */ 2158, 2193, 1755, 1755, 324, 2159, 700, 2161, 2162, 695, - /* 2310 */ 697, 690, 1755, 1755, 1755, 1755, 1755, 1755, 1755, 1755, - /* 2320 */ 2176, 1755, 1755, 1755, 1755, 1755, 1755, 1755, 1755, 1755, - /* 2330 */ 1755, 1755, 2126, 1755, 696, 1755, 1755, 2176, 1755, 1755, - /* 2340 */ 1755, 1755, 1755, 1755, 1755, 1755, 1755, 1755, 2158, 2126, - /* 2350 */ 1755, 696, 1755, 1755, 1755, 1755, 1755, 1755, 697, 1755, - /* 2360 */ 1755, 1755, 1755, 1755, 1755, 1755, 2157, 1755, 2158, 2193, - /* 2370 */ 1755, 1755, 325, 2159, 700, 2161, 2162, 695, 697, 690, - /* 2380 */ 1755, 1755, 1755, 2157, 1755, 2176, 2193, 1755, 1755, 326, - /* 2390 */ 2159, 700, 2161, 2162, 695, 1755, 690, 2126, 2158, 696, - /* 2400 */ 1755, 1755, 1755, 1755, 1755, 2176, 1755, 1755, 697, 1755, - /* 2410 */ 1755, 1755, 1755, 1755, 1755, 1755, 2158, 2126, 1755, 696, - /* 2420 */ 1755, 1755, 1755, 1755, 1755, 1755, 697, 1755, 1755, 1755, - /* 2430 */ 1755, 2157, 1755, 1755, 2193, 2176, 1755, 332, 2159, 700, - /* 2440 */ 2161, 2162, 695, 1755, 690, 1755, 2158, 2126, 1755, 696, - /* 2450 */ 1755, 2157, 1755, 2176, 2193, 1755, 697, 336, 2159, 700, - /* 2460 */ 2161, 2162, 695, 1755, 690, 2126, 1755, 696, 1755, 1755, - /* 2470 */ 1755, 1755, 1755, 1755, 1755, 1755, 2158, 1755, 1755, 1755, - /* 2480 */ 1755, 2157, 1755, 2176, 2193, 1755, 697, 328, 2159, 700, - /* 2490 */ 2161, 2162, 695, 1755, 690, 2126, 1755, 696, 1755, 2157, - /* 2500 */ 1755, 1755, 2193, 1755, 1755, 337, 2159, 700, 2161, 2162, - /* 2510 */ 695, 1755, 690, 2176, 1755, 1755, 1755, 1755, 1755, 1755, - /* 2520 */ 1755, 1755, 1755, 1755, 2158, 2126, 1755, 696, 1755, 2157, - /* 2530 */ 1755, 1755, 2193, 1755, 697, 329, 2159, 700, 2161, 2162, - /* 2540 */ 695, 1755, 690, 1755, 2158, 1755, 1755, 1755, 1755, 1755, - /* 2550 */ 1755, 1755, 1755, 1755, 697, 1755, 1755, 1755, 1755, 2157, - /* 2560 */ 1755, 2176, 2193, 1755, 1755, 338, 2159, 700, 2161, 2162, - /* 2570 */ 695, 1755, 690, 2126, 1755, 696, 1755, 1755, 1755, 1755, - /* 2580 */ 1755, 2176, 1755, 1755, 1755, 1755, 1755, 1755, 1755, 1755, - /* 2590 */ 1755, 1755, 1755, 2126, 1755, 696, 1755, 1755, 1755, 1755, - /* 2600 */ 1755, 1755, 1755, 1755, 1755, 1755, 1755, 2157, 1755, 1755, - /* 2610 */ 2193, 1755, 1755, 330, 2159, 700, 2161, 2162, 695, 1755, - /* 2620 */ 690, 1755, 2158, 1755, 1755, 1755, 1755, 2157, 1755, 1755, - /* 2630 */ 2193, 1755, 697, 343, 2159, 700, 2161, 2162, 695, 2158, - /* 2640 */ 690, 1755, 1755, 1755, 1755, 1755, 1755, 1755, 1755, 697, - /* 2650 */ 1755, 1755, 1755, 1755, 1755, 1755, 1755, 1755, 1755, 2176, - /* 2660 */ 1755, 1755, 1755, 1755, 1755, 1755, 1755, 1755, 1755, 1755, - /* 2670 */ 1755, 2126, 1755, 696, 1755, 1755, 2176, 1755, 1755, 1755, - /* 2680 */ 1755, 1755, 1755, 1755, 1755, 1755, 1755, 2158, 2126, 1755, - /* 2690 */ 696, 1755, 1755, 1755, 1755, 1755, 1755, 697, 1755, 1755, - /* 2700 */ 1755, 1755, 1755, 1755, 1755, 2157, 1755, 2158, 2193, 1755, - /* 2710 */ 1755, 344, 2159, 700, 2161, 2162, 695, 697, 690, 1755, - /* 2720 */ 1755, 1755, 2157, 1755, 2176, 2193, 1755, 1755, 2170, 2159, - /* 2730 */ 700, 2161, 2162, 695, 1755, 690, 2126, 2158, 696, 1755, - /* 2740 */ 1755, 1755, 1755, 1755, 2176, 1755, 1755, 697, 1755, 1755, - /* 2750 */ 1755, 1755, 1755, 1755, 1755, 2158, 2126, 1755, 696, 1755, - /* 2760 */ 1755, 1755, 1755, 1755, 1755, 697, 1755, 1755, 1755, 1755, - /* 2770 */ 2157, 1755, 1755, 2193, 2176, 1755, 2169, 2159, 700, 2161, - /* 2780 */ 2162, 695, 1755, 690, 1755, 2158, 2126, 1755, 696, 1755, - /* 2790 */ 2157, 1755, 2176, 2193, 1755, 697, 2168, 2159, 700, 2161, - /* 2800 */ 2162, 695, 1755, 690, 2126, 1755, 696, 1755, 1755, 1755, - /* 2810 */ 1755, 1755, 1755, 1755, 1755, 2158, 1755, 1755, 1755, 1755, - /* 2820 */ 2157, 1755, 2176, 2193, 1755, 697, 359, 2159, 700, 2161, - /* 2830 */ 2162, 695, 1755, 690, 2126, 1755, 696, 1755, 2157, 1755, - /* 2840 */ 1755, 2193, 1755, 1755, 360, 2159, 700, 2161, 2162, 695, - /* 2850 */ 1755, 690, 2176, 1755, 1755, 1755, 1755, 1755, 1755, 1755, - /* 2860 */ 1755, 1755, 1755, 2158, 2126, 1755, 696, 1755, 2157, 1755, - /* 2870 */ 1755, 2193, 1755, 697, 356, 2159, 700, 2161, 2162, 695, - /* 2880 */ 1755, 690, 1755, 2158, 1755, 1755, 1755, 1755, 1755, 1755, - /* 2890 */ 1755, 1755, 1755, 697, 1755, 1755, 1755, 1755, 2157, 1755, - /* 2900 */ 2176, 2193, 1755, 1755, 361, 2159, 700, 2161, 2162, 695, - /* 2910 */ 1755, 690, 2126, 1755, 696, 1755, 1755, 1755, 1755, 1755, - /* 2920 */ 2176, 1755, 1755, 1755, 1755, 1755, 1755, 1755, 1755, 1755, - /* 2930 */ 1755, 1755, 2126, 1755, 696, 1755, 1755, 1755, 1755, 1755, - /* 2940 */ 1755, 1755, 1755, 1755, 1755, 1755, 698, 1755, 1755, 2193, - /* 2950 */ 1755, 1755, 335, 2159, 700, 2161, 2162, 695, 1755, 690, - /* 2960 */ 1755, 1755, 1755, 1755, 1755, 1755, 2157, 1755, 1755, 2193, - /* 2970 */ 1755, 1755, 334, 2159, 700, 2161, 2162, 695, 1755, 690, + /* 0 */ 2126, 2193, 2171, 2082, 221, 690, 1960, 2154, 537, 1949, + /* 10 */ 1826, 670, 48, 46, 1706, 394, 2179, 1213, 2079, 677, + /* 20 */ 401, 2358, 1555, 41, 40, 135, 2175, 47, 45, 44, + /* 30 */ 43, 42, 574, 1636, 453, 1553, 2171, 539, 1582, 2211, + /* 40 */ 41, 40, 1781, 536, 47, 45, 44, 43, 42, 254, + /* 50 */ 1951, 2161, 1945, 706, 630, 531, 1215, 2353, 1218, 1219, + /* 60 */ 2175, 181, 1631, 529, 2177, 398, 525, 521, 19, 1238, + /* 70 */ 66, 1237, 2359, 188, 700, 1561, 30, 2354, 656, 348, + /* 80 */ 689, 369, 2065, 361, 140, 690, 1960, 2192, 1580, 2228, + /* 90 */ 667, 144, 112, 2194, 710, 2196, 2197, 705, 2177, 700, + /* 100 */ 796, 168, 1239, 15, 185, 135, 2281, 103, 700, 1901, + /* 110 */ 397, 2277, 579, 497, 2082, 416, 48, 46, 690, 1960, + /* 120 */ 415, 689, 1769, 190, 401, 264, 1555, 1665, 1371, 2080, + /* 130 */ 677, 2307, 1953, 1580, 38, 306, 1746, 1636, 193, 1553, + /* 140 */ 1638, 1639, 1806, 1362, 735, 734, 733, 1366, 732, 1368, + /* 150 */ 1369, 731, 728, 1805, 1377, 725, 1379, 1380, 722, 719, + /* 160 */ 716, 184, 630, 51, 655, 2353, 1631, 2353, 94, 62, + /* 170 */ 1611, 1621, 19, 2000, 212, 211, 1637, 1640, 675, 1561, + /* 180 */ 2359, 188, 654, 188, 1666, 2354, 656, 2354, 656, 2296, + /* 190 */ 288, 1556, 2161, 1554, 286, 2289, 666, 496, 136, 665, + /* 200 */ 169, 2353, 1795, 2161, 796, 41, 40, 15, 2193, 47, + /* 210 */ 45, 44, 43, 42, 62, 2293, 654, 188, 707, 1315, + /* 220 */ 435, 2354, 656, 1559, 1560, 1783, 1610, 1613, 1614, 1615, + /* 230 */ 1616, 1617, 1618, 1619, 1620, 702, 698, 1629, 1630, 1632, + /* 240 */ 1633, 1634, 1635, 2, 1638, 1639, 2211, 437, 433, 134, + /* 250 */ 133, 132, 131, 130, 129, 128, 127, 126, 2161, 1317, + /* 260 */ 706, 1784, 37, 399, 1660, 1661, 1662, 1663, 1664, 1668, + /* 270 */ 1669, 1670, 1671, 534, 1611, 1621, 535, 1819, 551, 1580, + /* 280 */ 1637, 1640, 125, 1464, 1465, 124, 123, 122, 121, 120, + /* 290 */ 119, 118, 117, 116, 2192, 1556, 2228, 1554, 645, 112, + /* 300 */ 2194, 710, 2196, 2197, 705, 650, 700, 2044, 395, 147, + /* 310 */ 1580, 151, 2252, 2281, 1581, 2193, 166, 397, 2277, 1238, + /* 320 */ 191, 1237, 667, 144, 1962, 670, 191, 1559, 1560, 1703, + /* 330 */ 1610, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 1620, 702, + /* 340 */ 698, 1629, 1630, 1632, 1633, 1634, 1635, 2, 12, 48, + /* 350 */ 46, 745, 1239, 2211, 2013, 410, 409, 401, 2182, 1555, + /* 360 */ 2358, 367, 62, 2353, 181, 2161, 1582, 706, 615, 2011, + /* 370 */ 1636, 191, 1553, 592, 591, 590, 690, 1960, 1562, 2357, + /* 380 */ 582, 141, 586, 2354, 2356, 2066, 585, 649, 651, 646, + /* 390 */ 639, 584, 589, 377, 376, 146, 56, 583, 2252, 1631, + /* 400 */ 252, 2192, 629, 2228, 251, 19, 112, 2194, 710, 2196, + /* 410 */ 2197, 705, 1561, 700, 2184, 2211, 297, 298, 185, 542, + /* 420 */ 2281, 296, 535, 1819, 397, 2277, 187, 2289, 2290, 2193, + /* 430 */ 142, 2294, 1221, 655, 1406, 1407, 2353, 796, 1579, 707, + /* 440 */ 15, 1828, 41, 40, 1272, 2308, 47, 45, 44, 43, + /* 450 */ 42, 654, 188, 48, 46, 1641, 2354, 656, 223, 2193, + /* 460 */ 1612, 401, 537, 1555, 1826, 93, 475, 2211, 356, 707, + /* 470 */ 648, 381, 166, 608, 1636, 474, 1553, 1638, 1639, 2161, + /* 480 */ 1963, 706, 2358, 125, 1273, 2353, 124, 123, 122, 121, + /* 490 */ 120, 119, 118, 117, 116, 2013, 62, 2211, 1804, 667, + /* 500 */ 144, 2357, 382, 1631, 12, 2354, 2355, 1611, 1621, 2161, + /* 510 */ 2011, 706, 109, 1637, 1640, 2192, 1561, 2228, 288, 191, + /* 520 */ 112, 2194, 710, 2196, 2197, 705, 60, 700, 1556, 145, + /* 530 */ 1554, 578, 2373, 627, 2281, 577, 1565, 1952, 397, 2277, + /* 540 */ 1710, 796, 690, 1960, 49, 2192, 1580, 2228, 2161, 2193, + /* 550 */ 170, 2194, 710, 2196, 2197, 705, 12, 700, 10, 707, + /* 560 */ 1559, 1560, 451, 1610, 1613, 1614, 1615, 1616, 1617, 1618, + /* 570 */ 1619, 1620, 702, 698, 1629, 1630, 1632, 1633, 1634, 1635, + /* 580 */ 2, 1638, 1639, 445, 1326, 444, 1702, 2211, 41, 40, + /* 590 */ 631, 2318, 47, 45, 44, 43, 42, 1325, 2296, 2161, + /* 600 */ 1583, 706, 669, 186, 2289, 2290, 165, 142, 2294, 1561, + /* 610 */ 1803, 1611, 1621, 690, 1960, 443, 406, 1637, 1640, 2006, + /* 620 */ 2008, 41, 40, 404, 2292, 47, 45, 44, 43, 42, + /* 630 */ 2296, 163, 1556, 452, 1554, 2192, 676, 2228, 383, 1962, + /* 640 */ 112, 2194, 710, 2196, 2197, 705, 2011, 700, 253, 689, + /* 650 */ 447, 2193, 2256, 191, 2281, 446, 2291, 2155, 397, 2277, + /* 660 */ 2161, 707, 506, 2315, 1559, 1560, 154, 1610, 1613, 1614, + /* 670 */ 1615, 1616, 1617, 1618, 1619, 1620, 702, 698, 1629, 1630, + /* 680 */ 1632, 1633, 1634, 1635, 2, 48, 46, 1937, 549, 2211, + /* 690 */ 2075, 457, 2061, 401, 755, 1555, 1612, 630, 667, 144, + /* 700 */ 2353, 2161, 2013, 706, 630, 1936, 1636, 2353, 1553, 391, + /* 710 */ 47, 45, 44, 43, 42, 2359, 188, 2011, 1526, 1527, + /* 720 */ 2354, 656, 2359, 188, 463, 2061, 55, 2354, 656, 14, + /* 730 */ 13, 1734, 51, 690, 1960, 1631, 1947, 2192, 202, 2228, + /* 740 */ 597, 676, 112, 2194, 710, 2196, 2197, 705, 1561, 700, + /* 750 */ 690, 1960, 266, 465, 2373, 607, 2281, 1862, 41, 40, + /* 760 */ 397, 2277, 47, 45, 44, 43, 42, 2193, 1802, 250, + /* 770 */ 480, 205, 745, 796, 690, 1960, 49, 704, 642, 641, + /* 780 */ 1732, 1733, 1735, 1736, 1737, 600, 490, 2061, 2357, 48, + /* 790 */ 46, 1935, 594, 674, 481, 2075, 1776, 401, 249, 1555, + /* 800 */ 404, 1581, 189, 2289, 2290, 2211, 142, 2294, 166, 692, + /* 810 */ 1636, 2253, 1553, 1638, 1639, 1846, 1962, 2161, 2161, 706, + /* 820 */ 743, 156, 155, 740, 739, 738, 153, 592, 591, 590, + /* 830 */ 2007, 2008, 1679, 210, 582, 141, 586, 593, 70, 1631, + /* 840 */ 585, 69, 52, 1611, 1621, 584, 589, 377, 376, 1637, + /* 850 */ 1640, 583, 1561, 2192, 1745, 2228, 1241, 1242, 342, 2194, + /* 860 */ 710, 2196, 2197, 705, 1556, 700, 1554, 2247, 41, 40, + /* 870 */ 1801, 1943, 47, 45, 44, 43, 42, 796, 570, 569, + /* 880 */ 15, 2193, 743, 156, 155, 740, 739, 738, 153, 1800, + /* 890 */ 613, 707, 207, 2328, 1799, 1964, 1559, 1560, 1775, 1610, + /* 900 */ 1613, 1614, 1615, 1616, 1617, 1618, 1619, 1620, 702, 698, + /* 910 */ 1629, 1630, 1632, 1633, 1634, 1635, 2, 1638, 1639, 2211, + /* 920 */ 2161, 1481, 1482, 606, 743, 156, 155, 740, 739, 738, + /* 930 */ 153, 2161, 86, 706, 2013, 85, 604, 630, 602, 2161, + /* 940 */ 2353, 396, 690, 1960, 2161, 690, 1960, 1611, 1621, 2011, + /* 950 */ 736, 690, 1960, 1637, 1640, 2359, 188, 1480, 1483, 259, + /* 960 */ 2354, 656, 550, 737, 1722, 1957, 2004, 2192, 1556, 2228, + /* 970 */ 1554, 255, 112, 2194, 710, 2196, 2197, 705, 9, 700, + /* 980 */ 588, 587, 34, 2147, 2373, 1330, 2281, 676, 41, 40, + /* 990 */ 397, 2277, 47, 45, 44, 43, 42, 1798, 1329, 191, + /* 1000 */ 1559, 1560, 1859, 1610, 1613, 1614, 1615, 1616, 1617, 1618, + /* 1010 */ 1619, 1620, 702, 698, 1629, 1630, 1632, 1633, 1634, 1635, + /* 1020 */ 2, 1646, 352, 167, 1578, 690, 1960, 1580, 327, 690, + /* 1030 */ 1960, 488, 423, 90, 504, 572, 571, 503, 2171, 685, + /* 1040 */ 407, 2075, 324, 73, 2144, 263, 72, 2161, 166, 673, + /* 1050 */ 371, 2013, 2180, 471, 671, 505, 1962, 349, 405, 1955, + /* 1060 */ 473, 701, 2175, 508, 1797, 658, 2011, 1794, 219, 516, + /* 1070 */ 514, 511, 773, 772, 771, 770, 413, 1938, 769, 768, + /* 1080 */ 148, 763, 762, 761, 760, 759, 758, 757, 158, 753, + /* 1090 */ 752, 751, 412, 411, 748, 747, 746, 176, 175, 1583, + /* 1100 */ 2177, 630, 690, 1960, 2353, 1583, 370, 237, 62, 374, + /* 1110 */ 700, 44, 43, 42, 2161, 690, 1960, 2161, 459, 2359, + /* 1120 */ 188, 149, 301, 173, 2354, 656, 690, 1960, 262, 690, + /* 1130 */ 1960, 568, 564, 560, 556, 687, 236, 1793, 662, 694, + /* 1140 */ 1667, 2253, 767, 765, 1792, 1791, 688, 111, 501, 307, + /* 1150 */ 1790, 495, 494, 493, 492, 487, 486, 485, 484, 483, + /* 1160 */ 479, 478, 477, 476, 351, 468, 467, 466, 659, 461, + /* 1170 */ 460, 368, 690, 1960, 2301, 1699, 91, 1612, 375, 234, + /* 1180 */ 373, 372, 1837, 576, 90, 1789, 1788, 2161, 81, 80, + /* 1190 */ 450, 2193, 408, 200, 2161, 2161, 1787, 2013, 1699, 741, + /* 1200 */ 2161, 707, 2004, 637, 595, 578, 442, 440, 1786, 577, + /* 1210 */ 1956, 742, 2012, 320, 2004, 1902, 1990, 350, 35, 54, + /* 1220 */ 431, 3, 2193, 429, 425, 421, 418, 443, 1672, 2211, + /* 1230 */ 756, 454, 707, 1922, 2346, 2161, 2161, 1218, 1219, 154, + /* 1240 */ 83, 2161, 137, 706, 455, 74, 2161, 233, 227, 580, + /* 1250 */ 242, 203, 2193, 240, 232, 547, 244, 265, 2161, 243, + /* 1260 */ 2211, 430, 707, 246, 2300, 191, 245, 248, 154, 581, + /* 1270 */ 247, 1313, 2161, 225, 706, 1835, 50, 2192, 610, 2228, + /* 1280 */ 609, 50, 112, 2194, 710, 2196, 2197, 705, 697, 700, + /* 1290 */ 2211, 1311, 1778, 1779, 2373, 84, 2281, 598, 1796, 1521, + /* 1300 */ 397, 2277, 2161, 643, 706, 1564, 270, 154, 2192, 108, + /* 1310 */ 2228, 2321, 283, 112, 2194, 710, 2196, 2197, 705, 105, + /* 1320 */ 700, 50, 294, 1563, 1555, 2373, 71, 2281, 1524, 152, + /* 1330 */ 154, 397, 2277, 14, 13, 64, 1731, 1553, 2192, 50, + /* 1340 */ 2228, 1730, 2193, 112, 2194, 710, 2196, 2197, 705, 50, + /* 1350 */ 700, 1829, 707, 410, 409, 2373, 277, 2281, 36, 714, + /* 1360 */ 139, 397, 2277, 1569, 41, 40, 272, 672, 47, 45, + /* 1370 */ 44, 43, 42, 2193, 1636, 1900, 1562, 1561, 663, 152, + /* 1380 */ 2211, 1478, 299, 707, 154, 749, 682, 1899, 2212, 303, + /* 1390 */ 1356, 750, 2161, 138, 706, 1673, 384, 152, 2070, 1622, + /* 1400 */ 414, 791, 796, 1631, 1820, 1825, 660, 1291, 2001, 319, + /* 1410 */ 2193, 2211, 2311, 1289, 668, 285, 1561, 282, 422, 1384, + /* 1420 */ 707, 1, 417, 2161, 5, 706, 365, 1586, 2192, 438, + /* 1430 */ 2228, 1502, 196, 112, 2194, 710, 2196, 2197, 705, 1388, + /* 1440 */ 700, 696, 439, 1657, 1395, 2254, 441, 2281, 2211, 195, + /* 1450 */ 198, 397, 2277, 1393, 314, 1579, 456, 157, 209, 2192, + /* 1460 */ 2161, 2228, 706, 1567, 112, 2194, 710, 2196, 2197, 705, + /* 1470 */ 1583, 700, 462, 458, 469, 499, 693, 2071, 2281, 464, + /* 1480 */ 1578, 1566, 397, 2277, 482, 491, 489, 2063, 498, 500, + /* 1490 */ 509, 510, 507, 1556, 213, 1554, 2192, 214, 2228, 2193, + /* 1500 */ 512, 113, 2194, 710, 2196, 2197, 705, 513, 700, 707, + /* 1510 */ 1584, 216, 532, 515, 517, 2281, 4, 533, 540, 2280, + /* 1520 */ 2277, 541, 543, 1581, 224, 1559, 1560, 226, 1585, 544, + /* 1530 */ 545, 1587, 1570, 546, 1565, 229, 548, 2211, 231, 88, + /* 1540 */ 89, 552, 235, 573, 355, 575, 1950, 114, 2193, 2161, + /* 1550 */ 239, 706, 1946, 612, 614, 92, 150, 618, 707, 315, + /* 1560 */ 256, 619, 617, 2135, 1573, 1575, 241, 159, 160, 1948, + /* 1570 */ 258, 260, 1944, 161, 2193, 162, 2132, 698, 1629, 1630, + /* 1580 */ 1632, 1633, 1634, 1635, 704, 2192, 2211, 2228, 625, 1509, + /* 1590 */ 113, 2194, 710, 2196, 2197, 705, 2131, 700, 2161, 644, + /* 1600 */ 706, 622, 634, 8, 2281, 2312, 2327, 680, 695, 2277, + /* 1610 */ 640, 2322, 2211, 387, 624, 2326, 647, 623, 268, 271, + /* 1620 */ 2303, 653, 276, 635, 2161, 2193, 706, 632, 633, 281, + /* 1630 */ 2376, 388, 1699, 661, 708, 707, 2228, 664, 2193, 113, + /* 1640 */ 2194, 710, 2196, 2197, 705, 143, 700, 1582, 707, 289, + /* 1650 */ 178, 1588, 278, 2281, 2076, 316, 98, 360, 2277, 2297, + /* 1660 */ 2192, 317, 2228, 2211, 678, 342, 2194, 710, 2196, 2197, + /* 1670 */ 705, 703, 700, 691, 2246, 2161, 2211, 706, 679, 683, + /* 1680 */ 2090, 279, 174, 684, 2089, 280, 100, 2193, 2161, 318, + /* 1690 */ 706, 2088, 393, 102, 61, 2262, 104, 707, 2352, 1961, + /* 1700 */ 712, 2005, 1923, 792, 2193, 284, 793, 321, 795, 2153, + /* 1710 */ 357, 2192, 310, 2228, 707, 358, 171, 2194, 710, 2196, + /* 1720 */ 2197, 705, 53, 700, 2192, 2211, 2228, 325, 323, 113, + /* 1730 */ 2194, 710, 2196, 2197, 705, 345, 700, 2161, 2152, 706, + /* 1740 */ 330, 344, 2211, 2281, 334, 2151, 78, 385, 2278, 2148, + /* 1750 */ 419, 420, 1546, 2193, 2161, 1547, 706, 194, 424, 2146, + /* 1760 */ 426, 427, 428, 707, 2145, 366, 2143, 657, 2374, 432, + /* 1770 */ 2142, 2141, 434, 2192, 436, 2228, 1537, 2193, 170, 2194, + /* 1780 */ 710, 2196, 2197, 705, 2122, 700, 197, 707, 2121, 199, + /* 1790 */ 2192, 2211, 2228, 1505, 79, 343, 2194, 710, 2196, 2197, + /* 1800 */ 705, 1504, 700, 2161, 2193, 706, 2103, 2102, 2101, 448, + /* 1810 */ 449, 2100, 2099, 2054, 707, 2211, 1455, 2053, 2050, 2319, + /* 1820 */ 386, 201, 2049, 82, 2193, 2048, 2047, 2161, 2052, 706, + /* 1830 */ 204, 2051, 2046, 2045, 707, 2043, 2042, 2041, 206, 2192, + /* 1840 */ 470, 2228, 2211, 2040, 336, 2194, 710, 2196, 2197, 705, + /* 1850 */ 472, 700, 2056, 2039, 2161, 2038, 706, 2037, 2036, 2035, + /* 1860 */ 2034, 2033, 2211, 2192, 2032, 2228, 2031, 392, 343, 2194, + /* 1870 */ 710, 2196, 2197, 705, 2161, 700, 706, 208, 2024, 2023, + /* 1880 */ 87, 2022, 2021, 2055, 2020, 2019, 215, 652, 2030, 2029, + /* 1890 */ 2192, 2193, 2228, 2028, 2027, 171, 2194, 710, 2196, 2197, + /* 1900 */ 705, 707, 700, 2026, 2025, 2018, 2017, 2016, 1457, 2015, + /* 1910 */ 2192, 502, 2228, 2014, 1327, 343, 2194, 710, 2196, 2197, + /* 1920 */ 705, 2193, 700, 353, 354, 1865, 1323, 1864, 1863, 2211, + /* 1930 */ 1331, 707, 217, 218, 400, 1861, 1858, 520, 1857, 519, + /* 1940 */ 524, 2161, 1850, 706, 523, 518, 522, 2375, 527, 526, + /* 1950 */ 1839, 528, 530, 1815, 1220, 76, 1814, 220, 2120, 2211, + /* 1960 */ 2110, 77, 182, 222, 402, 2098, 2181, 183, 538, 228, + /* 1970 */ 2097, 2161, 230, 706, 553, 554, 555, 2192, 2074, 2228, + /* 1980 */ 2193, 1939, 343, 2194, 710, 2196, 2197, 705, 616, 700, + /* 1990 */ 707, 1860, 1856, 1265, 1854, 558, 557, 1852, 559, 561, + /* 2000 */ 562, 563, 1849, 565, 566, 567, 799, 2192, 1834, 2228, + /* 2010 */ 1832, 1833, 343, 2194, 710, 2196, 2197, 705, 2211, 700, + /* 2020 */ 313, 1831, 1811, 1941, 1940, 1400, 1399, 764, 1314, 766, + /* 2030 */ 2161, 1312, 706, 1310, 1309, 1308, 180, 1847, 1307, 1301, + /* 2040 */ 1306, 63, 238, 1838, 789, 785, 781, 777, 1303, 311, + /* 2050 */ 1302, 1300, 378, 2193, 379, 1836, 380, 596, 1810, 1809, + /* 2060 */ 599, 601, 603, 707, 1808, 2119, 611, 605, 2228, 115, + /* 2070 */ 1531, 338, 2194, 710, 2196, 2197, 705, 1533, 700, 1530, + /* 2080 */ 1535, 1511, 29, 67, 2193, 1515, 2109, 2096, 1513, 110, + /* 2090 */ 164, 2211, 304, 620, 707, 2095, 2358, 20, 17, 1748, + /* 2100 */ 6, 21, 65, 2161, 31, 706, 57, 261, 7, 626, + /* 2110 */ 275, 638, 267, 621, 22, 1490, 1489, 274, 269, 2193, + /* 2120 */ 636, 172, 2211, 628, 1729, 686, 2182, 33, 24, 707, + /* 2130 */ 58, 273, 32, 23, 2161, 1721, 706, 1768, 18, 2192, + /* 2140 */ 1769, 2228, 95, 1763, 328, 2194, 710, 2196, 2197, 705, + /* 2150 */ 1762, 700, 389, 1767, 1766, 390, 2193, 2211, 287, 177, + /* 2160 */ 291, 2094, 2073, 292, 97, 1696, 707, 290, 59, 2161, + /* 2170 */ 2192, 706, 2228, 1695, 2072, 326, 2194, 710, 2196, 2197, + /* 2180 */ 705, 2193, 700, 96, 25, 295, 257, 99, 105, 293, + /* 2190 */ 305, 707, 1727, 300, 2211, 68, 26, 101, 1648, 11, + /* 2200 */ 13, 1647, 1571, 179, 2231, 2192, 2161, 2228, 706, 1658, + /* 2210 */ 329, 2194, 710, 2196, 2197, 705, 681, 700, 2193, 2211, + /* 2220 */ 302, 1603, 192, 711, 713, 1626, 1624, 403, 707, 699, + /* 2230 */ 39, 2161, 1623, 706, 16, 27, 717, 1595, 28, 720, + /* 2240 */ 1385, 715, 2192, 2193, 2228, 1382, 1381, 335, 2194, 710, + /* 2250 */ 2196, 2197, 705, 707, 700, 1378, 2211, 718, 721, 723, + /* 2260 */ 724, 726, 1372, 709, 1370, 729, 727, 2192, 2161, 2228, + /* 2270 */ 706, 730, 339, 2194, 710, 2196, 2197, 705, 1376, 700, + /* 2280 */ 106, 2211, 308, 1394, 1375, 1390, 107, 75, 1263, 1374, + /* 2290 */ 1373, 744, 1295, 2161, 1294, 706, 1293, 1292, 309, 1290, + /* 2300 */ 1288, 1287, 1286, 1321, 2192, 754, 2228, 1284, 1283, 331, + /* 2310 */ 2194, 710, 2196, 2197, 705, 2193, 700, 1282, 1281, 1280, + /* 2320 */ 1279, 1278, 1318, 1316, 1275, 707, 1274, 1271, 1270, 2192, + /* 2330 */ 2193, 2228, 1269, 1268, 340, 2194, 710, 2196, 2197, 705, + /* 2340 */ 707, 700, 1855, 774, 775, 776, 1853, 778, 2193, 779, + /* 2350 */ 1851, 782, 780, 2211, 783, 1848, 784, 786, 707, 788, + /* 2360 */ 1830, 790, 1210, 787, 1807, 2161, 312, 706, 2211, 794, + /* 2370 */ 1782, 1557, 798, 322, 797, 1782, 1782, 1782, 1782, 1782, + /* 2380 */ 2161, 1782, 706, 1782, 1782, 1782, 2211, 1782, 1782, 1782, + /* 2390 */ 1782, 1782, 1782, 1782, 1782, 2193, 1782, 1782, 2161, 1782, + /* 2400 */ 706, 2192, 1782, 2228, 1782, 707, 332, 2194, 710, 2196, + /* 2410 */ 2197, 705, 1782, 700, 1782, 1782, 2192, 2193, 2228, 1782, + /* 2420 */ 1782, 341, 2194, 710, 2196, 2197, 705, 707, 700, 1782, + /* 2430 */ 1782, 1782, 1782, 2211, 2192, 1782, 2228, 1782, 1782, 333, + /* 2440 */ 2194, 710, 2196, 2197, 705, 2161, 700, 706, 1782, 1782, + /* 2450 */ 1782, 1782, 1782, 1782, 1782, 2211, 1782, 1782, 1782, 1782, + /* 2460 */ 1782, 1782, 1782, 1782, 2193, 1782, 1782, 2161, 1782, 706, + /* 2470 */ 1782, 1782, 1782, 1782, 707, 1782, 1782, 1782, 1782, 1782, + /* 2480 */ 1782, 2192, 1782, 2228, 1782, 1782, 346, 2194, 710, 2196, + /* 2490 */ 2197, 705, 1782, 700, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2500 */ 1782, 1782, 2211, 2192, 1782, 2228, 1782, 1782, 347, 2194, + /* 2510 */ 710, 2196, 2197, 705, 2161, 700, 706, 1782, 1782, 1782, + /* 2520 */ 1782, 1782, 1782, 2193, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2530 */ 1782, 1782, 1782, 707, 1782, 1782, 2193, 1782, 1782, 1782, + /* 2540 */ 1782, 1782, 1782, 1782, 1782, 1782, 707, 1782, 1782, 1782, + /* 2550 */ 2192, 1782, 2228, 2193, 1782, 2205, 2194, 710, 2196, 2197, + /* 2560 */ 705, 2211, 700, 707, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2570 */ 1782, 1782, 1782, 2161, 2211, 706, 1782, 1782, 1782, 1782, + /* 2580 */ 1782, 1782, 1782, 1782, 1782, 2193, 2161, 1782, 706, 1782, + /* 2590 */ 1782, 2211, 1782, 1782, 1782, 707, 1782, 1782, 1782, 1782, + /* 2600 */ 1782, 1782, 1782, 2161, 1782, 706, 1782, 1782, 1782, 2192, + /* 2610 */ 1782, 2228, 1782, 1782, 2204, 2194, 710, 2196, 2197, 705, + /* 2620 */ 1782, 700, 2192, 2211, 2228, 1782, 1782, 2203, 2194, 710, + /* 2630 */ 2196, 2197, 705, 1782, 700, 2161, 1782, 706, 1782, 2192, + /* 2640 */ 1782, 2228, 1782, 1782, 362, 2194, 710, 2196, 2197, 705, + /* 2650 */ 1782, 700, 2193, 1782, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2660 */ 1782, 1782, 707, 1782, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2670 */ 1782, 2192, 2193, 2228, 1782, 1782, 363, 2194, 710, 2196, + /* 2680 */ 2197, 705, 707, 700, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2690 */ 2211, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2700 */ 1782, 1782, 2161, 1782, 706, 1782, 1782, 1782, 1782, 1782, + /* 2710 */ 2211, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2720 */ 1782, 1782, 2161, 1782, 706, 1782, 1782, 1782, 1782, 1782, + /* 2730 */ 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 2192, 1782, + /* 2740 */ 2228, 1782, 2193, 359, 2194, 710, 2196, 2197, 705, 1782, + /* 2750 */ 700, 1782, 707, 1782, 1782, 1782, 1782, 1782, 2192, 2193, + /* 2760 */ 2228, 1782, 1782, 364, 2194, 710, 2196, 2197, 705, 707, + /* 2770 */ 700, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2780 */ 2211, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2790 */ 1782, 1782, 2161, 1782, 706, 1782, 1782, 2211, 1782, 1782, + /* 2800 */ 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 2161, + /* 2810 */ 1782, 706, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2820 */ 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 708, 1782, + /* 2830 */ 2228, 1782, 1782, 338, 2194, 710, 2196, 2197, 705, 1782, + /* 2840 */ 700, 1782, 1782, 1782, 1782, 2192, 1782, 2228, 1782, 1782, + /* 2850 */ 337, 2194, 710, 2196, 2197, 705, 1782, 700, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 338, 367, 347, 348, 442, 443, 374, 371, 453, 375, - /* 10 */ 348, 456, 12, 13, 14, 0, 382, 383, 386, 343, - /* 20 */ 20, 366, 22, 347, 390, 349, 471, 472, 373, 347, - /* 30 */ 348, 476, 477, 33, 0, 35, 21, 375, 353, 24, - /* 40 */ 25, 26, 27, 28, 29, 30, 31, 32, 20, 387, - /* 50 */ 0, 389, 8, 9, 418, 370, 12, 13, 14, 15, - /* 60 */ 16, 61, 14, 378, 20, 8, 9, 67, 20, 12, - /* 70 */ 13, 14, 15, 16, 74, 0, 20, 12, 13, 14, - /* 80 */ 15, 16, 437, 421, 342, 440, 424, 345, 346, 427, - /* 90 */ 428, 429, 430, 431, 432, 433, 434, 435, 436, 99, - /* 100 */ 356, 44, 102, 69, 70, 71, 72, 73, 364, 75, - /* 110 */ 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, - /* 120 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 130 */ 448, 449, 450, 451, 99, 453, 454, 21, 138, 139, - /* 140 */ 24, 25, 26, 27, 28, 29, 30, 31, 32, 114, - /* 150 */ 115, 116, 117, 118, 119, 120, 121, 122, 123, 20, - /* 160 */ 125, 126, 127, 128, 129, 130, 138, 139, 168, 169, - /* 170 */ 347, 348, 8, 9, 174, 175, 12, 13, 14, 15, - /* 180 */ 16, 131, 132, 133, 134, 135, 136, 137, 188, 366, - /* 190 */ 190, 102, 347, 12, 13, 131, 373, 8, 9, 135, - /* 200 */ 406, 12, 13, 14, 15, 16, 131, 132, 133, 134, - /* 210 */ 135, 136, 137, 20, 170, 342, 35, 20, 345, 346, - /* 220 */ 220, 221, 66, 223, 224, 225, 226, 227, 228, 229, + /* 0 */ 375, 341, 367, 393, 346, 350, 351, 411, 350, 380, + /* 10 */ 352, 351, 12, 13, 14, 405, 381, 4, 408, 409, + /* 20 */ 20, 3, 22, 8, 9, 370, 391, 12, 13, 14, + /* 30 */ 15, 16, 377, 33, 350, 35, 367, 14, 20, 379, + /* 40 */ 8, 9, 338, 20, 12, 13, 14, 15, 16, 424, + /* 50 */ 381, 391, 380, 393, 458, 49, 43, 461, 45, 46, + /* 60 */ 391, 379, 62, 57, 429, 430, 60, 61, 68, 20, + /* 70 */ 4, 22, 476, 477, 439, 75, 44, 481, 482, 395, + /* 80 */ 20, 399, 400, 68, 35, 350, 351, 427, 20, 429, + /* 90 */ 350, 351, 432, 433, 434, 435, 436, 437, 429, 439, + /* 100 */ 100, 360, 53, 103, 444, 370, 446, 357, 439, 368, + /* 110 */ 450, 451, 377, 84, 393, 411, 12, 13, 350, 351, + /* 120 */ 416, 20, 104, 463, 20, 62, 22, 112, 100, 408, + /* 130 */ 409, 471, 382, 20, 447, 448, 104, 33, 370, 35, + /* 140 */ 140, 141, 341, 115, 116, 117, 118, 119, 120, 121, + /* 150 */ 122, 123, 124, 341, 126, 127, 128, 129, 130, 131, + /* 160 */ 132, 378, 458, 103, 458, 461, 62, 461, 105, 103, + /* 170 */ 170, 171, 68, 390, 145, 146, 176, 177, 20, 75, + /* 180 */ 476, 477, 476, 477, 169, 481, 482, 481, 482, 431, + /* 190 */ 172, 191, 391, 193, 454, 455, 456, 168, 458, 459, + /* 200 */ 340, 461, 342, 391, 100, 8, 9, 103, 341, 12, + /* 210 */ 13, 14, 15, 16, 103, 457, 476, 477, 351, 35, + /* 220 */ 186, 481, 482, 223, 224, 0, 226, 227, 228, 229, /* 230 */ 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, - /* 240 */ 240, 12, 13, 398, 338, 400, 18, 453, 20, 20, - /* 250 */ 456, 22, 1, 2, 348, 27, 67, 389, 30, 20, - /* 260 */ 20, 33, 33, 102, 35, 471, 472, 69, 70, 71, - /* 270 */ 476, 477, 404, 405, 76, 77, 78, 49, 363, 51, - /* 280 */ 82, 375, 33, 55, 4, 87, 88, 89, 90, 0, - /* 290 */ 61, 93, 377, 387, 375, 389, 67, 375, 49, 4, - /* 300 */ 111, 382, 387, 74, 382, 56, 57, 58, 59, 390, - /* 310 */ 61, 375, 390, 24, 25, 26, 27, 28, 29, 30, - /* 320 */ 31, 32, 337, 43, 339, 45, 46, 421, 99, 101, - /* 330 */ 424, 102, 396, 427, 428, 429, 430, 431, 432, 424, - /* 340 */ 434, 113, 102, 426, 180, 439, 257, 441, 375, 434, - /* 350 */ 101, 445, 446, 104, 103, 382, 167, 385, 406, 66, - /* 360 */ 388, 389, 410, 390, 458, 168, 169, 138, 139, 452, - /* 370 */ 142, 190, 466, 145, 146, 147, 148, 149, 150, 151, - /* 380 */ 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, - /* 390 */ 343, 163, 164, 165, 347, 20, 349, 168, 169, 0, - /* 400 */ 20, 347, 348, 174, 175, 453, 8, 9, 456, 3, - /* 410 */ 12, 13, 14, 15, 16, 375, 37, 188, 257, 190, - /* 420 */ 171, 172, 382, 471, 472, 176, 20, 178, 476, 477, - /* 430 */ 390, 335, 389, 244, 245, 246, 247, 248, 249, 250, - /* 440 */ 251, 252, 253, 254, 401, 196, 282, 404, 405, 220, - /* 450 */ 221, 348, 223, 224, 225, 226, 227, 228, 229, 230, - /* 460 */ 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, - /* 470 */ 241, 12, 13, 355, 355, 367, 347, 102, 375, 20, - /* 480 */ 347, 22, 338, 375, 105, 347, 107, 108, 338, 110, - /* 490 */ 372, 383, 33, 453, 35, 220, 456, 257, 348, 381, - /* 500 */ 381, 103, 406, 449, 450, 451, 410, 453, 454, 103, - /* 510 */ 131, 471, 472, 338, 135, 375, 476, 477, 138, 139, - /* 520 */ 61, 347, 348, 348, 391, 375, 67, 398, 83, 400, - /* 530 */ 390, 387, 429, 74, 347, 348, 398, 387, 400, 389, - /* 540 */ 366, 173, 267, 268, 269, 270, 271, 272, 273, 453, - /* 550 */ 375, 338, 456, 258, 174, 175, 157, 4, 99, 49, - /* 560 */ 353, 102, 387, 20, 389, 166, 56, 471, 472, 59, - /* 570 */ 60, 421, 476, 477, 424, 0, 170, 427, 428, 429, - /* 580 */ 430, 431, 432, 241, 434, 378, 8, 9, 143, 144, - /* 590 */ 12, 13, 14, 15, 16, 21, 421, 138, 139, 424, - /* 600 */ 387, 113, 427, 428, 429, 430, 431, 432, 34, 434, - /* 610 */ 36, 166, 437, 14, 439, 440, 441, 467, 468, 20, - /* 620 */ 445, 446, 375, 20, 132, 133, 22, 168, 169, 137, - /* 630 */ 262, 263, 264, 174, 175, 426, 449, 450, 451, 35, - /* 640 */ 453, 454, 395, 396, 69, 70, 71, 188, 375, 190, - /* 650 */ 133, 76, 77, 78, 137, 102, 383, 82, 338, 113, - /* 660 */ 14, 452, 87, 88, 89, 90, 20, 338, 93, 131, - /* 670 */ 132, 133, 134, 135, 136, 137, 61, 348, 74, 220, - /* 680 */ 221, 103, 223, 224, 225, 226, 227, 228, 229, 230, - /* 690 */ 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, - /* 700 */ 12, 13, 14, 99, 375, 102, 74, 387, 20, 35, - /* 710 */ 22, 20, 102, 22, 197, 338, 387, 200, 389, 104, - /* 720 */ 203, 33, 205, 35, 2, 348, 35, 350, 347, 348, - /* 730 */ 8, 9, 347, 348, 12, 13, 14, 15, 16, 39, - /* 740 */ 347, 348, 338, 426, 53, 61, 74, 366, 74, 61, - /* 750 */ 421, 366, 375, 424, 347, 348, 427, 428, 429, 430, - /* 760 */ 431, 432, 74, 434, 387, 406, 389, 168, 439, 452, - /* 770 */ 441, 2, 338, 366, 445, 446, 35, 8, 9, 12, - /* 780 */ 13, 12, 13, 14, 15, 16, 102, 99, 104, 22, - /* 790 */ 102, 387, 188, 0, 190, 466, 44, 187, 421, 189, - /* 800 */ 33, 424, 35, 338, 427, 428, 429, 430, 431, 432, - /* 810 */ 257, 434, 453, 348, 168, 456, 439, 13, 441, 347, - /* 820 */ 348, 387, 445, 446, 220, 221, 138, 139, 61, 219, - /* 830 */ 471, 472, 347, 348, 376, 476, 477, 367, 366, 35, - /* 840 */ 375, 74, 449, 450, 451, 375, 453, 454, 241, 456, - /* 850 */ 243, 366, 387, 383, 389, 103, 168, 169, 338, 66, - /* 860 */ 257, 338, 174, 175, 471, 472, 99, 257, 375, 476, - /* 870 */ 477, 347, 348, 360, 361, 382, 188, 363, 190, 453, - /* 880 */ 376, 453, 456, 390, 456, 338, 421, 347, 348, 424, - /* 890 */ 366, 377, 427, 428, 429, 430, 431, 432, 472, 434, - /* 900 */ 472, 387, 476, 477, 476, 477, 366, 387, 220, 221, - /* 910 */ 387, 223, 224, 225, 226, 227, 228, 229, 230, 231, - /* 920 */ 232, 233, 234, 235, 236, 237, 238, 239, 240, 12, - /* 930 */ 13, 190, 347, 348, 387, 338, 338, 20, 424, 22, - /* 940 */ 22, 257, 3, 478, 479, 348, 367, 350, 434, 376, - /* 950 */ 33, 366, 35, 35, 375, 188, 256, 190, 338, 8, - /* 960 */ 9, 338, 383, 12, 13, 14, 15, 16, 368, 8, - /* 970 */ 9, 371, 375, 12, 13, 14, 15, 16, 61, 20, - /* 980 */ 347, 348, 388, 389, 387, 387, 389, 220, 221, 347, - /* 990 */ 348, 74, 347, 348, 347, 348, 22, 0, 406, 366, - /* 1000 */ 233, 234, 235, 236, 237, 238, 239, 387, 366, 35, - /* 1010 */ 387, 366, 348, 366, 20, 376, 99, 99, 421, 102, - /* 1020 */ 20, 424, 22, 338, 427, 428, 429, 430, 431, 432, - /* 1030 */ 132, 434, 4, 348, 384, 350, 439, 387, 441, 347, - /* 1040 */ 348, 384, 445, 446, 387, 453, 167, 19, 456, 338, - /* 1050 */ 61, 347, 348, 53, 183, 138, 139, 393, 366, 338, - /* 1060 */ 375, 33, 338, 471, 472, 347, 348, 20, 476, 477, - /* 1070 */ 366, 338, 387, 99, 389, 347, 348, 49, 347, 348, - /* 1080 */ 20, 210, 211, 55, 366, 168, 169, 360, 361, 61, - /* 1090 */ 101, 174, 175, 104, 366, 45, 46, 366, 387, 201, - /* 1100 */ 202, 14, 15, 16, 14, 188, 421, 190, 387, 424, - /* 1110 */ 20, 387, 427, 428, 429, 430, 431, 432, 338, 434, - /* 1120 */ 387, 170, 406, 244, 439, 338, 441, 168, 338, 101, - /* 1130 */ 445, 446, 104, 254, 338, 255, 256, 220, 221, 348, - /* 1140 */ 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, - /* 1150 */ 233, 234, 235, 236, 237, 238, 239, 240, 12, 13, - /* 1160 */ 348, 338, 363, 438, 170, 440, 20, 387, 22, 453, - /* 1170 */ 0, 348, 456, 350, 387, 362, 377, 387, 365, 33, - /* 1180 */ 438, 35, 440, 387, 393, 113, 387, 471, 472, 22, - /* 1190 */ 48, 384, 476, 477, 387, 42, 0, 44, 375, 42, - /* 1200 */ 106, 44, 35, 109, 0, 393, 106, 61, 44, 109, - /* 1210 */ 387, 214, 389, 13, 0, 106, 277, 170, 109, 49, - /* 1220 */ 74, 0, 106, 424, 425, 109, 22, 204, 44, 206, - /* 1230 */ 170, 44, 44, 434, 162, 35, 22, 44, 44, 44, - /* 1240 */ 44, 138, 139, 22, 421, 99, 44, 424, 102, 67, - /* 1250 */ 427, 428, 429, 430, 431, 432, 44, 434, 47, 44, - /* 1260 */ 44, 0, 439, 35, 441, 102, 44, 103, 445, 446, - /* 1270 */ 44, 1, 2, 44, 44, 112, 44, 13, 13, 44, - /* 1280 */ 44, 44, 44, 44, 138, 139, 376, 103, 35, 376, - /* 1290 */ 103, 103, 376, 411, 364, 339, 103, 103, 397, 35, - /* 1300 */ 35, 480, 469, 18, 351, 103, 463, 363, 23, 363, - /* 1310 */ 375, 50, 170, 102, 168, 169, 351, 397, 103, 103, - /* 1320 */ 174, 175, 37, 38, 346, 103, 41, 74, 348, 103, - /* 1330 */ 397, 386, 103, 103, 188, 103, 190, 52, 103, 103, - /* 1340 */ 103, 103, 103, 455, 447, 473, 457, 62, 63, 64, - /* 1350 */ 65, 259, 423, 49, 422, 338, 20, 415, 203, 355, - /* 1360 */ 420, 355, 415, 186, 42, 348, 220, 221, 408, 223, - /* 1370 */ 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, - /* 1380 */ 234, 235, 236, 237, 238, 239, 240, 102, 20, 394, - /* 1390 */ 397, 167, 375, 394, 392, 20, 347, 394, 347, 392, - /* 1400 */ 392, 100, 220, 359, 387, 98, 389, 97, 347, 347, - /* 1410 */ 358, 357, 347, 347, 20, 340, 415, 48, 190, 340, - /* 1420 */ 344, 344, 355, 20, 338, 140, 389, 20, 355, 349, - /* 1430 */ 20, 407, 349, 355, 348, 347, 350, 355, 421, 355, - /* 1440 */ 355, 424, 54, 352, 427, 428, 429, 430, 431, 432, - /* 1450 */ 355, 434, 340, 352, 340, 347, 439, 387, 441, 207, - /* 1460 */ 419, 375, 445, 446, 375, 180, 181, 182, 375, 102, - /* 1470 */ 185, 375, 375, 387, 279, 389, 417, 375, 415, 375, - /* 1480 */ 375, 375, 375, 198, 199, 375, 338, 353, 375, 194, - /* 1490 */ 193, 353, 192, 281, 209, 414, 348, 212, 350, 347, - /* 1500 */ 215, 216, 217, 218, 219, 387, 387, 421, 397, 389, - /* 1510 */ 424, 413, 338, 427, 428, 429, 430, 431, 432, 397, - /* 1520 */ 434, 266, 348, 375, 387, 439, 462, 441, 265, 412, - /* 1530 */ 402, 445, 446, 462, 387, 387, 387, 389, 402, 465, - /* 1540 */ 274, 387, 257, 179, 276, 283, 275, 260, 423, 375, - /* 1550 */ 481, 278, 280, 348, 256, 20, 426, 347, 349, 353, - /* 1560 */ 461, 387, 20, 389, 402, 400, 353, 387, 387, 421, - /* 1570 */ 387, 402, 424, 387, 459, 427, 428, 429, 430, 431, - /* 1580 */ 432, 172, 434, 460, 387, 464, 338, 439, 462, 441, - /* 1590 */ 387, 399, 102, 445, 446, 421, 348, 353, 424, 475, - /* 1600 */ 353, 427, 428, 429, 430, 431, 432, 474, 434, 102, - /* 1610 */ 444, 371, 379, 439, 387, 441, 365, 348, 36, 445, - /* 1620 */ 446, 347, 341, 375, 340, 369, 409, 369, 353, 369, - /* 1630 */ 354, 0, 336, 0, 416, 387, 0, 389, 42, 0, - /* 1640 */ 35, 213, 35, 35, 403, 403, 35, 338, 0, 213, - /* 1650 */ 35, 35, 213, 0, 213, 0, 35, 348, 35, 22, - /* 1660 */ 0, 208, 0, 196, 0, 196, 190, 197, 188, 421, - /* 1670 */ 0, 0, 424, 338, 0, 427, 428, 429, 430, 431, - /* 1680 */ 432, 0, 434, 348, 375, 184, 183, 439, 0, 441, - /* 1690 */ 0, 47, 0, 445, 446, 0, 387, 0, 389, 42, - /* 1700 */ 0, 0, 0, 0, 0, 0, 0, 0, 157, 338, - /* 1710 */ 375, 35, 0, 157, 0, 0, 0, 0, 0, 348, - /* 1720 */ 42, 0, 387, 0, 389, 0, 0, 0, 0, 0, - /* 1730 */ 421, 0, 0, 424, 0, 0, 427, 428, 429, 430, - /* 1740 */ 431, 432, 0, 434, 0, 0, 375, 0, 0, 0, - /* 1750 */ 441, 0, 22, 0, 445, 446, 421, 0, 387, 424, - /* 1760 */ 389, 141, 427, 428, 429, 430, 431, 432, 0, 434, - /* 1770 */ 0, 0, 22, 338, 22, 35, 441, 0, 0, 0, - /* 1780 */ 445, 446, 179, 348, 48, 48, 0, 49, 0, 0, - /* 1790 */ 35, 0, 421, 35, 61, 424, 35, 338, 427, 428, - /* 1800 */ 429, 430, 431, 432, 61, 434, 39, 348, 49, 49, - /* 1810 */ 375, 39, 441, 39, 0, 61, 445, 446, 42, 35, - /* 1820 */ 39, 14, 387, 0, 389, 0, 39, 47, 0, 44, - /* 1830 */ 0, 0, 0, 40, 375, 39, 0, 47, 0, 47, - /* 1840 */ 0, 68, 39, 35, 49, 0, 387, 35, 389, 49, - /* 1850 */ 39, 0, 35, 49, 39, 0, 421, 35, 49, 424, - /* 1860 */ 39, 338, 427, 428, 429, 430, 431, 432, 0, 434, - /* 1870 */ 0, 348, 0, 0, 111, 109, 441, 35, 22, 0, - /* 1880 */ 421, 446, 1, 424, 35, 35, 427, 428, 429, 430, - /* 1890 */ 431, 432, 44, 434, 35, 35, 35, 35, 375, 35, - /* 1900 */ 19, 35, 44, 380, 35, 22, 35, 0, 22, 0, - /* 1910 */ 387, 22, 389, 0, 33, 22, 51, 35, 0, 338, - /* 1920 */ 35, 0, 35, 0, 22, 20, 35, 468, 102, 348, - /* 1930 */ 49, 35, 35, 0, 170, 338, 103, 56, 57, 58, - /* 1940 */ 59, 102, 61, 35, 421, 348, 22, 424, 0, 22, - /* 1950 */ 427, 428, 429, 430, 431, 432, 375, 434, 0, 0, - /* 1960 */ 195, 380, 48, 3, 170, 102, 191, 100, 387, 172, - /* 1970 */ 389, 170, 375, 44, 261, 177, 48, 103, 102, 44, - /* 1980 */ 44, 103, 101, 103, 387, 104, 389, 103, 98, 47, - /* 1990 */ 102, 44, 3, 47, 102, 44, 338, 102, 44, 35, - /* 2000 */ 102, 35, 421, 103, 103, 424, 348, 261, 427, 428, - /* 2010 */ 429, 430, 431, 432, 35, 434, 35, 136, 421, 35, - /* 2020 */ 35, 424, 338, 261, 427, 428, 429, 430, 431, 432, - /* 2030 */ 47, 434, 348, 375, 103, 103, 44, 47, 255, 0, - /* 2040 */ 0, 0, 0, 102, 39, 387, 47, 389, 173, 0, - /* 2050 */ 39, 102, 171, 103, 47, 103, 112, 176, 338, 375, - /* 2060 */ 44, 242, 47, 102, 380, 102, 2, 470, 348, 102, - /* 2070 */ 171, 387, 102, 389, 100, 100, 22, 196, 102, 421, - /* 2080 */ 103, 103, 424, 103, 102, 427, 428, 429, 430, 431, - /* 2090 */ 432, 47, 434, 102, 338, 375, 102, 102, 22, 103, - /* 2100 */ 102, 113, 35, 103, 348, 421, 35, 387, 424, 389, - /* 2110 */ 102, 427, 428, 429, 430, 431, 432, 103, 434, 35, - /* 2120 */ 338, 102, 35, 220, 103, 102, 222, 103, 35, 102, - /* 2130 */ 348, 375, 103, 35, 102, 102, 380, 479, 44, 102, - /* 2140 */ 35, 421, 124, 387, 424, 389, 124, 427, 428, 429, - /* 2150 */ 430, 431, 432, 102, 434, 124, 436, 375, 124, 22, - /* 2160 */ 68, 35, 380, 67, 35, 35, 35, 35, 35, 387, - /* 2170 */ 35, 389, 44, 74, 35, 96, 35, 421, 35, 35, - /* 2180 */ 424, 22, 35, 427, 428, 429, 430, 431, 432, 35, - /* 2190 */ 434, 35, 74, 35, 22, 338, 35, 35, 35, 35, - /* 2200 */ 35, 0, 35, 421, 49, 348, 424, 39, 0, 427, - /* 2210 */ 428, 429, 430, 431, 432, 338, 434, 35, 49, 39, - /* 2220 */ 0, 0, 35, 49, 39, 348, 35, 49, 39, 0, - /* 2230 */ 35, 35, 375, 0, 22, 21, 482, 22, 22, 21, - /* 2240 */ 482, 482, 20, 482, 387, 482, 389, 482, 482, 482, - /* 2250 */ 482, 482, 375, 482, 482, 482, 482, 482, 482, 482, - /* 2260 */ 482, 482, 482, 482, 387, 482, 389, 482, 482, 482, - /* 2270 */ 482, 482, 482, 482, 482, 482, 482, 482, 421, 482, - /* 2280 */ 482, 424, 482, 338, 427, 428, 429, 430, 431, 432, - /* 2290 */ 482, 434, 482, 348, 482, 482, 482, 482, 421, 482, - /* 2300 */ 338, 424, 482, 482, 427, 428, 429, 430, 431, 432, - /* 2310 */ 348, 434, 482, 482, 482, 482, 482, 482, 482, 482, - /* 2320 */ 375, 482, 482, 482, 482, 482, 482, 482, 482, 482, - /* 2330 */ 482, 482, 387, 482, 389, 482, 482, 375, 482, 482, - /* 2340 */ 482, 482, 482, 482, 482, 482, 482, 482, 338, 387, - /* 2350 */ 482, 389, 482, 482, 482, 482, 482, 482, 348, 482, - /* 2360 */ 482, 482, 482, 482, 482, 482, 421, 482, 338, 424, - /* 2370 */ 482, 482, 427, 428, 429, 430, 431, 432, 348, 434, - /* 2380 */ 482, 482, 482, 421, 482, 375, 424, 482, 482, 427, - /* 2390 */ 428, 429, 430, 431, 432, 482, 434, 387, 338, 389, - /* 2400 */ 482, 482, 482, 482, 482, 375, 482, 482, 348, 482, - /* 2410 */ 482, 482, 482, 482, 482, 482, 338, 387, 482, 389, - /* 2420 */ 482, 482, 482, 482, 482, 482, 348, 482, 482, 482, - /* 2430 */ 482, 421, 482, 482, 424, 375, 482, 427, 428, 429, - /* 2440 */ 430, 431, 432, 482, 434, 482, 338, 387, 482, 389, - /* 2450 */ 482, 421, 482, 375, 424, 482, 348, 427, 428, 429, - /* 2460 */ 430, 431, 432, 482, 434, 387, 482, 389, 482, 482, - /* 2470 */ 482, 482, 482, 482, 482, 482, 338, 482, 482, 482, - /* 2480 */ 482, 421, 482, 375, 424, 482, 348, 427, 428, 429, - /* 2490 */ 430, 431, 432, 482, 434, 387, 482, 389, 482, 421, - /* 2500 */ 482, 482, 424, 482, 482, 427, 428, 429, 430, 431, - /* 2510 */ 432, 482, 434, 375, 482, 482, 482, 482, 482, 482, - /* 2520 */ 482, 482, 482, 482, 338, 387, 482, 389, 482, 421, - /* 2530 */ 482, 482, 424, 482, 348, 427, 428, 429, 430, 431, - /* 2540 */ 432, 482, 434, 482, 338, 482, 482, 482, 482, 482, - /* 2550 */ 482, 482, 482, 482, 348, 482, 482, 482, 482, 421, - /* 2560 */ 482, 375, 424, 482, 482, 427, 428, 429, 430, 431, - /* 2570 */ 432, 482, 434, 387, 482, 389, 482, 482, 482, 482, - /* 2580 */ 482, 375, 482, 482, 482, 482, 482, 482, 482, 482, - /* 2590 */ 482, 482, 482, 387, 482, 389, 482, 482, 482, 482, - /* 2600 */ 482, 482, 482, 482, 482, 482, 482, 421, 482, 482, - /* 2610 */ 424, 482, 482, 427, 428, 429, 430, 431, 432, 482, - /* 2620 */ 434, 482, 338, 482, 482, 482, 482, 421, 482, 482, - /* 2630 */ 424, 482, 348, 427, 428, 429, 430, 431, 432, 338, - /* 2640 */ 434, 482, 482, 482, 482, 482, 482, 482, 482, 348, - /* 2650 */ 482, 482, 482, 482, 482, 482, 482, 482, 482, 375, - /* 2660 */ 482, 482, 482, 482, 482, 482, 482, 482, 482, 482, - /* 2670 */ 482, 387, 482, 389, 482, 482, 375, 482, 482, 482, - /* 2680 */ 482, 482, 482, 482, 482, 482, 482, 338, 387, 482, - /* 2690 */ 389, 482, 482, 482, 482, 482, 482, 348, 482, 482, - /* 2700 */ 482, 482, 482, 482, 482, 421, 482, 338, 424, 482, - /* 2710 */ 482, 427, 428, 429, 430, 431, 432, 348, 434, 482, - /* 2720 */ 482, 482, 421, 482, 375, 424, 482, 482, 427, 428, - /* 2730 */ 429, 430, 431, 432, 482, 434, 387, 338, 389, 482, - /* 2740 */ 482, 482, 482, 482, 375, 482, 482, 348, 482, 482, - /* 2750 */ 482, 482, 482, 482, 482, 338, 387, 482, 389, 482, - /* 2760 */ 482, 482, 482, 482, 482, 348, 482, 482, 482, 482, - /* 2770 */ 421, 482, 482, 424, 375, 482, 427, 428, 429, 430, - /* 2780 */ 431, 432, 482, 434, 482, 338, 387, 482, 389, 482, - /* 2790 */ 421, 482, 375, 424, 482, 348, 427, 428, 429, 430, - /* 2800 */ 431, 432, 482, 434, 387, 482, 389, 482, 482, 482, - /* 2810 */ 482, 482, 482, 482, 482, 338, 482, 482, 482, 482, - /* 2820 */ 421, 482, 375, 424, 482, 348, 427, 428, 429, 430, - /* 2830 */ 431, 432, 482, 434, 387, 482, 389, 482, 421, 482, - /* 2840 */ 482, 424, 482, 482, 427, 428, 429, 430, 431, 432, - /* 2850 */ 482, 434, 375, 482, 482, 482, 482, 482, 482, 482, - /* 2860 */ 482, 482, 482, 338, 387, 482, 389, 482, 421, 482, - /* 2870 */ 482, 424, 482, 348, 427, 428, 429, 430, 431, 432, - /* 2880 */ 482, 434, 482, 338, 482, 482, 482, 482, 482, 482, - /* 2890 */ 482, 482, 482, 348, 482, 482, 482, 482, 421, 482, - /* 2900 */ 375, 424, 482, 482, 427, 428, 429, 430, 431, 432, - /* 2910 */ 482, 434, 387, 482, 389, 482, 482, 482, 482, 482, - /* 2920 */ 375, 482, 482, 482, 482, 482, 482, 482, 482, 482, - /* 2930 */ 482, 482, 387, 482, 389, 482, 482, 482, 482, 482, - /* 2940 */ 482, 482, 482, 482, 482, 482, 421, 482, 482, 424, - /* 2950 */ 482, 482, 427, 428, 429, 430, 431, 432, 482, 434, - /* 2960 */ 482, 482, 482, 482, 482, 482, 421, 482, 482, 424, - /* 2970 */ 482, 482, 427, 428, 429, 430, 431, 432, 482, 434, - /* 2980 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 2990 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3000 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3010 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3020 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3030 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3040 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3050 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3060 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3070 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3080 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3090 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3100 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3110 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3120 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3130 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3140 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3150 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3160 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3170 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3180 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3190 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3200 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3210 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3220 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3230 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3240 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3250 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3260 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3270 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3280 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3290 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3300 */ 335, 335, 335, 335, 335, 335, 335, 335, 335, 335, - /* 3310 */ 335, 335, 335, 335, 335, + /* 240 */ 240, 241, 242, 243, 140, 141, 379, 213, 214, 24, + /* 250 */ 25, 26, 27, 28, 29, 30, 31, 32, 391, 75, + /* 260 */ 393, 0, 247, 248, 249, 250, 251, 252, 253, 254, + /* 270 */ 255, 256, 257, 345, 170, 171, 348, 349, 67, 20, + /* 280 */ 176, 177, 21, 170, 171, 24, 25, 26, 27, 28, + /* 290 */ 29, 30, 31, 32, 427, 191, 429, 193, 175, 432, + /* 300 */ 433, 434, 435, 436, 437, 20, 439, 0, 371, 442, + /* 310 */ 20, 444, 445, 446, 20, 341, 379, 450, 451, 20, + /* 320 */ 260, 22, 350, 351, 387, 351, 260, 223, 224, 4, + /* 330 */ 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, + /* 340 */ 236, 237, 238, 239, 240, 241, 242, 243, 244, 12, + /* 350 */ 13, 67, 53, 379, 379, 12, 13, 20, 47, 22, + /* 360 */ 458, 386, 103, 461, 379, 391, 20, 393, 114, 394, + /* 370 */ 33, 260, 35, 70, 71, 72, 350, 351, 35, 477, + /* 380 */ 77, 78, 79, 481, 482, 400, 83, 351, 265, 266, + /* 390 */ 267, 88, 89, 90, 91, 442, 370, 94, 445, 62, + /* 400 */ 135, 427, 48, 429, 139, 68, 432, 433, 434, 435, + /* 410 */ 436, 437, 75, 439, 103, 379, 134, 135, 444, 345, + /* 420 */ 446, 139, 348, 349, 450, 451, 454, 455, 456, 341, + /* 430 */ 458, 459, 14, 458, 140, 141, 461, 100, 20, 351, + /* 440 */ 103, 353, 8, 9, 35, 471, 12, 13, 14, 15, + /* 450 */ 16, 476, 477, 12, 13, 14, 481, 482, 346, 341, + /* 460 */ 170, 20, 350, 22, 352, 200, 159, 379, 203, 351, + /* 470 */ 434, 206, 379, 208, 33, 168, 35, 140, 141, 391, + /* 480 */ 387, 393, 458, 21, 75, 461, 24, 25, 26, 27, + /* 490 */ 28, 29, 30, 31, 32, 379, 103, 379, 341, 350, + /* 500 */ 351, 477, 386, 62, 244, 481, 482, 170, 171, 391, + /* 510 */ 394, 393, 357, 176, 177, 427, 75, 429, 172, 260, + /* 520 */ 432, 433, 434, 435, 436, 437, 172, 439, 191, 374, + /* 530 */ 193, 133, 444, 179, 446, 137, 193, 382, 450, 451, + /* 540 */ 14, 100, 350, 351, 103, 427, 20, 429, 391, 341, + /* 550 */ 432, 433, 434, 435, 436, 437, 244, 439, 246, 351, + /* 560 */ 223, 224, 370, 226, 227, 228, 229, 230, 231, 232, + /* 570 */ 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, + /* 580 */ 243, 140, 141, 190, 22, 192, 261, 379, 8, 9, + /* 590 */ 472, 473, 12, 13, 14, 15, 16, 35, 431, 391, + /* 600 */ 20, 393, 453, 454, 455, 456, 172, 458, 459, 75, + /* 610 */ 341, 170, 171, 350, 351, 222, 389, 176, 177, 392, + /* 620 */ 393, 8, 9, 371, 457, 12, 13, 14, 15, 16, + /* 630 */ 431, 379, 191, 370, 193, 427, 350, 429, 386, 387, + /* 640 */ 432, 433, 434, 435, 436, 437, 394, 439, 134, 20, + /* 650 */ 411, 341, 444, 260, 446, 416, 457, 411, 450, 451, + /* 660 */ 391, 351, 100, 353, 223, 224, 44, 226, 227, 228, + /* 670 */ 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, + /* 680 */ 239, 240, 241, 242, 243, 12, 13, 0, 402, 379, + /* 690 */ 404, 350, 351, 20, 75, 22, 170, 458, 350, 351, + /* 700 */ 461, 391, 379, 393, 458, 0, 33, 461, 35, 386, + /* 710 */ 12, 13, 14, 15, 16, 476, 477, 394, 204, 205, + /* 720 */ 481, 482, 476, 477, 350, 351, 104, 481, 482, 1, + /* 730 */ 2, 223, 103, 350, 351, 62, 380, 427, 397, 429, + /* 740 */ 4, 350, 432, 433, 434, 435, 436, 437, 75, 439, + /* 750 */ 350, 351, 172, 370, 444, 19, 446, 0, 8, 9, + /* 760 */ 450, 451, 12, 13, 14, 15, 16, 341, 341, 33, + /* 770 */ 370, 397, 67, 100, 350, 351, 103, 351, 270, 271, + /* 780 */ 272, 273, 274, 275, 276, 49, 350, 351, 3, 12, + /* 790 */ 13, 0, 56, 402, 370, 404, 183, 20, 62, 22, + /* 800 */ 371, 20, 454, 455, 456, 379, 458, 459, 379, 443, + /* 810 */ 33, 445, 35, 140, 141, 0, 387, 391, 391, 393, + /* 820 */ 133, 134, 135, 136, 137, 138, 139, 70, 71, 72, + /* 830 */ 392, 393, 104, 397, 77, 78, 79, 22, 102, 62, + /* 840 */ 83, 105, 103, 170, 171, 88, 89, 90, 91, 176, + /* 850 */ 177, 94, 75, 427, 104, 429, 54, 55, 432, 433, + /* 860 */ 434, 435, 436, 437, 191, 439, 193, 441, 8, 9, + /* 870 */ 341, 380, 12, 13, 14, 15, 16, 100, 355, 356, + /* 880 */ 103, 341, 133, 134, 135, 136, 137, 138, 139, 341, + /* 890 */ 411, 351, 62, 353, 341, 380, 223, 224, 285, 226, + /* 900 */ 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, + /* 910 */ 237, 238, 239, 240, 241, 242, 243, 140, 141, 379, + /* 920 */ 391, 140, 141, 21, 133, 134, 135, 136, 137, 138, + /* 930 */ 139, 391, 102, 393, 379, 105, 34, 458, 36, 391, + /* 940 */ 461, 386, 350, 351, 391, 350, 351, 170, 171, 394, + /* 950 */ 114, 350, 351, 176, 177, 476, 477, 176, 177, 380, + /* 960 */ 481, 482, 370, 388, 104, 370, 391, 427, 191, 429, + /* 970 */ 193, 370, 432, 433, 434, 435, 436, 437, 39, 439, + /* 980 */ 364, 365, 2, 0, 444, 22, 446, 350, 8, 9, + /* 990 */ 450, 451, 12, 13, 14, 15, 16, 341, 35, 260, + /* 1000 */ 223, 224, 0, 226, 227, 228, 229, 230, 231, 232, + /* 1010 */ 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, + /* 1020 */ 243, 14, 18, 18, 20, 350, 351, 20, 23, 350, + /* 1030 */ 351, 27, 49, 359, 30, 355, 356, 33, 367, 402, + /* 1040 */ 371, 404, 37, 38, 0, 370, 41, 391, 379, 370, + /* 1050 */ 376, 379, 381, 49, 411, 51, 387, 52, 386, 385, + /* 1060 */ 56, 380, 391, 100, 341, 280, 394, 341, 63, 64, + /* 1070 */ 65, 66, 70, 71, 72, 73, 74, 0, 76, 77, + /* 1080 */ 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, + /* 1090 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 20, + /* 1100 */ 429, 458, 350, 351, 461, 20, 102, 33, 103, 37, + /* 1110 */ 439, 14, 15, 16, 391, 350, 351, 391, 114, 476, + /* 1120 */ 477, 44, 370, 49, 481, 482, 350, 351, 417, 350, + /* 1130 */ 351, 57, 58, 59, 60, 370, 62, 341, 44, 443, + /* 1140 */ 169, 445, 364, 365, 341, 341, 370, 142, 144, 370, + /* 1150 */ 341, 147, 148, 149, 150, 151, 152, 153, 154, 155, + /* 1160 */ 156, 157, 158, 159, 160, 161, 162, 163, 44, 165, + /* 1170 */ 166, 167, 350, 351, 258, 259, 102, 170, 106, 105, + /* 1180 */ 108, 109, 0, 111, 359, 341, 341, 391, 183, 184, + /* 1190 */ 185, 341, 370, 188, 391, 391, 341, 379, 259, 388, + /* 1200 */ 391, 351, 391, 353, 22, 133, 201, 202, 341, 137, + /* 1210 */ 385, 388, 394, 372, 391, 368, 375, 212, 247, 42, + /* 1220 */ 215, 44, 341, 218, 219, 220, 221, 222, 257, 379, + /* 1230 */ 366, 22, 351, 369, 353, 391, 391, 45, 46, 44, + /* 1240 */ 42, 391, 44, 393, 35, 114, 391, 173, 174, 13, + /* 1250 */ 107, 172, 341, 110, 180, 181, 107, 172, 391, 110, + /* 1260 */ 379, 217, 351, 107, 353, 260, 110, 107, 44, 13, + /* 1270 */ 110, 35, 391, 199, 393, 0, 44, 427, 207, 429, + /* 1280 */ 209, 44, 432, 433, 434, 435, 436, 437, 68, 439, + /* 1290 */ 379, 35, 140, 141, 444, 164, 446, 22, 342, 104, + /* 1300 */ 450, 451, 391, 474, 393, 35, 44, 44, 427, 103, + /* 1310 */ 429, 401, 485, 432, 433, 434, 435, 436, 437, 113, + /* 1320 */ 439, 44, 44, 35, 22, 444, 44, 446, 104, 44, + /* 1330 */ 44, 450, 451, 1, 2, 44, 104, 35, 427, 44, + /* 1340 */ 429, 104, 341, 432, 433, 434, 435, 436, 437, 44, + /* 1350 */ 439, 0, 351, 12, 13, 444, 468, 446, 2, 44, + /* 1360 */ 354, 450, 451, 22, 8, 9, 104, 104, 12, 13, + /* 1370 */ 14, 15, 16, 341, 33, 367, 35, 75, 284, 44, + /* 1380 */ 379, 104, 104, 351, 44, 13, 104, 367, 379, 104, + /* 1390 */ 104, 13, 391, 44, 393, 104, 410, 44, 401, 104, + /* 1400 */ 354, 50, 100, 62, 349, 351, 282, 35, 390, 104, + /* 1410 */ 341, 379, 401, 35, 460, 478, 75, 452, 49, 104, + /* 1420 */ 351, 462, 412, 391, 262, 393, 428, 20, 427, 206, + /* 1430 */ 429, 189, 359, 432, 433, 434, 435, 436, 437, 104, + /* 1440 */ 439, 100, 421, 223, 104, 444, 421, 446, 379, 426, + /* 1450 */ 359, 450, 451, 104, 414, 20, 351, 104, 42, 427, + /* 1460 */ 391, 429, 393, 193, 432, 433, 434, 435, 436, 437, + /* 1470 */ 20, 439, 351, 398, 396, 169, 444, 401, 446, 398, + /* 1480 */ 20, 193, 450, 451, 350, 398, 351, 350, 396, 396, + /* 1490 */ 101, 363, 99, 191, 362, 193, 427, 350, 429, 341, + /* 1500 */ 98, 432, 433, 434, 435, 436, 437, 361, 439, 351, + /* 1510 */ 20, 350, 343, 350, 350, 446, 48, 347, 343, 450, + /* 1520 */ 451, 347, 421, 20, 359, 223, 224, 359, 20, 393, + /* 1530 */ 352, 20, 191, 413, 193, 359, 352, 379, 359, 359, + /* 1540 */ 359, 350, 359, 343, 343, 379, 379, 350, 341, 391, + /* 1550 */ 379, 393, 379, 210, 425, 103, 423, 197, 351, 421, + /* 1560 */ 357, 420, 196, 391, 223, 224, 379, 379, 379, 379, + /* 1570 */ 419, 357, 379, 379, 341, 379, 391, 236, 237, 238, + /* 1580 */ 239, 240, 241, 242, 351, 427, 379, 429, 350, 195, + /* 1590 */ 432, 433, 434, 435, 436, 437, 391, 439, 391, 269, + /* 1600 */ 393, 393, 391, 277, 446, 401, 467, 268, 450, 451, + /* 1610 */ 391, 401, 379, 391, 412, 467, 391, 418, 406, 406, + /* 1620 */ 470, 182, 469, 279, 391, 341, 393, 263, 278, 412, + /* 1630 */ 486, 286, 259, 281, 427, 351, 429, 283, 341, 432, + /* 1640 */ 433, 434, 435, 436, 437, 351, 439, 20, 351, 357, + /* 1650 */ 352, 20, 466, 446, 404, 406, 357, 450, 451, 431, + /* 1660 */ 427, 406, 429, 379, 391, 432, 433, 434, 435, 436, + /* 1670 */ 437, 438, 439, 440, 441, 391, 379, 393, 391, 174, + /* 1680 */ 391, 465, 467, 403, 391, 464, 357, 341, 391, 375, + /* 1690 */ 393, 391, 391, 357, 103, 449, 103, 351, 480, 351, + /* 1700 */ 383, 391, 369, 36, 341, 479, 344, 350, 343, 0, + /* 1710 */ 407, 427, 357, 429, 351, 407, 432, 433, 434, 435, + /* 1720 */ 436, 437, 415, 439, 427, 379, 429, 339, 358, 432, + /* 1730 */ 433, 434, 435, 436, 437, 422, 439, 391, 0, 393, + /* 1740 */ 373, 373, 379, 446, 373, 0, 42, 384, 451, 0, + /* 1750 */ 35, 216, 35, 341, 391, 35, 393, 35, 216, 0, + /* 1760 */ 35, 35, 216, 351, 0, 216, 0, 483, 484, 35, + /* 1770 */ 0, 0, 22, 427, 35, 429, 211, 341, 432, 433, + /* 1780 */ 434, 435, 436, 437, 0, 439, 199, 351, 0, 199, + /* 1790 */ 427, 379, 429, 193, 200, 432, 433, 434, 435, 436, + /* 1800 */ 437, 191, 439, 391, 341, 393, 0, 0, 0, 187, + /* 1810 */ 186, 0, 0, 0, 351, 379, 47, 0, 0, 473, + /* 1820 */ 384, 47, 0, 42, 341, 0, 0, 391, 0, 393, + /* 1830 */ 47, 0, 0, 0, 351, 0, 0, 0, 159, 427, + /* 1840 */ 35, 429, 379, 0, 432, 433, 434, 435, 436, 437, + /* 1850 */ 159, 439, 0, 0, 391, 0, 393, 0, 0, 0, + /* 1860 */ 0, 0, 379, 427, 0, 429, 0, 384, 432, 433, + /* 1870 */ 434, 435, 436, 437, 391, 439, 393, 47, 0, 0, + /* 1880 */ 42, 0, 0, 0, 0, 0, 62, 475, 0, 0, + /* 1890 */ 427, 341, 429, 0, 0, 432, 433, 434, 435, 436, + /* 1900 */ 437, 351, 439, 0, 0, 0, 0, 0, 22, 0, + /* 1910 */ 427, 143, 429, 0, 22, 432, 433, 434, 435, 436, + /* 1920 */ 437, 341, 439, 48, 48, 0, 35, 0, 0, 379, + /* 1930 */ 22, 351, 62, 62, 384, 0, 0, 39, 0, 49, + /* 1940 */ 39, 391, 0, 393, 49, 35, 35, 484, 49, 35, + /* 1950 */ 0, 39, 35, 0, 14, 39, 0, 42, 0, 379, + /* 1960 */ 0, 39, 44, 40, 384, 0, 47, 47, 47, 39, + /* 1970 */ 0, 391, 182, 393, 35, 49, 39, 427, 0, 429, + /* 1980 */ 341, 0, 432, 433, 434, 435, 436, 437, 1, 439, + /* 1990 */ 351, 0, 0, 69, 0, 49, 35, 0, 39, 35, + /* 2000 */ 49, 39, 0, 35, 49, 39, 19, 427, 0, 429, + /* 2010 */ 0, 0, 432, 433, 434, 435, 436, 437, 379, 439, + /* 2020 */ 33, 0, 0, 0, 0, 35, 22, 44, 35, 44, + /* 2030 */ 391, 35, 393, 35, 35, 35, 49, 0, 35, 22, + /* 2040 */ 35, 112, 110, 0, 57, 58, 59, 60, 35, 62, + /* 2050 */ 35, 35, 22, 341, 22, 0, 22, 51, 0, 0, + /* 2060 */ 35, 35, 35, 351, 0, 0, 427, 22, 429, 20, + /* 2070 */ 35, 432, 433, 434, 435, 436, 437, 35, 439, 35, + /* 2080 */ 104, 35, 103, 103, 341, 198, 0, 0, 22, 102, + /* 2090 */ 194, 379, 105, 22, 351, 0, 3, 44, 264, 104, + /* 2100 */ 48, 44, 3, 391, 103, 393, 172, 174, 48, 178, + /* 2110 */ 47, 99, 103, 172, 44, 172, 172, 44, 104, 341, + /* 2120 */ 101, 103, 379, 178, 104, 138, 47, 44, 44, 351, + /* 2130 */ 258, 103, 103, 264, 391, 104, 393, 104, 264, 427, + /* 2140 */ 104, 429, 103, 35, 432, 433, 434, 435, 436, 437, + /* 2150 */ 35, 439, 35, 35, 35, 35, 341, 379, 47, 47, + /* 2160 */ 173, 0, 0, 47, 39, 104, 351, 180, 44, 391, + /* 2170 */ 427, 393, 429, 104, 0, 432, 433, 434, 435, 436, + /* 2180 */ 437, 341, 439, 103, 103, 103, 199, 39, 113, 104, + /* 2190 */ 47, 351, 104, 103, 379, 103, 44, 103, 101, 245, + /* 2200 */ 2, 101, 22, 47, 103, 427, 391, 429, 393, 223, + /* 2210 */ 432, 433, 434, 435, 436, 437, 175, 439, 341, 379, + /* 2220 */ 173, 22, 47, 114, 35, 104, 104, 35, 351, 103, + /* 2230 */ 103, 391, 104, 393, 103, 103, 35, 104, 103, 35, + /* 2240 */ 104, 103, 427, 341, 429, 104, 104, 432, 433, 434, + /* 2250 */ 435, 436, 437, 351, 439, 104, 379, 103, 103, 35, + /* 2260 */ 103, 35, 104, 225, 104, 35, 103, 427, 391, 429, + /* 2270 */ 393, 103, 432, 433, 434, 435, 436, 437, 125, 439, + /* 2280 */ 103, 379, 44, 35, 125, 22, 103, 103, 69, 125, + /* 2290 */ 125, 68, 35, 391, 35, 393, 35, 35, 44, 35, + /* 2300 */ 35, 35, 35, 75, 427, 97, 429, 35, 35, 432, + /* 2310 */ 433, 434, 435, 436, 437, 341, 439, 35, 22, 35, + /* 2320 */ 35, 35, 75, 35, 35, 351, 35, 35, 35, 427, + /* 2330 */ 341, 429, 22, 35, 432, 433, 434, 435, 436, 437, + /* 2340 */ 351, 439, 0, 35, 49, 39, 0, 35, 341, 49, + /* 2350 */ 0, 35, 39, 379, 49, 0, 39, 35, 351, 39, + /* 2360 */ 0, 35, 35, 49, 0, 391, 22, 393, 379, 21, + /* 2370 */ 487, 22, 20, 22, 21, 487, 487, 487, 487, 487, + /* 2380 */ 391, 487, 393, 487, 487, 487, 379, 487, 487, 487, + /* 2390 */ 487, 487, 487, 487, 487, 341, 487, 487, 391, 487, + /* 2400 */ 393, 427, 487, 429, 487, 351, 432, 433, 434, 435, + /* 2410 */ 436, 437, 487, 439, 487, 487, 427, 341, 429, 487, + /* 2420 */ 487, 432, 433, 434, 435, 436, 437, 351, 439, 487, + /* 2430 */ 487, 487, 487, 379, 427, 487, 429, 487, 487, 432, + /* 2440 */ 433, 434, 435, 436, 437, 391, 439, 393, 487, 487, + /* 2450 */ 487, 487, 487, 487, 487, 379, 487, 487, 487, 487, + /* 2460 */ 487, 487, 487, 487, 341, 487, 487, 391, 487, 393, + /* 2470 */ 487, 487, 487, 487, 351, 487, 487, 487, 487, 487, + /* 2480 */ 487, 427, 487, 429, 487, 487, 432, 433, 434, 435, + /* 2490 */ 436, 437, 487, 439, 487, 487, 487, 487, 487, 487, + /* 2500 */ 487, 487, 379, 427, 487, 429, 487, 487, 432, 433, + /* 2510 */ 434, 435, 436, 437, 391, 439, 393, 487, 487, 487, + /* 2520 */ 487, 487, 487, 341, 487, 487, 487, 487, 487, 487, + /* 2530 */ 487, 487, 487, 351, 487, 487, 341, 487, 487, 487, + /* 2540 */ 487, 487, 487, 487, 487, 487, 351, 487, 487, 487, + /* 2550 */ 427, 487, 429, 341, 487, 432, 433, 434, 435, 436, + /* 2560 */ 437, 379, 439, 351, 487, 487, 487, 487, 487, 487, + /* 2570 */ 487, 487, 487, 391, 379, 393, 487, 487, 487, 487, + /* 2580 */ 487, 487, 487, 487, 487, 341, 391, 487, 393, 487, + /* 2590 */ 487, 379, 487, 487, 487, 351, 487, 487, 487, 487, + /* 2600 */ 487, 487, 487, 391, 487, 393, 487, 487, 487, 427, + /* 2610 */ 487, 429, 487, 487, 432, 433, 434, 435, 436, 437, + /* 2620 */ 487, 439, 427, 379, 429, 487, 487, 432, 433, 434, + /* 2630 */ 435, 436, 437, 487, 439, 391, 487, 393, 487, 427, + /* 2640 */ 487, 429, 487, 487, 432, 433, 434, 435, 436, 437, + /* 2650 */ 487, 439, 341, 487, 487, 487, 487, 487, 487, 487, + /* 2660 */ 487, 487, 351, 487, 487, 487, 487, 487, 487, 487, + /* 2670 */ 487, 427, 341, 429, 487, 487, 432, 433, 434, 435, + /* 2680 */ 436, 437, 351, 439, 487, 487, 487, 487, 487, 487, + /* 2690 */ 379, 487, 487, 487, 487, 487, 487, 487, 487, 487, + /* 2700 */ 487, 487, 391, 487, 393, 487, 487, 487, 487, 487, + /* 2710 */ 379, 487, 487, 487, 487, 487, 487, 487, 487, 487, + /* 2720 */ 487, 487, 391, 487, 393, 487, 487, 487, 487, 487, + /* 2730 */ 487, 487, 487, 487, 487, 487, 487, 487, 427, 487, + /* 2740 */ 429, 487, 341, 432, 433, 434, 435, 436, 437, 487, + /* 2750 */ 439, 487, 351, 487, 487, 487, 487, 487, 427, 341, + /* 2760 */ 429, 487, 487, 432, 433, 434, 435, 436, 437, 351, + /* 2770 */ 439, 487, 487, 487, 487, 487, 487, 487, 487, 487, + /* 2780 */ 379, 487, 487, 487, 487, 487, 487, 487, 487, 487, + /* 2790 */ 487, 487, 391, 487, 393, 487, 487, 379, 487, 487, + /* 2800 */ 487, 487, 487, 487, 487, 487, 487, 487, 487, 391, + /* 2810 */ 487, 393, 487, 487, 487, 487, 487, 487, 487, 487, + /* 2820 */ 487, 487, 487, 487, 487, 487, 487, 487, 427, 487, + /* 2830 */ 429, 487, 487, 432, 433, 434, 435, 436, 437, 487, + /* 2840 */ 439, 487, 487, 487, 487, 427, 487, 429, 487, 487, + /* 2850 */ 432, 433, 434, 435, 436, 437, 487, 439, 338, 338, + /* 2860 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2870 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2880 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2890 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2900 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2910 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2920 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2930 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2940 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2950 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2960 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2970 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2980 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2990 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3000 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3010 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3020 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3030 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3040 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3050 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3060 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3070 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3080 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3090 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3100 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3110 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3120 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3130 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3140 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3150 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3160 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3170 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3180 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3190 */ 338, 338, 338, 338, 338, 338, }; -#define YY_SHIFT_COUNT (786) +#define YY_SHIFT_COUNT (799) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2233) +#define YY_SHIFT_MAX (2364) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 1285, 0, 229, 0, 459, 459, 459, 459, 459, 459, - /* 10 */ 459, 459, 459, 459, 459, 459, 688, 917, 917, 1146, - /* 20 */ 917, 917, 917, 917, 917, 917, 917, 917, 917, 917, - /* 30 */ 917, 917, 917, 917, 917, 917, 917, 917, 917, 917, - /* 40 */ 917, 917, 917, 917, 917, 917, 917, 917, 917, 917, - /* 50 */ 917, 240, 603, 610, 375, 684, 89, 161, 89, 375, - /* 60 */ 375, 767, 89, 767, 767, 553, 89, 56, 380, 139, - /* 70 */ 139, 380, 280, 280, 197, 28, 48, 48, 139, 139, - /* 80 */ 139, 139, 139, 139, 139, 193, 139, 139, 156, 56, - /* 90 */ 139, 139, 239, 139, 56, 139, 193, 139, 193, 56, - /* 100 */ 139, 139, 56, 139, 56, 56, 56, 139, 293, 228, - /* 110 */ 189, 189, 198, 116, 604, 604, 604, 604, 604, 604, - /* 120 */ 604, 604, 604, 604, 604, 604, 604, 604, 604, 604, - /* 130 */ 604, 604, 604, 379, 406, 197, 28, 674, 994, 994, - /* 140 */ 994, 793, 607, 607, 674, 543, 543, 543, 156, 488, - /* 150 */ 342, 56, 632, 56, 632, 632, 546, 672, 35, 35, - /* 160 */ 35, 35, 35, 35, 35, 35, 1881, 575, 15, 44, - /* 170 */ 164, 275, 691, 368, 181, 181, 599, 646, 1000, 1047, - /* 180 */ 1050, 1090, 64, 1060, 880, 700, 939, 880, 1153, 295, - /* 190 */ 959, 1092, 1304, 1336, 1155, 156, 1336, 156, 1177, 1322, - /* 200 */ 1368, 1322, 1224, 1375, 1375, 1322, 1224, 1224, 1301, 1307, - /* 210 */ 1375, 1310, 1375, 1375, 1375, 1394, 1369, 1394, 1369, 1336, - /* 220 */ 156, 1403, 156, 1407, 1410, 156, 1407, 156, 156, 156, - /* 230 */ 1375, 156, 1388, 1388, 1394, 56, 56, 56, 56, 56, - /* 240 */ 56, 56, 56, 56, 56, 56, 1375, 1394, 632, 632, - /* 250 */ 632, 1252, 1367, 1336, 293, 1295, 1297, 1403, 293, 1300, - /* 260 */ 1375, 1368, 1368, 632, 1255, 1263, 632, 1255, 1263, 632, - /* 270 */ 632, 56, 1266, 1364, 1255, 1268, 1271, 1287, 1092, 1262, - /* 280 */ 1272, 1273, 1298, 543, 1535, 1375, 1407, 293, 293, 1542, - /* 290 */ 1263, 632, 632, 632, 632, 632, 1263, 632, 1409, 293, - /* 300 */ 546, 293, 543, 1490, 1507, 632, 672, 1375, 293, 1582, - /* 310 */ 1394, 2980, 2980, 2980, 2980, 2980, 2980, 2980, 2980, 2980, - /* 320 */ 34, 249, 289, 1028, 398, 57, 578, 50, 722, 769, - /* 330 */ 951, 75, 961, 961, 961, 961, 961, 961, 961, 961, - /* 340 */ 961, 538, 517, 65, 65, 445, 510, 871, 399, 989, - /* 350 */ 918, 974, 574, 898, 492, 492, 1087, 251, 879, 1087, - /* 360 */ 1087, 1087, 1170, 997, 752, 1167, 1157, 1072, 1196, 1094, - /* 370 */ 1100, 1109, 1116, 804, 1200, 1204, 1214, 1221, 1023, 1164, - /* 380 */ 1184, 615, 1187, 1188, 1193, 1103, 1195, 1212, 1142, 1194, - /* 390 */ 1202, 1215, 1216, 1222, 1226, 1270, 1229, 1182, 1230, 1211, - /* 400 */ 1232, 1235, 1236, 1237, 1238, 1239, 1163, 741, 1228, 1264, - /* 410 */ 1265, 1253, 1261, 1631, 1633, 1636, 1596, 1639, 1605, 1428, - /* 420 */ 1607, 1608, 1611, 1436, 1648, 1615, 1616, 1439, 1653, 1441, - /* 430 */ 1655, 1621, 1670, 1637, 1660, 1623, 1453, 1662, 1467, 1664, - /* 440 */ 1469, 1470, 1476, 1480, 1671, 1674, 1681, 1501, 1503, 1688, - /* 450 */ 1690, 1644, 1692, 1695, 1697, 1657, 1700, 1701, 1702, 1703, - /* 460 */ 1704, 1705, 1706, 1707, 1551, 1676, 1712, 1556, 1714, 1715, - /* 470 */ 1716, 1717, 1718, 1723, 1725, 1726, 1727, 1728, 1729, 1731, - /* 480 */ 1732, 1734, 1735, 1742, 1678, 1721, 1744, 1745, 1747, 1748, - /* 490 */ 1749, 1730, 1751, 1753, 1757, 1620, 1768, 1770, 1750, 1736, - /* 500 */ 1752, 1737, 1771, 1733, 1740, 1777, 1743, 1778, 1754, 1779, - /* 510 */ 1786, 1755, 1738, 1767, 1788, 1758, 1759, 1772, 1789, 1761, - /* 520 */ 1760, 1774, 1791, 1784, 1814, 1776, 1781, 1785, 1780, 1790, - /* 530 */ 1807, 1792, 1823, 1793, 1787, 1825, 1828, 1830, 1796, 1603, - /* 540 */ 1831, 1832, 1836, 1773, 1838, 1840, 1808, 1795, 1803, 1845, - /* 550 */ 1812, 1800, 1811, 1851, 1817, 1804, 1815, 1855, 1822, 1809, - /* 560 */ 1821, 1868, 1870, 1872, 1873, 1763, 1766, 1842, 1856, 1879, - /* 570 */ 1849, 1850, 1859, 1860, 1861, 1862, 1864, 1848, 1858, 1866, - /* 580 */ 1869, 1883, 1871, 1907, 1886, 1909, 1889, 1865, 1913, 1893, - /* 590 */ 1882, 1918, 1885, 1921, 1887, 1923, 1902, 1905, 1891, 1896, - /* 600 */ 1897, 1833, 1826, 1933, 1764, 1839, 1765, 1908, 1924, 1948, - /* 610 */ 1775, 1927, 1794, 1797, 1958, 1959, 1801, 1798, 1960, 1929, - /* 620 */ 1713, 1863, 1874, 1876, 1914, 1867, 1928, 1890, 1878, 1935, - /* 630 */ 1936, 1880, 1888, 1892, 1895, 1884, 1947, 1942, 1946, 1898, - /* 640 */ 1951, 1746, 1900, 1901, 1989, 1954, 1762, 1964, 1966, 1979, - /* 650 */ 1981, 1984, 1985, 1931, 1932, 1983, 1783, 1992, 1990, 2039, - /* 660 */ 2040, 2041, 2042, 1941, 2005, 1780, 1999, 1949, 1950, 1952, - /* 670 */ 1961, 1963, 1875, 1967, 2049, 2011, 1899, 1970, 1944, 1780, - /* 680 */ 2007, 2016, 1974, 1819, 1975, 2064, 2054, 1903, 1976, 1977, - /* 690 */ 1982, 1978, 1991, 1980, 2015, 1994, 1995, 2044, 1996, 2076, - /* 700 */ 1904, 1998, 1988, 2000, 2067, 2071, 2008, 2014, 2084, 2019, - /* 710 */ 2021, 2087, 2023, 2024, 2093, 2027, 2029, 2098, 2032, 2018, - /* 720 */ 2022, 2031, 2034, 2033, 2094, 2037, 2105, 2051, 2094, 2094, - /* 730 */ 2137, 2092, 2096, 2126, 2129, 2130, 2131, 2132, 2133, 2135, - /* 740 */ 2139, 2099, 2079, 2128, 2141, 2143, 2144, 2159, 2147, 2154, - /* 750 */ 2156, 2118, 1848, 2158, 1858, 2161, 2162, 2163, 2164, 2172, - /* 760 */ 2165, 2201, 2167, 2155, 2168, 2208, 2182, 2169, 2180, 2220, - /* 770 */ 2187, 2174, 2185, 2221, 2191, 2178, 2189, 2229, 2195, 2196, - /* 780 */ 2233, 2212, 2214, 2215, 2216, 2218, 2222, + /* 0 */ 1005, 0, 104, 0, 337, 337, 337, 337, 337, 337, + /* 10 */ 337, 337, 337, 337, 337, 337, 441, 673, 673, 777, + /* 20 */ 673, 673, 673, 673, 673, 673, 673, 673, 673, 673, + /* 30 */ 673, 673, 673, 673, 673, 673, 673, 673, 673, 673, + /* 40 */ 673, 673, 673, 673, 673, 673, 673, 673, 673, 673, + /* 50 */ 673, 60, 259, 393, 629, 111, 739, 111, 629, 629, + /* 60 */ 111, 1341, 111, 1341, 1341, 66, 111, 68, 781, 101, + /* 70 */ 101, 781, 13, 13, 113, 294, 23, 23, 101, 101, + /* 80 */ 101, 101, 101, 101, 101, 101, 101, 101, 158, 101, + /* 90 */ 101, 211, 68, 101, 101, 285, 68, 101, 158, 101, + /* 100 */ 158, 68, 101, 101, 68, 101, 68, 68, 68, 101, + /* 110 */ 284, 1004, 15, 15, 303, 462, 1302, 1302, 1302, 1302, + /* 120 */ 1302, 1302, 1302, 1302, 1302, 1302, 1302, 1302, 1302, 1302, + /* 130 */ 1302, 1302, 1302, 1302, 1302, 1072, 18, 113, 294, 802, + /* 140 */ 802, 184, 346, 346, 346, 705, 312, 312, 184, 211, + /* 150 */ 254, 260, 68, 534, 68, 534, 534, 836, 619, 28, + /* 160 */ 28, 28, 28, 28, 28, 28, 28, 1987, 757, 261, + /* 170 */ 580, 613, 508, 49, 123, 343, 343, 526, 354, 1007, + /* 180 */ 299, 1079, 1192, 418, 398, 1085, 916, 939, 785, 916, + /* 190 */ 1177, 325, 290, 1162, 1369, 1407, 1223, 211, 1407, 211, + /* 200 */ 1242, 1435, 1416, 1450, 1435, 1416, 1306, 1460, 1435, 1460, + /* 210 */ 1416, 1306, 1306, 1389, 1393, 1460, 1402, 1460, 1460, 1460, + /* 220 */ 1490, 1468, 1490, 1468, 1407, 211, 1503, 211, 1508, 1511, + /* 230 */ 211, 1508, 211, 211, 211, 1460, 211, 1490, 68, 68, + /* 240 */ 68, 68, 68, 68, 68, 68, 68, 68, 68, 1460, + /* 250 */ 1490, 534, 534, 534, 1343, 1452, 1407, 284, 1360, 1366, + /* 260 */ 1503, 284, 1394, 1162, 1460, 1450, 1450, 534, 1330, 1339, + /* 270 */ 534, 1330, 1339, 534, 534, 68, 1326, 1439, 1330, 1344, + /* 280 */ 1350, 1364, 1162, 1345, 1354, 1352, 1373, 1435, 1627, 1508, + /* 290 */ 284, 284, 1631, 1339, 534, 534, 534, 534, 534, 1339, + /* 300 */ 534, 1505, 284, 836, 284, 1435, 1591, 1593, 534, 619, + /* 310 */ 1460, 284, 1667, 1490, 2858, 2858, 2858, 2858, 2858, 2858, + /* 320 */ 2858, 2858, 2858, 1002, 1074, 225, 32, 736, 750, 860, + /* 330 */ 687, 980, 1356, 434, 791, 197, 197, 197, 197, 197, + /* 340 */ 197, 197, 197, 197, 749, 265, 698, 698, 29, 6, + /* 350 */ 34, 307, 830, 562, 963, 902, 514, 282, 282, 1097, + /* 360 */ 728, 971, 1097, 1097, 1097, 983, 1044, 622, 1209, 1198, + /* 370 */ 1131, 1077, 1143, 1149, 1156, 1160, 1236, 1256, 815, 1182, + /* 380 */ 1275, 1071, 1195, 1224, 63, 1232, 1237, 1262, 1152, 1124, + /* 390 */ 1094, 1263, 1277, 1278, 1282, 1285, 1286, 1332, 1291, 1220, + /* 400 */ 1295, 311, 1305, 1315, 1335, 1340, 1349, 1353, 1206, 1270, + /* 410 */ 1288, 1372, 1378, 409, 1351, 1709, 1738, 1745, 1704, 1749, + /* 420 */ 1715, 1535, 1717, 1720, 1722, 1542, 1759, 1725, 1726, 1546, + /* 430 */ 1764, 1549, 1766, 1734, 1770, 1750, 1771, 1739, 1565, 1784, + /* 440 */ 1587, 1788, 1590, 1594, 1600, 1610, 1806, 1807, 1808, 1622, + /* 450 */ 1624, 1811, 1812, 1769, 1813, 1817, 1818, 1774, 1822, 1781, + /* 460 */ 1825, 1826, 1828, 1783, 1831, 1832, 1833, 1835, 1836, 1837, + /* 470 */ 1679, 1805, 1843, 1691, 1852, 1853, 1855, 1857, 1858, 1859, + /* 480 */ 1860, 1861, 1864, 1866, 1888, 1889, 1893, 1894, 1903, 1904, + /* 490 */ 1830, 1878, 1838, 1879, 1881, 1882, 1883, 1884, 1885, 1886, + /* 500 */ 1905, 1906, 1907, 1768, 1909, 1913, 1892, 1875, 1908, 1876, + /* 510 */ 1925, 1824, 1891, 1927, 1870, 1928, 1871, 1935, 1936, 1910, + /* 520 */ 1890, 1898, 1938, 1911, 1895, 1901, 1942, 1914, 1899, 1912, + /* 530 */ 1950, 1917, 1953, 1915, 1916, 1918, 1919, 1920, 1940, 1921, + /* 540 */ 1956, 1923, 1922, 1958, 1960, 1965, 1930, 1790, 1970, 1978, + /* 550 */ 1981, 1924, 1991, 1992, 1939, 1926, 1937, 1994, 1961, 1946, + /* 560 */ 1959, 1997, 1964, 1951, 1962, 2002, 1968, 1955, 1966, 2008, + /* 570 */ 2010, 2011, 2021, 2022, 2023, 1929, 1932, 1990, 2004, 2024, + /* 580 */ 1993, 1996, 1998, 1999, 2000, 2003, 2005, 1983, 1985, 2013, + /* 590 */ 2015, 2017, 2016, 2037, 2030, 2043, 2032, 2006, 2055, 2034, + /* 600 */ 2025, 2058, 2026, 2059, 2027, 2064, 2045, 2049, 2035, 2042, + /* 610 */ 2044, 1976, 1979, 2065, 1934, 1980, 1887, 2046, 2066, 2086, + /* 620 */ 1896, 2071, 1941, 1933, 2087, 2095, 1943, 1931, 1944, 1945, + /* 630 */ 2093, 2053, 1834, 2001, 1995, 2009, 2052, 2019, 2060, 2012, + /* 640 */ 2014, 2057, 2070, 2020, 2018, 2028, 2029, 2031, 2073, 2063, + /* 650 */ 2079, 2039, 2083, 1869, 2033, 2036, 2099, 2084, 1874, 2108, + /* 660 */ 2115, 2117, 2118, 2119, 2120, 2061, 2069, 2111, 1872, 2124, + /* 670 */ 2112, 2161, 2162, 2080, 2125, 1919, 2116, 2081, 2085, 2088, + /* 680 */ 2082, 2090, 2041, 2092, 2174, 2148, 2047, 2094, 2075, 1919, + /* 690 */ 2143, 2152, 2097, 1954, 2100, 2198, 2180, 1986, 2101, 2121, + /* 700 */ 2126, 2122, 2127, 2128, 2156, 2131, 2132, 2175, 2133, 2199, + /* 710 */ 2038, 2135, 2109, 2136, 2189, 2192, 2138, 2141, 2201, 2154, + /* 720 */ 2142, 2204, 2155, 2151, 2224, 2157, 2158, 2226, 2163, 2160, + /* 730 */ 2230, 2168, 2153, 2159, 2164, 2165, 2177, 2238, 2183, 2248, + /* 740 */ 2184, 2238, 2238, 2263, 2219, 2223, 2257, 2259, 2261, 2262, + /* 750 */ 2264, 2265, 2266, 2267, 2228, 2208, 2254, 2272, 2273, 2282, + /* 760 */ 2296, 2284, 2285, 2286, 2247, 1983, 2288, 1985, 2289, 2291, + /* 770 */ 2292, 2293, 2310, 2298, 2342, 2308, 2295, 2306, 2346, 2312, + /* 780 */ 2300, 2313, 2350, 2316, 2305, 2317, 2355, 2322, 2314, 2320, + /* 790 */ 2360, 2326, 2327, 2364, 2344, 2348, 2349, 2351, 2353, 2352, }; -#define YY_REDUCE_COUNT (319) -#define YY_REDUCE_MIN (-445) -#define YY_REDUCE_MAX (2545) +#define YY_REDUCE_COUNT (322) +#define YY_REDUCE_MIN (-404) +#define YY_REDUCE_MAX (2418) static const short yy_reduce_ofst[] = { - /* 0 */ 96, -94, 175, 329, 377, 597, 685, 823, 1086, 1148, - /* 10 */ 1017, 1174, 1248, 1309, 1335, 1371, -338, 150, 465, 1435, - /* 20 */ 1459, 1523, 1581, 1597, 1658, 1684, 1720, 1756, 1782, 1857, - /* 30 */ 1877, 1945, 1962, 2010, 2030, 2060, 2078, 2108, 2138, 2186, - /* 40 */ 2206, 2284, 2301, 2349, 2369, 2399, 2417, 2447, 2477, 2525, - /* 50 */ 2545, 393, 40, -48, -318, -206, 359, 592, 716, 54, - /* 60 */ 187, 799, -445, -85, 514, 426, 428, -366, 43, -345, - /* 70 */ -177, -132, -258, -127, 247, -28, -324, 47, 174, 381, - /* 80 */ 385, 407, 472, 485, 524, -155, 540, 585, 118, -81, - /* 90 */ 633, 642, 103, 645, -78, 647, 129, 692, 138, 108, - /* 100 */ 704, 718, -27, 728, 470, 493, 579, 731, -315, 133, - /* 110 */ -438, -438, -256, -15, 144, 213, 320, 404, 434, 520, - /* 120 */ 523, 547, 598, 620, 623, 711, 721, 724, 733, 780, - /* 130 */ 787, 790, 796, -368, -83, -64, 594, 513, -83, 209, - /* 140 */ 317, 207, 725, 742, 727, 664, 791, 812, 119, -364, - /* 150 */ -355, 273, 650, 140, 657, 807, 600, 813, 458, 504, - /* 160 */ 573, 639, 910, 913, 916, 910, 882, 930, 956, 901, - /* 170 */ 821, 833, 953, 843, 944, 946, 935, 935, 965, 920, - /* 180 */ 978, 980, 945, 933, 888, 888, 872, 888, 897, 889, - /* 190 */ 935, 929, 932, 942, 940, 1004, 947, 1006, 960, 995, - /* 200 */ 993, 999, 1002, 1049, 1051, 1003, 1007, 1008, 1044, 1052, - /* 210 */ 1061, 1054, 1062, 1065, 1066, 1075, 1076, 1079, 1077, 1001, - /* 220 */ 1067, 1037, 1073, 1080, 1024, 1078, 1083, 1082, 1084, 1085, - /* 230 */ 1088, 1095, 1091, 1101, 1112, 1089, 1093, 1096, 1097, 1102, - /* 240 */ 1104, 1105, 1106, 1107, 1110, 1113, 1108, 1114, 1070, 1118, - /* 250 */ 1119, 1041, 1059, 1063, 1134, 1081, 1098, 1120, 1138, 1117, - /* 260 */ 1152, 1111, 1122, 1137, 1064, 1128, 1147, 1071, 1136, 1149, - /* 270 */ 1154, 935, 1074, 1121, 1126, 1099, 1123, 1115, 1125, 1069, - /* 280 */ 1124, 1133, 888, 1205, 1130, 1210, 1209, 1206, 1213, 1165, - /* 290 */ 1162, 1180, 1181, 1183, 1186, 1197, 1169, 1203, 1192, 1244, - /* 300 */ 1240, 1247, 1269, 1166, 1233, 1227, 1251, 1274, 1275, 1281, - /* 310 */ 1284, 1217, 1218, 1241, 1242, 1256, 1258, 1260, 1276, 1296, + /* 0 */ -296, -340, -133, -26, 88, 310, 540, 850, 881, 911, + /* 10 */ 208, 1001, 1032, 1069, 1158, 1207, 1233, 118, 1284, 1297, + /* 20 */ 1346, 1363, 1436, 1412, 1463, 1483, 426, 1550, 1580, 1639, + /* 30 */ 1712, 1743, 1778, 1815, 1840, 1877, 1902, 1974, 1989, 2007, + /* 40 */ 2054, 2076, 2123, 2182, 2195, 2212, 2244, 2311, 2331, 2401, + /* 50 */ 2418, -260, -25, 239, 149, -404, 246, 479, -28, 348, + /* 60 */ 643, -365, -294, -331, 671, -98, 24, 252, -390, -345, + /* 70 */ -265, -279, -72, 74, -318, 227, -342, 112, -232, 26, + /* 80 */ 192, 263, 341, 374, 383, 400, 424, 436, 286, 592, + /* 90 */ 595, 674, 116, 601, 675, 36, 323, 679, 391, 752, + /* 100 */ 637, -63, 765, 776, 555, 779, 429, 672, 669, 822, + /* 110 */ 155, -316, -313, -313, -259, -140, -199, -188, 157, 269, + /* 120 */ 427, 529, 548, 553, 656, 723, 726, 796, 803, 804, + /* 130 */ 809, 844, 845, 855, 867, -217, -242, -15, 438, 523, + /* 140 */ 680, 616, -242, 167, 199, -250, 366, 696, 778, 825, + /* 150 */ -375, -47, 93, 575, 818, 811, 823, 841, 864, -371, + /* 160 */ -328, 356, 491, 515, 579, 681, 515, 711, 847, 956, + /* 170 */ 910, 827, 829, 1006, 888, 1008, 1020, 1009, 986, 1009, + /* 180 */ 1046, 997, 1055, 1054, 1018, 1011, 954, 954, 937, 954, + /* 190 */ 965, 959, 1009, 1010, 998, 1021, 1023, 1073, 1025, 1091, + /* 200 */ 1040, 1105, 1075, 1076, 1121, 1081, 1078, 1134, 1135, 1137, + /* 210 */ 1087, 1092, 1093, 1128, 1132, 1147, 1146, 1161, 1163, 1164, + /* 220 */ 1169, 1170, 1175, 1174, 1101, 1165, 1136, 1168, 1178, 1120, + /* 230 */ 1176, 1184, 1179, 1180, 1181, 1191, 1183, 1200, 1166, 1167, + /* 240 */ 1171, 1173, 1187, 1188, 1189, 1190, 1193, 1194, 1196, 1197, + /* 250 */ 1201, 1172, 1185, 1205, 1129, 1133, 1138, 1203, 1141, 1151, + /* 260 */ 1208, 1214, 1199, 1202, 1238, 1204, 1210, 1211, 1139, 1212, + /* 270 */ 1219, 1148, 1213, 1222, 1225, 1009, 1150, 1153, 1215, 1186, + /* 280 */ 1216, 1221, 1217, 1144, 1218, 1226, 954, 1294, 1228, 1298, + /* 290 */ 1292, 1299, 1250, 1249, 1273, 1287, 1289, 1293, 1300, 1255, + /* 300 */ 1301, 1280, 1329, 1314, 1336, 1348, 1246, 1317, 1310, 1333, + /* 310 */ 1357, 1355, 1362, 1365, 1307, 1313, 1303, 1308, 1367, 1368, + /* 320 */ 1371, 1370, 1388, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 10 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 20 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 30 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 40 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 50 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 60 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 70 */ 1753, 1753, 1753, 1753, 2030, 1753, 1753, 1753, 1753, 1753, - /* 80 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1839, 1753, - /* 90 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 100 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1837, 2023, - /* 110 */ 2248, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 120 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 130 */ 1753, 1753, 1753, 1753, 2260, 1753, 1753, 1753, 2260, 2260, - /* 140 */ 2260, 1837, 2220, 2220, 1753, 1753, 1753, 1753, 1839, 2090, - /* 150 */ 1753, 1753, 1753, 1753, 1753, 1753, 1958, 1753, 1753, 1753, - /* 160 */ 1753, 1753, 1982, 1753, 1753, 1753, 2082, 1753, 1753, 2285, - /* 170 */ 2341, 1753, 1753, 2288, 1753, 1753, 1753, 1753, 1753, 2035, - /* 180 */ 1753, 1753, 1912, 2275, 2252, 2266, 2325, 2253, 2250, 2269, - /* 190 */ 1753, 2279, 1753, 1753, 2104, 1839, 1753, 1839, 2069, 2028, - /* 200 */ 1753, 2028, 2025, 1753, 1753, 2028, 2025, 2025, 1901, 1897, - /* 210 */ 1753, 1895, 1753, 1753, 1753, 1753, 1800, 1753, 1800, 1753, - /* 220 */ 1839, 1753, 1839, 1753, 1753, 1839, 1753, 1839, 1839, 1839, - /* 230 */ 1753, 1839, 1814, 1814, 1753, 1753, 1753, 1753, 1753, 1753, - /* 240 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 250 */ 1753, 2102, 2088, 1753, 1837, 2080, 2078, 1753, 1837, 2076, - /* 260 */ 1753, 1753, 1753, 1753, 2296, 2294, 1753, 2296, 2294, 1753, - /* 270 */ 1753, 1753, 2310, 2306, 2296, 2314, 2312, 2281, 2279, 2344, - /* 280 */ 2331, 2327, 2266, 1753, 1753, 1753, 1753, 1837, 1837, 1753, - /* 290 */ 2294, 1753, 1753, 1753, 1753, 1753, 2294, 1753, 1753, 1837, - /* 300 */ 1753, 1837, 1753, 1753, 1928, 1753, 1753, 1753, 1837, 1785, - /* 310 */ 1753, 2071, 2093, 2053, 2053, 1961, 1961, 1961, 1840, 1758, - /* 320 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 330 */ 1753, 1753, 2309, 2308, 2175, 1753, 2224, 2223, 2222, 2213, - /* 340 */ 2174, 1924, 1753, 2173, 2172, 1753, 1753, 1753, 1753, 1753, - /* 350 */ 1753, 1753, 1753, 1753, 2044, 2043, 2166, 1753, 1753, 2167, - /* 360 */ 2165, 2164, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 370 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 380 */ 1753, 1753, 1753, 1753, 1753, 1753, 2328, 2332, 1753, 1753, - /* 390 */ 1753, 1753, 1753, 1753, 1753, 2249, 1753, 1753, 1753, 2148, - /* 400 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 410 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 420 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 430 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 440 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 450 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 460 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 470 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 480 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 490 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 500 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 510 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 520 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1790, 2153, 1753, - /* 530 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 540 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 550 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 560 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 570 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1878, 1877, 1753, - /* 580 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 590 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 600 */ 1753, 2157, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 610 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 2324, 2282, - /* 620 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 630 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 2148, 1753, - /* 640 */ 2307, 1753, 1753, 2322, 1753, 2326, 1753, 1753, 1753, 1753, - /* 650 */ 1753, 1753, 1753, 2259, 2255, 1753, 1753, 2251, 1753, 1753, - /* 660 */ 1753, 1753, 1753, 1753, 1753, 2156, 1753, 1753, 1753, 1753, - /* 670 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 2147, - /* 680 */ 1753, 2210, 1753, 1753, 1753, 2244, 1753, 1753, 2195, 1753, - /* 690 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 2157, 1753, - /* 700 */ 2160, 1753, 1753, 1753, 1753, 1753, 1955, 1753, 1753, 1753, - /* 710 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1940, - /* 720 */ 1938, 1937, 1936, 1753, 1968, 1753, 1753, 1753, 1964, 1963, - /* 730 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 740 */ 1753, 1753, 1753, 1858, 1753, 1753, 1753, 1753, 1753, 1753, - /* 750 */ 1753, 1753, 1850, 1753, 1849, 1753, 1753, 1753, 1753, 1753, - /* 760 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 770 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, 1753, - /* 780 */ 1753, 1753, 1753, 1753, 1753, 1753, 1753, + /* 0 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 10 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 20 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 30 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 40 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 50 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 60 */ 2091, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 70 */ 1780, 1780, 1780, 1780, 2064, 1780, 1780, 1780, 1780, 1780, + /* 80 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 90 */ 1780, 1869, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 100 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 110 */ 1867, 2057, 2283, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 120 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 130 */ 1780, 1780, 1780, 1780, 1780, 1780, 2295, 1780, 1780, 1843, + /* 140 */ 1843, 1780, 2295, 2295, 2295, 1867, 2255, 2255, 1780, 1869, + /* 150 */ 2125, 1780, 1780, 1780, 1780, 1780, 1780, 1989, 1780, 1780, + /* 160 */ 1780, 1780, 1780, 2013, 1780, 1780, 1780, 2117, 1780, 1780, + /* 170 */ 2320, 2377, 1780, 1780, 2323, 1780, 1780, 1780, 1780, 1780, + /* 180 */ 1780, 2069, 1780, 1780, 1942, 2310, 2287, 2301, 2361, 2288, + /* 190 */ 2285, 2304, 1780, 2314, 1780, 1780, 2139, 1869, 1780, 1869, + /* 200 */ 2104, 1780, 2062, 1780, 1780, 2062, 2059, 1780, 1780, 1780, + /* 210 */ 2062, 2059, 2059, 1931, 1927, 1780, 1925, 1780, 1780, 1780, + /* 220 */ 1780, 1827, 1780, 1827, 1780, 1869, 1780, 1869, 1780, 1780, + /* 230 */ 1869, 1780, 1869, 1869, 1869, 1780, 1869, 1780, 1780, 1780, + /* 240 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 250 */ 1780, 1780, 1780, 1780, 2137, 2123, 1780, 1867, 2115, 2113, + /* 260 */ 1780, 1867, 2111, 2314, 1780, 1780, 1780, 1780, 2331, 2329, + /* 270 */ 1780, 2331, 2329, 1780, 1780, 1780, 2345, 2341, 2331, 2350, + /* 280 */ 2347, 2316, 2314, 2380, 2367, 2363, 2301, 1780, 1780, 1780, + /* 290 */ 1867, 1867, 1780, 2329, 1780, 1780, 1780, 1780, 1780, 2329, + /* 300 */ 1780, 1780, 1867, 1780, 1867, 1780, 1780, 1958, 1780, 1780, + /* 310 */ 1780, 1867, 1812, 1780, 2106, 2128, 2087, 2087, 1992, 1992, + /* 320 */ 1992, 1870, 1785, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 330 */ 1780, 1780, 1780, 1780, 1780, 2344, 2343, 2210, 1780, 2259, + /* 340 */ 2258, 2257, 2248, 2209, 1954, 1780, 2208, 2207, 1780, 1780, + /* 350 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 2078, 2077, 2201, + /* 360 */ 1780, 1780, 2202, 2200, 2199, 1780, 1780, 1780, 1780, 1780, + /* 370 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 380 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 2364, + /* 390 */ 2368, 1780, 1780, 1780, 1780, 1780, 1780, 2284, 1780, 1780, + /* 400 */ 1780, 2183, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 410 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 420 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 430 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 440 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 450 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 460 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 470 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 480 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 490 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 500 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 510 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 520 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 530 */ 1780, 1780, 1780, 1780, 1780, 1817, 2188, 1780, 1780, 1780, + /* 540 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 550 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 560 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 570 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 580 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1908, 1907, 1780, + /* 590 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 600 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 610 */ 1780, 2192, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 620 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 630 */ 2360, 2317, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 640 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 650 */ 2183, 1780, 2342, 1780, 1780, 2358, 1780, 2362, 1780, 1780, + /* 660 */ 1780, 1780, 1780, 1780, 1780, 2294, 2290, 1780, 1780, 2286, + /* 670 */ 1780, 1780, 1780, 1780, 1780, 2191, 1780, 1780, 1780, 1780, + /* 680 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 2182, + /* 690 */ 1780, 2245, 1780, 1780, 1780, 2279, 1780, 1780, 2230, 1780, + /* 700 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 2192, 1780, + /* 710 */ 2195, 1780, 1780, 1780, 1780, 1780, 1986, 1780, 1780, 1780, + /* 720 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 730 */ 1780, 1780, 1970, 1968, 1967, 1966, 1780, 1999, 1780, 1780, + /* 740 */ 1780, 1995, 1994, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 750 */ 1780, 1780, 1780, 1780, 1780, 1780, 1888, 1780, 1780, 1780, + /* 760 */ 1780, 1780, 1780, 1780, 1780, 1880, 1780, 1879, 1780, 1780, + /* 770 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 780 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 790 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1130,6 +1107,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* RESTORE => nothing */ 0, /* NK_IPTOKEN => nothing */ 0, /* FORCE => nothing */ + 0, /* UNSAFE => nothing */ 0, /* LOCAL => nothing */ 0, /* QNODE => nothing */ 0, /* BNODE => nothing */ @@ -1175,7 +1153,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* MAX_SPEED => nothing */ 0, /* START => nothing */ 0, /* TIMESTAMP => nothing */ - 284, /* END => ABORT */ + 287, /* END => ABORT */ 0, /* TABLE => nothing */ 0, /* NK_LP => nothing */ 0, /* NK_RP => nothing */ @@ -1205,6 +1183,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* MEDIUMBLOB => nothing */ 0, /* BLOB => nothing */ 0, /* VARBINARY => nothing */ + 0, /* GEOMETRY => nothing */ 0, /* DECIMAL => nothing */ 0, /* COMMENT => nothing */ 0, /* MAX_DELAY => nothing */ @@ -1251,8 +1230,9 @@ static const YYCODETYPE yyFallback[] = { 0, /* INTERVAL => nothing */ 0, /* COUNT => nothing */ 0, /* LAST_ROW => nothing */ - 0, /* TOPIC => nothing */ 0, /* META => nothing */ + 0, /* ONLY => nothing */ + 0, /* TOPIC => nothing */ 0, /* CONSUMER => nothing */ 0, /* GROUP => nothing */ 0, /* DESC => nothing */ @@ -1360,56 +1340,56 @@ static const YYCODETYPE yyFallback[] = { 0, /* ASC => nothing */ 0, /* NULLS => nothing */ 0, /* ABORT => nothing */ - 284, /* AFTER => ABORT */ - 284, /* ATTACH => ABORT */ - 284, /* BEFORE => ABORT */ - 284, /* BEGIN => ABORT */ - 284, /* BITAND => ABORT */ - 284, /* BITNOT => ABORT */ - 284, /* BITOR => ABORT */ - 284, /* BLOCKS => ABORT */ - 284, /* CHANGE => ABORT */ - 284, /* COMMA => ABORT */ - 284, /* CONCAT => ABORT */ - 284, /* CONFLICT => ABORT */ - 284, /* COPY => ABORT */ - 284, /* DEFERRED => ABORT */ - 284, /* DELIMITERS => ABORT */ - 284, /* DETACH => ABORT */ - 284, /* DIVIDE => ABORT */ - 284, /* DOT => ABORT */ - 284, /* EACH => ABORT */ - 284, /* FAIL => ABORT */ - 284, /* FILE => ABORT */ - 284, /* FOR => ABORT */ - 284, /* GLOB => ABORT */ - 284, /* ID => ABORT */ - 284, /* IMMEDIATE => ABORT */ - 284, /* IMPORT => ABORT */ - 284, /* INITIALLY => ABORT */ - 284, /* INSTEAD => ABORT */ - 284, /* ISNULL => ABORT */ - 284, /* KEY => ABORT */ - 284, /* MODULES => ABORT */ - 284, /* NK_BITNOT => ABORT */ - 284, /* NK_SEMI => ABORT */ - 284, /* NOTNULL => ABORT */ - 284, /* OF => ABORT */ - 284, /* PLUS => ABORT */ - 284, /* PRIVILEGE => ABORT */ - 284, /* RAISE => ABORT */ - 284, /* RESTRICT => ABORT */ - 284, /* ROW => ABORT */ - 284, /* SEMI => ABORT */ - 284, /* STAR => ABORT */ - 284, /* STATEMENT => ABORT */ - 284, /* STRICT => ABORT */ - 284, /* STRING => ABORT */ - 284, /* TIMES => ABORT */ - 284, /* VALUES => ABORT */ - 284, /* VARIABLE => ABORT */ - 284, /* VIEW => ABORT */ - 284, /* WAL => ABORT */ + 287, /* AFTER => ABORT */ + 287, /* ATTACH => ABORT */ + 287, /* BEFORE => ABORT */ + 287, /* BEGIN => ABORT */ + 287, /* BITAND => ABORT */ + 287, /* BITNOT => ABORT */ + 287, /* BITOR => ABORT */ + 287, /* BLOCKS => ABORT */ + 287, /* CHANGE => ABORT */ + 287, /* COMMA => ABORT */ + 287, /* CONCAT => ABORT */ + 287, /* CONFLICT => ABORT */ + 287, /* COPY => ABORT */ + 287, /* DEFERRED => ABORT */ + 287, /* DELIMITERS => ABORT */ + 287, /* DETACH => ABORT */ + 287, /* DIVIDE => ABORT */ + 287, /* DOT => ABORT */ + 287, /* EACH => ABORT */ + 287, /* FAIL => ABORT */ + 287, /* FILE => ABORT */ + 287, /* FOR => ABORT */ + 287, /* GLOB => ABORT */ + 287, /* ID => ABORT */ + 287, /* IMMEDIATE => ABORT */ + 287, /* IMPORT => ABORT */ + 287, /* INITIALLY => ABORT */ + 287, /* INSTEAD => ABORT */ + 287, /* ISNULL => ABORT */ + 287, /* KEY => ABORT */ + 287, /* MODULES => ABORT */ + 287, /* NK_BITNOT => ABORT */ + 287, /* NK_SEMI => ABORT */ + 287, /* NOTNULL => ABORT */ + 287, /* OF => ABORT */ + 287, /* PLUS => ABORT */ + 287, /* PRIVILEGE => ABORT */ + 287, /* RAISE => ABORT */ + 287, /* RESTRICT => ABORT */ + 287, /* ROW => ABORT */ + 287, /* SEMI => ABORT */ + 287, /* STAR => ABORT */ + 287, /* STATEMENT => ABORT */ + 287, /* STRICT => ABORT */ + 287, /* STRING => ABORT */ + 287, /* TIMES => ABORT */ + 287, /* VALUES => ABORT */ + 287, /* VARIABLE => ABORT */ + 287, /* VIEW => ABORT */ + 287, /* WAL => ABORT */ }; #endif /* YYFALLBACK */ @@ -1552,433 +1532,438 @@ static const char *const yyTokenName[] = { /* 52 */ "RESTORE", /* 53 */ "NK_IPTOKEN", /* 54 */ "FORCE", - /* 55 */ "LOCAL", - /* 56 */ "QNODE", - /* 57 */ "BNODE", - /* 58 */ "SNODE", - /* 59 */ "MNODE", - /* 60 */ "VNODE", - /* 61 */ "DATABASE", - /* 62 */ "USE", - /* 63 */ "FLUSH", - /* 64 */ "TRIM", - /* 65 */ "COMPACT", - /* 66 */ "IF", - /* 67 */ "NOT", - /* 68 */ "EXISTS", - /* 69 */ "BUFFER", - /* 70 */ "CACHEMODEL", - /* 71 */ "CACHESIZE", - /* 72 */ "COMP", - /* 73 */ "DURATION", - /* 74 */ "NK_VARIABLE", - /* 75 */ "MAXROWS", - /* 76 */ "MINROWS", - /* 77 */ "KEEP", - /* 78 */ "PAGES", - /* 79 */ "PAGESIZE", - /* 80 */ "TSDB_PAGESIZE", - /* 81 */ "PRECISION", - /* 82 */ "REPLICA", - /* 83 */ "VGROUPS", - /* 84 */ "SINGLE_STABLE", - /* 85 */ "RETENTIONS", - /* 86 */ "SCHEMALESS", - /* 87 */ "WAL_LEVEL", - /* 88 */ "WAL_FSYNC_PERIOD", - /* 89 */ "WAL_RETENTION_PERIOD", - /* 90 */ "WAL_RETENTION_SIZE", - /* 91 */ "WAL_ROLL_PERIOD", - /* 92 */ "WAL_SEGMENT_SIZE", - /* 93 */ "STT_TRIGGER", - /* 94 */ "TABLE_PREFIX", - /* 95 */ "TABLE_SUFFIX", - /* 96 */ "NK_COLON", - /* 97 */ "MAX_SPEED", - /* 98 */ "START", - /* 99 */ "TIMESTAMP", - /* 100 */ "END", - /* 101 */ "TABLE", - /* 102 */ "NK_LP", - /* 103 */ "NK_RP", - /* 104 */ "STABLE", - /* 105 */ "ADD", - /* 106 */ "COLUMN", - /* 107 */ "MODIFY", - /* 108 */ "RENAME", - /* 109 */ "TAG", - /* 110 */ "SET", - /* 111 */ "NK_EQ", - /* 112 */ "USING", - /* 113 */ "TAGS", - /* 114 */ "BOOL", - /* 115 */ "TINYINT", - /* 116 */ "SMALLINT", - /* 117 */ "INT", - /* 118 */ "INTEGER", - /* 119 */ "BIGINT", - /* 120 */ "FLOAT", - /* 121 */ "DOUBLE", - /* 122 */ "BINARY", - /* 123 */ "NCHAR", - /* 124 */ "UNSIGNED", - /* 125 */ "JSON", - /* 126 */ "VARCHAR", - /* 127 */ "MEDIUMBLOB", - /* 128 */ "BLOB", - /* 129 */ "VARBINARY", - /* 130 */ "DECIMAL", - /* 131 */ "COMMENT", - /* 132 */ "MAX_DELAY", - /* 133 */ "WATERMARK", - /* 134 */ "ROLLUP", - /* 135 */ "TTL", - /* 136 */ "SMA", - /* 137 */ "DELETE_MARK", - /* 138 */ "FIRST", - /* 139 */ "LAST", - /* 140 */ "SHOW", - /* 141 */ "PRIVILEGES", - /* 142 */ "DATABASES", - /* 143 */ "TABLES", - /* 144 */ "STABLES", - /* 145 */ "MNODES", - /* 146 */ "QNODES", - /* 147 */ "FUNCTIONS", - /* 148 */ "INDEXES", - /* 149 */ "ACCOUNTS", - /* 150 */ "APPS", - /* 151 */ "CONNECTIONS", - /* 152 */ "LICENCES", - /* 153 */ "GRANTS", - /* 154 */ "QUERIES", - /* 155 */ "SCORES", - /* 156 */ "TOPICS", - /* 157 */ "VARIABLES", - /* 158 */ "CLUSTER", - /* 159 */ "BNODES", - /* 160 */ "SNODES", - /* 161 */ "TRANSACTIONS", - /* 162 */ "DISTRIBUTED", - /* 163 */ "CONSUMERS", - /* 164 */ "SUBSCRIPTIONS", - /* 165 */ "VNODES", - /* 166 */ "ALIVE", - /* 167 */ "LIKE", - /* 168 */ "TBNAME", - /* 169 */ "QTAGS", - /* 170 */ "AS", - /* 171 */ "INDEX", - /* 172 */ "FUNCTION", - /* 173 */ "INTERVAL", - /* 174 */ "COUNT", - /* 175 */ "LAST_ROW", - /* 176 */ "TOPIC", - /* 177 */ "META", - /* 178 */ "CONSUMER", - /* 179 */ "GROUP", - /* 180 */ "DESC", - /* 181 */ "DESCRIBE", - /* 182 */ "RESET", - /* 183 */ "QUERY", - /* 184 */ "CACHE", - /* 185 */ "EXPLAIN", - /* 186 */ "ANALYZE", - /* 187 */ "VERBOSE", - /* 188 */ "NK_BOOL", - /* 189 */ "RATIO", - /* 190 */ "NK_FLOAT", - /* 191 */ "OUTPUTTYPE", - /* 192 */ "AGGREGATE", - /* 193 */ "BUFSIZE", - /* 194 */ "LANGUAGE", - /* 195 */ "REPLACE", - /* 196 */ "STREAM", - /* 197 */ "INTO", - /* 198 */ "PAUSE", - /* 199 */ "RESUME", - /* 200 */ "TRIGGER", - /* 201 */ "AT_ONCE", - /* 202 */ "WINDOW_CLOSE", - /* 203 */ "IGNORE", - /* 204 */ "EXPIRED", - /* 205 */ "FILL_HISTORY", - /* 206 */ "UPDATE", - /* 207 */ "SUBTABLE", - /* 208 */ "UNTREATED", - /* 209 */ "KILL", - /* 210 */ "CONNECTION", - /* 211 */ "TRANSACTION", - /* 212 */ "BALANCE", - /* 213 */ "VGROUP", - /* 214 */ "LEADER", - /* 215 */ "MERGE", - /* 216 */ "REDISTRIBUTE", - /* 217 */ "SPLIT", - /* 218 */ "DELETE", - /* 219 */ "INSERT", - /* 220 */ "NULL", - /* 221 */ "NK_QUESTION", - /* 222 */ "NK_ARROW", - /* 223 */ "ROWTS", - /* 224 */ "QSTART", - /* 225 */ "QEND", - /* 226 */ "QDURATION", - /* 227 */ "WSTART", - /* 228 */ "WEND", - /* 229 */ "WDURATION", - /* 230 */ "IROWTS", - /* 231 */ "ISFILLED", - /* 232 */ "CAST", - /* 233 */ "NOW", - /* 234 */ "TODAY", - /* 235 */ "TIMEZONE", - /* 236 */ "CLIENT_VERSION", - /* 237 */ "SERVER_VERSION", - /* 238 */ "SERVER_STATUS", - /* 239 */ "CURRENT_USER", - /* 240 */ "CASE", - /* 241 */ "WHEN", - /* 242 */ "THEN", - /* 243 */ "ELSE", - /* 244 */ "BETWEEN", - /* 245 */ "IS", - /* 246 */ "NK_LT", - /* 247 */ "NK_GT", - /* 248 */ "NK_LE", - /* 249 */ "NK_GE", - /* 250 */ "NK_NE", - /* 251 */ "MATCH", - /* 252 */ "NMATCH", - /* 253 */ "CONTAINS", - /* 254 */ "IN", - /* 255 */ "JOIN", - /* 256 */ "INNER", - /* 257 */ "SELECT", - /* 258 */ "DISTINCT", - /* 259 */ "WHERE", - /* 260 */ "PARTITION", - /* 261 */ "BY", - /* 262 */ "SESSION", - /* 263 */ "STATE_WINDOW", - /* 264 */ "EVENT_WINDOW", - /* 265 */ "SLIDING", - /* 266 */ "FILL", - /* 267 */ "VALUE", - /* 268 */ "VALUE_F", - /* 269 */ "NONE", - /* 270 */ "PREV", - /* 271 */ "NULL_F", - /* 272 */ "LINEAR", - /* 273 */ "NEXT", - /* 274 */ "HAVING", - /* 275 */ "RANGE", - /* 276 */ "EVERY", - /* 277 */ "ORDER", - /* 278 */ "SLIMIT", - /* 279 */ "SOFFSET", - /* 280 */ "LIMIT", - /* 281 */ "OFFSET", - /* 282 */ "ASC", - /* 283 */ "NULLS", - /* 284 */ "ABORT", - /* 285 */ "AFTER", - /* 286 */ "ATTACH", - /* 287 */ "BEFORE", - /* 288 */ "BEGIN", - /* 289 */ "BITAND", - /* 290 */ "BITNOT", - /* 291 */ "BITOR", - /* 292 */ "BLOCKS", - /* 293 */ "CHANGE", - /* 294 */ "COMMA", - /* 295 */ "CONCAT", - /* 296 */ "CONFLICT", - /* 297 */ "COPY", - /* 298 */ "DEFERRED", - /* 299 */ "DELIMITERS", - /* 300 */ "DETACH", - /* 301 */ "DIVIDE", - /* 302 */ "DOT", - /* 303 */ "EACH", - /* 304 */ "FAIL", - /* 305 */ "FILE", - /* 306 */ "FOR", - /* 307 */ "GLOB", - /* 308 */ "ID", - /* 309 */ "IMMEDIATE", - /* 310 */ "IMPORT", - /* 311 */ "INITIALLY", - /* 312 */ "INSTEAD", - /* 313 */ "ISNULL", - /* 314 */ "KEY", - /* 315 */ "MODULES", - /* 316 */ "NK_BITNOT", - /* 317 */ "NK_SEMI", - /* 318 */ "NOTNULL", - /* 319 */ "OF", - /* 320 */ "PLUS", - /* 321 */ "PRIVILEGE", - /* 322 */ "RAISE", - /* 323 */ "RESTRICT", - /* 324 */ "ROW", - /* 325 */ "SEMI", - /* 326 */ "STAR", - /* 327 */ "STATEMENT", - /* 328 */ "STRICT", - /* 329 */ "STRING", - /* 330 */ "TIMES", - /* 331 */ "VALUES", - /* 332 */ "VARIABLE", - /* 333 */ "VIEW", - /* 334 */ "WAL", - /* 335 */ "cmd", - /* 336 */ "account_options", - /* 337 */ "alter_account_options", - /* 338 */ "literal", - /* 339 */ "alter_account_option", - /* 340 */ "user_name", - /* 341 */ "sysinfo_opt", - /* 342 */ "privileges", - /* 343 */ "priv_level", - /* 344 */ "with_opt", - /* 345 */ "priv_type_list", - /* 346 */ "priv_type", - /* 347 */ "db_name", - /* 348 */ "table_name", - /* 349 */ "topic_name", - /* 350 */ "search_condition", - /* 351 */ "dnode_endpoint", - /* 352 */ "force_opt", - /* 353 */ "not_exists_opt", - /* 354 */ "db_options", - /* 355 */ "exists_opt", - /* 356 */ "alter_db_options", - /* 357 */ "speed_opt", - /* 358 */ "start_opt", - /* 359 */ "end_opt", - /* 360 */ "integer_list", - /* 361 */ "variable_list", - /* 362 */ "retention_list", - /* 363 */ "signed", - /* 364 */ "alter_db_option", - /* 365 */ "retention", - /* 366 */ "full_table_name", - /* 367 */ "column_def_list", - /* 368 */ "tags_def_opt", - /* 369 */ "table_options", - /* 370 */ "multi_create_clause", - /* 371 */ "tags_def", - /* 372 */ "multi_drop_clause", - /* 373 */ "alter_table_clause", - /* 374 */ "alter_table_options", - /* 375 */ "column_name", - /* 376 */ "type_name", - /* 377 */ "signed_literal", - /* 378 */ "create_subtable_clause", - /* 379 */ "specific_cols_opt", - /* 380 */ "expression_list", - /* 381 */ "drop_table_clause", - /* 382 */ "col_name_list", - /* 383 */ "column_def", - /* 384 */ "duration_list", - /* 385 */ "rollup_func_list", - /* 386 */ "alter_table_option", - /* 387 */ "duration_literal", - /* 388 */ "rollup_func_name", - /* 389 */ "function_name", - /* 390 */ "col_name", - /* 391 */ "db_name_cond_opt", - /* 392 */ "like_pattern_opt", - /* 393 */ "table_name_cond", - /* 394 */ "from_db_opt", - /* 395 */ "tag_list_opt", - /* 396 */ "tag_item", - /* 397 */ "column_alias", - /* 398 */ "full_index_name", - /* 399 */ "index_options", - /* 400 */ "index_name", - /* 401 */ "func_list", - /* 402 */ "sliding_opt", - /* 403 */ "sma_stream_opt", - /* 404 */ "func", - /* 405 */ "sma_func_name", - /* 406 */ "query_or_subquery", - /* 407 */ "cgroup_name", - /* 408 */ "analyze_opt", - /* 409 */ "explain_options", - /* 410 */ "insert_query", - /* 411 */ "or_replace_opt", - /* 412 */ "agg_func_opt", - /* 413 */ "bufsize_opt", - /* 414 */ "language_opt", - /* 415 */ "stream_name", - /* 416 */ "stream_options", - /* 417 */ "col_list_opt", - /* 418 */ "tag_def_or_ref_opt", - /* 419 */ "subtable_opt", - /* 420 */ "ignore_opt", - /* 421 */ "expression", - /* 422 */ "dnode_list", - /* 423 */ "where_clause_opt", - /* 424 */ "literal_func", - /* 425 */ "literal_list", - /* 426 */ "table_alias", - /* 427 */ "expr_or_subquery", - /* 428 */ "pseudo_column", - /* 429 */ "column_reference", - /* 430 */ "function_expression", - /* 431 */ "case_when_expression", - /* 432 */ "star_func", - /* 433 */ "star_func_para_list", - /* 434 */ "noarg_func", - /* 435 */ "other_para_list", - /* 436 */ "star_func_para", - /* 437 */ "when_then_list", - /* 438 */ "case_when_else_opt", - /* 439 */ "common_expression", - /* 440 */ "when_then_expr", - /* 441 */ "predicate", - /* 442 */ "compare_op", - /* 443 */ "in_op", - /* 444 */ "in_predicate_value", - /* 445 */ "boolean_value_expression", - /* 446 */ "boolean_primary", - /* 447 */ "from_clause_opt", - /* 448 */ "table_reference_list", - /* 449 */ "table_reference", - /* 450 */ "table_primary", - /* 451 */ "joined_table", - /* 452 */ "alias_opt", - /* 453 */ "subquery", - /* 454 */ "parenthesized_joined_table", - /* 455 */ "join_type", - /* 456 */ "query_specification", - /* 457 */ "set_quantifier_opt", - /* 458 */ "select_list", - /* 459 */ "partition_by_clause_opt", - /* 460 */ "range_opt", - /* 461 */ "every_opt", - /* 462 */ "fill_opt", - /* 463 */ "twindow_clause_opt", - /* 464 */ "group_by_clause_opt", - /* 465 */ "having_clause_opt", - /* 466 */ "select_item", - /* 467 */ "partition_list", - /* 468 */ "partition_item", - /* 469 */ "fill_mode", - /* 470 */ "group_by_list", - /* 471 */ "query_expression", - /* 472 */ "query_simple", - /* 473 */ "order_by_clause_opt", - /* 474 */ "slimit_clause_opt", - /* 475 */ "limit_clause_opt", - /* 476 */ "union_query_expression", - /* 477 */ "query_simple_or_subquery", - /* 478 */ "sort_specification_list", - /* 479 */ "sort_specification", - /* 480 */ "ordering_specification_opt", - /* 481 */ "null_ordering_opt", + /* 55 */ "UNSAFE", + /* 56 */ "LOCAL", + /* 57 */ "QNODE", + /* 58 */ "BNODE", + /* 59 */ "SNODE", + /* 60 */ "MNODE", + /* 61 */ "VNODE", + /* 62 */ "DATABASE", + /* 63 */ "USE", + /* 64 */ "FLUSH", + /* 65 */ "TRIM", + /* 66 */ "COMPACT", + /* 67 */ "IF", + /* 68 */ "NOT", + /* 69 */ "EXISTS", + /* 70 */ "BUFFER", + /* 71 */ "CACHEMODEL", + /* 72 */ "CACHESIZE", + /* 73 */ "COMP", + /* 74 */ "DURATION", + /* 75 */ "NK_VARIABLE", + /* 76 */ "MAXROWS", + /* 77 */ "MINROWS", + /* 78 */ "KEEP", + /* 79 */ "PAGES", + /* 80 */ "PAGESIZE", + /* 81 */ "TSDB_PAGESIZE", + /* 82 */ "PRECISION", + /* 83 */ "REPLICA", + /* 84 */ "VGROUPS", + /* 85 */ "SINGLE_STABLE", + /* 86 */ "RETENTIONS", + /* 87 */ "SCHEMALESS", + /* 88 */ "WAL_LEVEL", + /* 89 */ "WAL_FSYNC_PERIOD", + /* 90 */ "WAL_RETENTION_PERIOD", + /* 91 */ "WAL_RETENTION_SIZE", + /* 92 */ "WAL_ROLL_PERIOD", + /* 93 */ "WAL_SEGMENT_SIZE", + /* 94 */ "STT_TRIGGER", + /* 95 */ "TABLE_PREFIX", + /* 96 */ "TABLE_SUFFIX", + /* 97 */ "NK_COLON", + /* 98 */ "MAX_SPEED", + /* 99 */ "START", + /* 100 */ "TIMESTAMP", + /* 101 */ "END", + /* 102 */ "TABLE", + /* 103 */ "NK_LP", + /* 104 */ "NK_RP", + /* 105 */ "STABLE", + /* 106 */ "ADD", + /* 107 */ "COLUMN", + /* 108 */ "MODIFY", + /* 109 */ "RENAME", + /* 110 */ "TAG", + /* 111 */ "SET", + /* 112 */ "NK_EQ", + /* 113 */ "USING", + /* 114 */ "TAGS", + /* 115 */ "BOOL", + /* 116 */ "TINYINT", + /* 117 */ "SMALLINT", + /* 118 */ "INT", + /* 119 */ "INTEGER", + /* 120 */ "BIGINT", + /* 121 */ "FLOAT", + /* 122 */ "DOUBLE", + /* 123 */ "BINARY", + /* 124 */ "NCHAR", + /* 125 */ "UNSIGNED", + /* 126 */ "JSON", + /* 127 */ "VARCHAR", + /* 128 */ "MEDIUMBLOB", + /* 129 */ "BLOB", + /* 130 */ "VARBINARY", + /* 131 */ "GEOMETRY", + /* 132 */ "DECIMAL", + /* 133 */ "COMMENT", + /* 134 */ "MAX_DELAY", + /* 135 */ "WATERMARK", + /* 136 */ "ROLLUP", + /* 137 */ "TTL", + /* 138 */ "SMA", + /* 139 */ "DELETE_MARK", + /* 140 */ "FIRST", + /* 141 */ "LAST", + /* 142 */ "SHOW", + /* 143 */ "PRIVILEGES", + /* 144 */ "DATABASES", + /* 145 */ "TABLES", + /* 146 */ "STABLES", + /* 147 */ "MNODES", + /* 148 */ "QNODES", + /* 149 */ "FUNCTIONS", + /* 150 */ "INDEXES", + /* 151 */ "ACCOUNTS", + /* 152 */ "APPS", + /* 153 */ "CONNECTIONS", + /* 154 */ "LICENCES", + /* 155 */ "GRANTS", + /* 156 */ "QUERIES", + /* 157 */ "SCORES", + /* 158 */ "TOPICS", + /* 159 */ "VARIABLES", + /* 160 */ "CLUSTER", + /* 161 */ "BNODES", + /* 162 */ "SNODES", + /* 163 */ "TRANSACTIONS", + /* 164 */ "DISTRIBUTED", + /* 165 */ "CONSUMERS", + /* 166 */ "SUBSCRIPTIONS", + /* 167 */ "VNODES", + /* 168 */ "ALIVE", + /* 169 */ "LIKE", + /* 170 */ "TBNAME", + /* 171 */ "QTAGS", + /* 172 */ "AS", + /* 173 */ "INDEX", + /* 174 */ "FUNCTION", + /* 175 */ "INTERVAL", + /* 176 */ "COUNT", + /* 177 */ "LAST_ROW", + /* 178 */ "META", + /* 179 */ "ONLY", + /* 180 */ "TOPIC", + /* 181 */ "CONSUMER", + /* 182 */ "GROUP", + /* 183 */ "DESC", + /* 184 */ "DESCRIBE", + /* 185 */ "RESET", + /* 186 */ "QUERY", + /* 187 */ "CACHE", + /* 188 */ "EXPLAIN", + /* 189 */ "ANALYZE", + /* 190 */ "VERBOSE", + /* 191 */ "NK_BOOL", + /* 192 */ "RATIO", + /* 193 */ "NK_FLOAT", + /* 194 */ "OUTPUTTYPE", + /* 195 */ "AGGREGATE", + /* 196 */ "BUFSIZE", + /* 197 */ "LANGUAGE", + /* 198 */ "REPLACE", + /* 199 */ "STREAM", + /* 200 */ "INTO", + /* 201 */ "PAUSE", + /* 202 */ "RESUME", + /* 203 */ "TRIGGER", + /* 204 */ "AT_ONCE", + /* 205 */ "WINDOW_CLOSE", + /* 206 */ "IGNORE", + /* 207 */ "EXPIRED", + /* 208 */ "FILL_HISTORY", + /* 209 */ "UPDATE", + /* 210 */ "SUBTABLE", + /* 211 */ "UNTREATED", + /* 212 */ "KILL", + /* 213 */ "CONNECTION", + /* 214 */ "TRANSACTION", + /* 215 */ "BALANCE", + /* 216 */ "VGROUP", + /* 217 */ "LEADER", + /* 218 */ "MERGE", + /* 219 */ "REDISTRIBUTE", + /* 220 */ "SPLIT", + /* 221 */ "DELETE", + /* 222 */ "INSERT", + /* 223 */ "NULL", + /* 224 */ "NK_QUESTION", + /* 225 */ "NK_ARROW", + /* 226 */ "ROWTS", + /* 227 */ "QSTART", + /* 228 */ "QEND", + /* 229 */ "QDURATION", + /* 230 */ "WSTART", + /* 231 */ "WEND", + /* 232 */ "WDURATION", + /* 233 */ "IROWTS", + /* 234 */ "ISFILLED", + /* 235 */ "CAST", + /* 236 */ "NOW", + /* 237 */ "TODAY", + /* 238 */ "TIMEZONE", + /* 239 */ "CLIENT_VERSION", + /* 240 */ "SERVER_VERSION", + /* 241 */ "SERVER_STATUS", + /* 242 */ "CURRENT_USER", + /* 243 */ "CASE", + /* 244 */ "WHEN", + /* 245 */ "THEN", + /* 246 */ "ELSE", + /* 247 */ "BETWEEN", + /* 248 */ "IS", + /* 249 */ "NK_LT", + /* 250 */ "NK_GT", + /* 251 */ "NK_LE", + /* 252 */ "NK_GE", + /* 253 */ "NK_NE", + /* 254 */ "MATCH", + /* 255 */ "NMATCH", + /* 256 */ "CONTAINS", + /* 257 */ "IN", + /* 258 */ "JOIN", + /* 259 */ "INNER", + /* 260 */ "SELECT", + /* 261 */ "DISTINCT", + /* 262 */ "WHERE", + /* 263 */ "PARTITION", + /* 264 */ "BY", + /* 265 */ "SESSION", + /* 266 */ "STATE_WINDOW", + /* 267 */ "EVENT_WINDOW", + /* 268 */ "SLIDING", + /* 269 */ "FILL", + /* 270 */ "VALUE", + /* 271 */ "VALUE_F", + /* 272 */ "NONE", + /* 273 */ "PREV", + /* 274 */ "NULL_F", + /* 275 */ "LINEAR", + /* 276 */ "NEXT", + /* 277 */ "HAVING", + /* 278 */ "RANGE", + /* 279 */ "EVERY", + /* 280 */ "ORDER", + /* 281 */ "SLIMIT", + /* 282 */ "SOFFSET", + /* 283 */ "LIMIT", + /* 284 */ "OFFSET", + /* 285 */ "ASC", + /* 286 */ "NULLS", + /* 287 */ "ABORT", + /* 288 */ "AFTER", + /* 289 */ "ATTACH", + /* 290 */ "BEFORE", + /* 291 */ "BEGIN", + /* 292 */ "BITAND", + /* 293 */ "BITNOT", + /* 294 */ "BITOR", + /* 295 */ "BLOCKS", + /* 296 */ "CHANGE", + /* 297 */ "COMMA", + /* 298 */ "CONCAT", + /* 299 */ "CONFLICT", + /* 300 */ "COPY", + /* 301 */ "DEFERRED", + /* 302 */ "DELIMITERS", + /* 303 */ "DETACH", + /* 304 */ "DIVIDE", + /* 305 */ "DOT", + /* 306 */ "EACH", + /* 307 */ "FAIL", + /* 308 */ "FILE", + /* 309 */ "FOR", + /* 310 */ "GLOB", + /* 311 */ "ID", + /* 312 */ "IMMEDIATE", + /* 313 */ "IMPORT", + /* 314 */ "INITIALLY", + /* 315 */ "INSTEAD", + /* 316 */ "ISNULL", + /* 317 */ "KEY", + /* 318 */ "MODULES", + /* 319 */ "NK_BITNOT", + /* 320 */ "NK_SEMI", + /* 321 */ "NOTNULL", + /* 322 */ "OF", + /* 323 */ "PLUS", + /* 324 */ "PRIVILEGE", + /* 325 */ "RAISE", + /* 326 */ "RESTRICT", + /* 327 */ "ROW", + /* 328 */ "SEMI", + /* 329 */ "STAR", + /* 330 */ "STATEMENT", + /* 331 */ "STRICT", + /* 332 */ "STRING", + /* 333 */ "TIMES", + /* 334 */ "VALUES", + /* 335 */ "VARIABLE", + /* 336 */ "VIEW", + /* 337 */ "WAL", + /* 338 */ "cmd", + /* 339 */ "account_options", + /* 340 */ "alter_account_options", + /* 341 */ "literal", + /* 342 */ "alter_account_option", + /* 343 */ "user_name", + /* 344 */ "sysinfo_opt", + /* 345 */ "privileges", + /* 346 */ "priv_level", + /* 347 */ "with_opt", + /* 348 */ "priv_type_list", + /* 349 */ "priv_type", + /* 350 */ "db_name", + /* 351 */ "table_name", + /* 352 */ "topic_name", + /* 353 */ "search_condition", + /* 354 */ "dnode_endpoint", + /* 355 */ "force_opt", + /* 356 */ "unsafe_opt", + /* 357 */ "not_exists_opt", + /* 358 */ "db_options", + /* 359 */ "exists_opt", + /* 360 */ "alter_db_options", + /* 361 */ "speed_opt", + /* 362 */ "start_opt", + /* 363 */ "end_opt", + /* 364 */ "integer_list", + /* 365 */ "variable_list", + /* 366 */ "retention_list", + /* 367 */ "signed", + /* 368 */ "alter_db_option", + /* 369 */ "retention", + /* 370 */ "full_table_name", + /* 371 */ "column_def_list", + /* 372 */ "tags_def_opt", + /* 373 */ "table_options", + /* 374 */ "multi_create_clause", + /* 375 */ "tags_def", + /* 376 */ "multi_drop_clause", + /* 377 */ "alter_table_clause", + /* 378 */ "alter_table_options", + /* 379 */ "column_name", + /* 380 */ "type_name", + /* 381 */ "signed_literal", + /* 382 */ "create_subtable_clause", + /* 383 */ "specific_cols_opt", + /* 384 */ "expression_list", + /* 385 */ "drop_table_clause", + /* 386 */ "col_name_list", + /* 387 */ "column_def", + /* 388 */ "duration_list", + /* 389 */ "rollup_func_list", + /* 390 */ "alter_table_option", + /* 391 */ "duration_literal", + /* 392 */ "rollup_func_name", + /* 393 */ "function_name", + /* 394 */ "col_name", + /* 395 */ "db_name_cond_opt", + /* 396 */ "like_pattern_opt", + /* 397 */ "table_name_cond", + /* 398 */ "from_db_opt", + /* 399 */ "tag_list_opt", + /* 400 */ "tag_item", + /* 401 */ "column_alias", + /* 402 */ "full_index_name", + /* 403 */ "index_options", + /* 404 */ "index_name", + /* 405 */ "func_list", + /* 406 */ "sliding_opt", + /* 407 */ "sma_stream_opt", + /* 408 */ "func", + /* 409 */ "sma_func_name", + /* 410 */ "with_meta", + /* 411 */ "query_or_subquery", + /* 412 */ "where_clause_opt", + /* 413 */ "cgroup_name", + /* 414 */ "analyze_opt", + /* 415 */ "explain_options", + /* 416 */ "insert_query", + /* 417 */ "or_replace_opt", + /* 418 */ "agg_func_opt", + /* 419 */ "bufsize_opt", + /* 420 */ "language_opt", + /* 421 */ "stream_name", + /* 422 */ "stream_options", + /* 423 */ "col_list_opt", + /* 424 */ "tag_def_or_ref_opt", + /* 425 */ "subtable_opt", + /* 426 */ "ignore_opt", + /* 427 */ "expression", + /* 428 */ "dnode_list", + /* 429 */ "literal_func", + /* 430 */ "literal_list", + /* 431 */ "table_alias", + /* 432 */ "expr_or_subquery", + /* 433 */ "pseudo_column", + /* 434 */ "column_reference", + /* 435 */ "function_expression", + /* 436 */ "case_when_expression", + /* 437 */ "star_func", + /* 438 */ "star_func_para_list", + /* 439 */ "noarg_func", + /* 440 */ "other_para_list", + /* 441 */ "star_func_para", + /* 442 */ "when_then_list", + /* 443 */ "case_when_else_opt", + /* 444 */ "common_expression", + /* 445 */ "when_then_expr", + /* 446 */ "predicate", + /* 447 */ "compare_op", + /* 448 */ "in_op", + /* 449 */ "in_predicate_value", + /* 450 */ "boolean_value_expression", + /* 451 */ "boolean_primary", + /* 452 */ "from_clause_opt", + /* 453 */ "table_reference_list", + /* 454 */ "table_reference", + /* 455 */ "table_primary", + /* 456 */ "joined_table", + /* 457 */ "alias_opt", + /* 458 */ "subquery", + /* 459 */ "parenthesized_joined_table", + /* 460 */ "join_type", + /* 461 */ "query_specification", + /* 462 */ "set_quantifier_opt", + /* 463 */ "select_list", + /* 464 */ "partition_by_clause_opt", + /* 465 */ "range_opt", + /* 466 */ "every_opt", + /* 467 */ "fill_opt", + /* 468 */ "twindow_clause_opt", + /* 469 */ "group_by_clause_opt", + /* 470 */ "having_clause_opt", + /* 471 */ "select_item", + /* 472 */ "partition_list", + /* 473 */ "partition_item", + /* 474 */ "fill_mode", + /* 475 */ "group_by_list", + /* 476 */ "query_expression", + /* 477 */ "query_simple", + /* 478 */ "order_by_clause_opt", + /* 479 */ "slimit_clause_opt", + /* 480 */ "limit_clause_opt", + /* 481 */ "union_query_expression", + /* 482 */ "query_simple_or_subquery", + /* 483 */ "sort_specification_list", + /* 484 */ "sort_specification", + /* 485 */ "ordering_specification_opt", + /* 486 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -2036,547 +2021,556 @@ static const char *const yyRuleName[] = { /* 47 */ "cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER", /* 48 */ "cmd ::= DROP DNODE NK_INTEGER force_opt", /* 49 */ "cmd ::= DROP DNODE dnode_endpoint force_opt", - /* 50 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING", - /* 51 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING", - /* 52 */ "cmd ::= ALTER ALL DNODES NK_STRING", - /* 53 */ "cmd ::= ALTER ALL DNODES NK_STRING NK_STRING", - /* 54 */ "cmd ::= RESTORE DNODE NK_INTEGER", - /* 55 */ "dnode_endpoint ::= NK_STRING", - /* 56 */ "dnode_endpoint ::= NK_ID", - /* 57 */ "dnode_endpoint ::= NK_IPTOKEN", - /* 58 */ "force_opt ::=", - /* 59 */ "force_opt ::= FORCE", - /* 60 */ "cmd ::= ALTER LOCAL NK_STRING", - /* 61 */ "cmd ::= ALTER LOCAL NK_STRING NK_STRING", - /* 62 */ "cmd ::= CREATE QNODE ON DNODE NK_INTEGER", - /* 63 */ "cmd ::= DROP QNODE ON DNODE NK_INTEGER", - /* 64 */ "cmd ::= RESTORE QNODE ON DNODE NK_INTEGER", - /* 65 */ "cmd ::= CREATE BNODE ON DNODE NK_INTEGER", - /* 66 */ "cmd ::= DROP BNODE ON DNODE NK_INTEGER", - /* 67 */ "cmd ::= CREATE SNODE ON DNODE NK_INTEGER", - /* 68 */ "cmd ::= DROP SNODE ON DNODE NK_INTEGER", - /* 69 */ "cmd ::= CREATE MNODE ON DNODE NK_INTEGER", - /* 70 */ "cmd ::= DROP MNODE ON DNODE NK_INTEGER", - /* 71 */ "cmd ::= RESTORE MNODE ON DNODE NK_INTEGER", - /* 72 */ "cmd ::= RESTORE VNODE ON DNODE NK_INTEGER", - /* 73 */ "cmd ::= CREATE DATABASE not_exists_opt db_name db_options", - /* 74 */ "cmd ::= DROP DATABASE exists_opt db_name", - /* 75 */ "cmd ::= USE db_name", - /* 76 */ "cmd ::= ALTER DATABASE db_name alter_db_options", - /* 77 */ "cmd ::= FLUSH DATABASE db_name", - /* 78 */ "cmd ::= TRIM DATABASE db_name speed_opt", - /* 79 */ "cmd ::= COMPACT DATABASE db_name start_opt end_opt", - /* 80 */ "not_exists_opt ::= IF NOT EXISTS", - /* 81 */ "not_exists_opt ::=", - /* 82 */ "exists_opt ::= IF EXISTS", - /* 83 */ "exists_opt ::=", - /* 84 */ "db_options ::=", - /* 85 */ "db_options ::= db_options BUFFER NK_INTEGER", - /* 86 */ "db_options ::= db_options CACHEMODEL NK_STRING", - /* 87 */ "db_options ::= db_options CACHESIZE NK_INTEGER", - /* 88 */ "db_options ::= db_options COMP NK_INTEGER", - /* 89 */ "db_options ::= db_options DURATION NK_INTEGER", - /* 90 */ "db_options ::= db_options DURATION NK_VARIABLE", - /* 91 */ "db_options ::= db_options MAXROWS NK_INTEGER", - /* 92 */ "db_options ::= db_options MINROWS NK_INTEGER", - /* 93 */ "db_options ::= db_options KEEP integer_list", - /* 94 */ "db_options ::= db_options KEEP variable_list", - /* 95 */ "db_options ::= db_options PAGES NK_INTEGER", - /* 96 */ "db_options ::= db_options PAGESIZE NK_INTEGER", - /* 97 */ "db_options ::= db_options TSDB_PAGESIZE NK_INTEGER", - /* 98 */ "db_options ::= db_options PRECISION NK_STRING", - /* 99 */ "db_options ::= db_options REPLICA NK_INTEGER", - /* 100 */ "db_options ::= db_options VGROUPS NK_INTEGER", - /* 101 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", - /* 102 */ "db_options ::= db_options RETENTIONS retention_list", - /* 103 */ "db_options ::= db_options SCHEMALESS NK_INTEGER", - /* 104 */ "db_options ::= db_options WAL_LEVEL NK_INTEGER", - /* 105 */ "db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER", - /* 106 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER", - /* 107 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", - /* 108 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER", - /* 109 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", - /* 110 */ "db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER", - /* 111 */ "db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER", - /* 112 */ "db_options ::= db_options STT_TRIGGER NK_INTEGER", - /* 113 */ "db_options ::= db_options TABLE_PREFIX signed", - /* 114 */ "db_options ::= db_options TABLE_SUFFIX signed", - /* 115 */ "alter_db_options ::= alter_db_option", - /* 116 */ "alter_db_options ::= alter_db_options alter_db_option", - /* 117 */ "alter_db_option ::= BUFFER NK_INTEGER", - /* 118 */ "alter_db_option ::= CACHEMODEL NK_STRING", - /* 119 */ "alter_db_option ::= CACHESIZE NK_INTEGER", - /* 120 */ "alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER", - /* 121 */ "alter_db_option ::= KEEP integer_list", - /* 122 */ "alter_db_option ::= KEEP variable_list", - /* 123 */ "alter_db_option ::= PAGES NK_INTEGER", - /* 124 */ "alter_db_option ::= REPLICA NK_INTEGER", - /* 125 */ "alter_db_option ::= WAL_LEVEL NK_INTEGER", - /* 126 */ "alter_db_option ::= STT_TRIGGER NK_INTEGER", - /* 127 */ "alter_db_option ::= MINROWS NK_INTEGER", - /* 128 */ "alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER", - /* 129 */ "alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", - /* 130 */ "alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER", - /* 131 */ "alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", - /* 132 */ "integer_list ::= NK_INTEGER", - /* 133 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", - /* 134 */ "variable_list ::= NK_VARIABLE", - /* 135 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", - /* 136 */ "retention_list ::= retention", - /* 137 */ "retention_list ::= retention_list NK_COMMA retention", - /* 138 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", - /* 139 */ "speed_opt ::=", - /* 140 */ "speed_opt ::= MAX_SPEED NK_INTEGER", - /* 141 */ "start_opt ::=", - /* 142 */ "start_opt ::= START WITH NK_INTEGER", - /* 143 */ "start_opt ::= START WITH NK_STRING", - /* 144 */ "start_opt ::= START WITH TIMESTAMP NK_STRING", - /* 145 */ "end_opt ::=", - /* 146 */ "end_opt ::= END WITH NK_INTEGER", - /* 147 */ "end_opt ::= END WITH NK_STRING", - /* 148 */ "end_opt ::= END WITH TIMESTAMP NK_STRING", - /* 149 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", - /* 150 */ "cmd ::= CREATE TABLE multi_create_clause", - /* 151 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", - /* 152 */ "cmd ::= DROP TABLE multi_drop_clause", - /* 153 */ "cmd ::= DROP STABLE exists_opt full_table_name", - /* 154 */ "cmd ::= ALTER TABLE alter_table_clause", - /* 155 */ "cmd ::= ALTER STABLE alter_table_clause", - /* 156 */ "alter_table_clause ::= full_table_name alter_table_options", - /* 157 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", - /* 158 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", - /* 159 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", - /* 160 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", - /* 161 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", - /* 162 */ "alter_table_clause ::= full_table_name DROP TAG column_name", - /* 163 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", - /* 164 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", - /* 165 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", - /* 166 */ "multi_create_clause ::= create_subtable_clause", - /* 167 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", - /* 168 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options", - /* 169 */ "multi_drop_clause ::= drop_table_clause", - /* 170 */ "multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause", - /* 171 */ "drop_table_clause ::= exists_opt full_table_name", - /* 172 */ "specific_cols_opt ::=", - /* 173 */ "specific_cols_opt ::= NK_LP col_name_list NK_RP", - /* 174 */ "full_table_name ::= table_name", - /* 175 */ "full_table_name ::= db_name NK_DOT table_name", - /* 176 */ "column_def_list ::= column_def", - /* 177 */ "column_def_list ::= column_def_list NK_COMMA column_def", - /* 178 */ "column_def ::= column_name type_name", - /* 179 */ "type_name ::= BOOL", - /* 180 */ "type_name ::= TINYINT", - /* 181 */ "type_name ::= SMALLINT", - /* 182 */ "type_name ::= INT", - /* 183 */ "type_name ::= INTEGER", - /* 184 */ "type_name ::= BIGINT", - /* 185 */ "type_name ::= FLOAT", - /* 186 */ "type_name ::= DOUBLE", - /* 187 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", - /* 188 */ "type_name ::= TIMESTAMP", - /* 189 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", - /* 190 */ "type_name ::= TINYINT UNSIGNED", - /* 191 */ "type_name ::= SMALLINT UNSIGNED", - /* 192 */ "type_name ::= INT UNSIGNED", - /* 193 */ "type_name ::= BIGINT UNSIGNED", - /* 194 */ "type_name ::= JSON", - /* 195 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", - /* 196 */ "type_name ::= MEDIUMBLOB", - /* 197 */ "type_name ::= BLOB", - /* 198 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", - /* 199 */ "type_name ::= DECIMAL", - /* 200 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", - /* 201 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", - /* 202 */ "tags_def_opt ::=", - /* 203 */ "tags_def_opt ::= tags_def", - /* 204 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", - /* 205 */ "table_options ::=", - /* 206 */ "table_options ::= table_options COMMENT NK_STRING", - /* 207 */ "table_options ::= table_options MAX_DELAY duration_list", - /* 208 */ "table_options ::= table_options WATERMARK duration_list", - /* 209 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", - /* 210 */ "table_options ::= table_options TTL NK_INTEGER", - /* 211 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", - /* 212 */ "table_options ::= table_options DELETE_MARK duration_list", - /* 213 */ "alter_table_options ::= alter_table_option", - /* 214 */ "alter_table_options ::= alter_table_options alter_table_option", - /* 215 */ "alter_table_option ::= COMMENT NK_STRING", - /* 216 */ "alter_table_option ::= TTL NK_INTEGER", - /* 217 */ "duration_list ::= duration_literal", - /* 218 */ "duration_list ::= duration_list NK_COMMA duration_literal", - /* 219 */ "rollup_func_list ::= rollup_func_name", - /* 220 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", - /* 221 */ "rollup_func_name ::= function_name", - /* 222 */ "rollup_func_name ::= FIRST", - /* 223 */ "rollup_func_name ::= LAST", - /* 224 */ "col_name_list ::= col_name", - /* 225 */ "col_name_list ::= col_name_list NK_COMMA col_name", - /* 226 */ "col_name ::= column_name", - /* 227 */ "cmd ::= SHOW DNODES", - /* 228 */ "cmd ::= SHOW USERS", - /* 229 */ "cmd ::= SHOW USER PRIVILEGES", - /* 230 */ "cmd ::= SHOW DATABASES", - /* 231 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", - /* 232 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", - /* 233 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", - /* 234 */ "cmd ::= SHOW MNODES", - /* 235 */ "cmd ::= SHOW QNODES", - /* 236 */ "cmd ::= SHOW FUNCTIONS", - /* 237 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", - /* 238 */ "cmd ::= SHOW STREAMS", - /* 239 */ "cmd ::= SHOW ACCOUNTS", - /* 240 */ "cmd ::= SHOW APPS", - /* 241 */ "cmd ::= SHOW CONNECTIONS", - /* 242 */ "cmd ::= SHOW LICENCES", - /* 243 */ "cmd ::= SHOW GRANTS", - /* 244 */ "cmd ::= SHOW CREATE DATABASE db_name", - /* 245 */ "cmd ::= SHOW CREATE TABLE full_table_name", - /* 246 */ "cmd ::= SHOW CREATE STABLE full_table_name", - /* 247 */ "cmd ::= SHOW QUERIES", - /* 248 */ "cmd ::= SHOW SCORES", - /* 249 */ "cmd ::= SHOW TOPICS", - /* 250 */ "cmd ::= SHOW VARIABLES", - /* 251 */ "cmd ::= SHOW CLUSTER VARIABLES", - /* 252 */ "cmd ::= SHOW LOCAL VARIABLES", - /* 253 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt", - /* 254 */ "cmd ::= SHOW BNODES", - /* 255 */ "cmd ::= SHOW SNODES", - /* 256 */ "cmd ::= SHOW CLUSTER", - /* 257 */ "cmd ::= SHOW TRANSACTIONS", - /* 258 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", - /* 259 */ "cmd ::= SHOW CONSUMERS", - /* 260 */ "cmd ::= SHOW SUBSCRIPTIONS", - /* 261 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", - /* 262 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", - /* 263 */ "cmd ::= SHOW VNODES NK_INTEGER", - /* 264 */ "cmd ::= SHOW VNODES NK_STRING", - /* 265 */ "cmd ::= SHOW db_name_cond_opt ALIVE", - /* 266 */ "cmd ::= SHOW CLUSTER ALIVE", - /* 267 */ "db_name_cond_opt ::=", - /* 268 */ "db_name_cond_opt ::= db_name NK_DOT", - /* 269 */ "like_pattern_opt ::=", - /* 270 */ "like_pattern_opt ::= LIKE NK_STRING", - /* 271 */ "table_name_cond ::= table_name", - /* 272 */ "from_db_opt ::=", - /* 273 */ "from_db_opt ::= FROM db_name", - /* 274 */ "tag_list_opt ::=", - /* 275 */ "tag_list_opt ::= tag_item", - /* 276 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", - /* 277 */ "tag_item ::= TBNAME", - /* 278 */ "tag_item ::= QTAGS", - /* 279 */ "tag_item ::= column_name", - /* 280 */ "tag_item ::= column_name column_alias", - /* 281 */ "tag_item ::= column_name AS column_alias", - /* 282 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", - /* 283 */ "cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP", - /* 284 */ "cmd ::= DROP INDEX exists_opt full_index_name", - /* 285 */ "full_index_name ::= index_name", - /* 286 */ "full_index_name ::= db_name NK_DOT index_name", - /* 287 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", - /* 288 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", - /* 289 */ "func_list ::= func", - /* 290 */ "func_list ::= func_list NK_COMMA func", - /* 291 */ "func ::= sma_func_name NK_LP expression_list NK_RP", - /* 292 */ "sma_func_name ::= function_name", - /* 293 */ "sma_func_name ::= COUNT", - /* 294 */ "sma_func_name ::= FIRST", - /* 295 */ "sma_func_name ::= LAST", - /* 296 */ "sma_func_name ::= LAST_ROW", - /* 297 */ "sma_stream_opt ::=", - /* 298 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", - /* 299 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", - /* 300 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", - /* 301 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", - /* 302 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", - /* 303 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", - /* 304 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", - /* 305 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", - /* 306 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 307 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", - /* 308 */ "cmd ::= DESC full_table_name", - /* 309 */ "cmd ::= DESCRIBE full_table_name", - /* 310 */ "cmd ::= RESET QUERY CACHE", - /* 311 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", - /* 312 */ "cmd ::= EXPLAIN analyze_opt explain_options insert_query", - /* 313 */ "analyze_opt ::=", - /* 314 */ "analyze_opt ::= ANALYZE", - /* 315 */ "explain_options ::=", - /* 316 */ "explain_options ::= explain_options VERBOSE NK_BOOL", - /* 317 */ "explain_options ::= explain_options RATIO NK_FLOAT", - /* 318 */ "cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt", - /* 319 */ "cmd ::= DROP FUNCTION exists_opt function_name", - /* 320 */ "agg_func_opt ::=", - /* 321 */ "agg_func_opt ::= AGGREGATE", - /* 322 */ "bufsize_opt ::=", - /* 323 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", - /* 324 */ "language_opt ::=", - /* 325 */ "language_opt ::= LANGUAGE NK_STRING", - /* 326 */ "or_replace_opt ::=", - /* 327 */ "or_replace_opt ::= OR REPLACE", - /* 328 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery", - /* 329 */ "cmd ::= DROP STREAM exists_opt stream_name", - /* 330 */ "cmd ::= PAUSE STREAM exists_opt stream_name", - /* 331 */ "cmd ::= RESUME STREAM exists_opt ignore_opt stream_name", - /* 332 */ "col_list_opt ::=", - /* 333 */ "col_list_opt ::= NK_LP col_name_list NK_RP", - /* 334 */ "tag_def_or_ref_opt ::=", - /* 335 */ "tag_def_or_ref_opt ::= tags_def", - /* 336 */ "tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP", - /* 337 */ "stream_options ::=", - /* 338 */ "stream_options ::= stream_options TRIGGER AT_ONCE", - /* 339 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", - /* 340 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", - /* 341 */ "stream_options ::= stream_options WATERMARK duration_literal", - /* 342 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", - /* 343 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", - /* 344 */ "stream_options ::= stream_options DELETE_MARK duration_literal", - /* 345 */ "stream_options ::= stream_options IGNORE UPDATE NK_INTEGER", - /* 346 */ "subtable_opt ::=", - /* 347 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", - /* 348 */ "ignore_opt ::=", - /* 349 */ "ignore_opt ::= IGNORE UNTREATED", - /* 350 */ "cmd ::= KILL CONNECTION NK_INTEGER", - /* 351 */ "cmd ::= KILL QUERY NK_STRING", - /* 352 */ "cmd ::= KILL TRANSACTION NK_INTEGER", - /* 353 */ "cmd ::= BALANCE VGROUP", - /* 354 */ "cmd ::= BALANCE VGROUP LEADER", - /* 355 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", - /* 356 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", - /* 357 */ "cmd ::= SPLIT VGROUP NK_INTEGER", - /* 358 */ "dnode_list ::= DNODE NK_INTEGER", - /* 359 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", - /* 360 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", - /* 361 */ "cmd ::= query_or_subquery", - /* 362 */ "cmd ::= insert_query", - /* 363 */ "insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", - /* 364 */ "insert_query ::= INSERT INTO full_table_name query_or_subquery", - /* 365 */ "literal ::= NK_INTEGER", - /* 366 */ "literal ::= NK_FLOAT", - /* 367 */ "literal ::= NK_STRING", - /* 368 */ "literal ::= NK_BOOL", - /* 369 */ "literal ::= TIMESTAMP NK_STRING", - /* 370 */ "literal ::= duration_literal", - /* 371 */ "literal ::= NULL", - /* 372 */ "literal ::= NK_QUESTION", - /* 373 */ "duration_literal ::= NK_VARIABLE", - /* 374 */ "signed ::= NK_INTEGER", - /* 375 */ "signed ::= NK_PLUS NK_INTEGER", - /* 376 */ "signed ::= NK_MINUS NK_INTEGER", - /* 377 */ "signed ::= NK_FLOAT", - /* 378 */ "signed ::= NK_PLUS NK_FLOAT", - /* 379 */ "signed ::= NK_MINUS NK_FLOAT", - /* 380 */ "signed_literal ::= signed", - /* 381 */ "signed_literal ::= NK_STRING", - /* 382 */ "signed_literal ::= NK_BOOL", - /* 383 */ "signed_literal ::= TIMESTAMP NK_STRING", - /* 384 */ "signed_literal ::= duration_literal", - /* 385 */ "signed_literal ::= NULL", - /* 386 */ "signed_literal ::= literal_func", - /* 387 */ "signed_literal ::= NK_QUESTION", - /* 388 */ "literal_list ::= signed_literal", - /* 389 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 390 */ "db_name ::= NK_ID", - /* 391 */ "table_name ::= NK_ID", - /* 392 */ "column_name ::= NK_ID", - /* 393 */ "function_name ::= NK_ID", - /* 394 */ "table_alias ::= NK_ID", - /* 395 */ "column_alias ::= NK_ID", - /* 396 */ "user_name ::= NK_ID", - /* 397 */ "topic_name ::= NK_ID", - /* 398 */ "stream_name ::= NK_ID", - /* 399 */ "cgroup_name ::= NK_ID", - /* 400 */ "index_name ::= NK_ID", - /* 401 */ "expr_or_subquery ::= expression", - /* 402 */ "expression ::= literal", - /* 403 */ "expression ::= pseudo_column", - /* 404 */ "expression ::= column_reference", - /* 405 */ "expression ::= function_expression", - /* 406 */ "expression ::= case_when_expression", - /* 407 */ "expression ::= NK_LP expression NK_RP", - /* 408 */ "expression ::= NK_PLUS expr_or_subquery", - /* 409 */ "expression ::= NK_MINUS expr_or_subquery", - /* 410 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", - /* 411 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", - /* 412 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", - /* 413 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", - /* 414 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", - /* 415 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 416 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", - /* 417 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", - /* 418 */ "expression_list ::= expr_or_subquery", - /* 419 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", - /* 420 */ "column_reference ::= column_name", - /* 421 */ "column_reference ::= table_name NK_DOT column_name", - /* 422 */ "pseudo_column ::= ROWTS", - /* 423 */ "pseudo_column ::= TBNAME", - /* 424 */ "pseudo_column ::= table_name NK_DOT TBNAME", - /* 425 */ "pseudo_column ::= QSTART", - /* 426 */ "pseudo_column ::= QEND", - /* 427 */ "pseudo_column ::= QDURATION", - /* 428 */ "pseudo_column ::= WSTART", - /* 429 */ "pseudo_column ::= WEND", - /* 430 */ "pseudo_column ::= WDURATION", - /* 431 */ "pseudo_column ::= IROWTS", - /* 432 */ "pseudo_column ::= ISFILLED", - /* 433 */ "pseudo_column ::= QTAGS", - /* 434 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 435 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 436 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", - /* 437 */ "function_expression ::= literal_func", - /* 438 */ "literal_func ::= noarg_func NK_LP NK_RP", - /* 439 */ "literal_func ::= NOW", - /* 440 */ "noarg_func ::= NOW", - /* 441 */ "noarg_func ::= TODAY", - /* 442 */ "noarg_func ::= TIMEZONE", - /* 443 */ "noarg_func ::= DATABASE", - /* 444 */ "noarg_func ::= CLIENT_VERSION", - /* 445 */ "noarg_func ::= SERVER_VERSION", - /* 446 */ "noarg_func ::= SERVER_STATUS", - /* 447 */ "noarg_func ::= CURRENT_USER", - /* 448 */ "noarg_func ::= USER", - /* 449 */ "star_func ::= COUNT", - /* 450 */ "star_func ::= FIRST", - /* 451 */ "star_func ::= LAST", - /* 452 */ "star_func ::= LAST_ROW", - /* 453 */ "star_func_para_list ::= NK_STAR", - /* 454 */ "star_func_para_list ::= other_para_list", - /* 455 */ "other_para_list ::= star_func_para", - /* 456 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 457 */ "star_func_para ::= expr_or_subquery", - /* 458 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 459 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", - /* 460 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", - /* 461 */ "when_then_list ::= when_then_expr", - /* 462 */ "when_then_list ::= when_then_list when_then_expr", - /* 463 */ "when_then_expr ::= WHEN common_expression THEN common_expression", - /* 464 */ "case_when_else_opt ::=", - /* 465 */ "case_when_else_opt ::= ELSE common_expression", - /* 466 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", - /* 467 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", - /* 468 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", - /* 469 */ "predicate ::= expr_or_subquery IS NULL", - /* 470 */ "predicate ::= expr_or_subquery IS NOT NULL", - /* 471 */ "predicate ::= expr_or_subquery in_op in_predicate_value", - /* 472 */ "compare_op ::= NK_LT", - /* 473 */ "compare_op ::= NK_GT", - /* 474 */ "compare_op ::= NK_LE", - /* 475 */ "compare_op ::= NK_GE", - /* 476 */ "compare_op ::= NK_NE", - /* 477 */ "compare_op ::= NK_EQ", - /* 478 */ "compare_op ::= LIKE", - /* 479 */ "compare_op ::= NOT LIKE", - /* 480 */ "compare_op ::= MATCH", - /* 481 */ "compare_op ::= NMATCH", - /* 482 */ "compare_op ::= CONTAINS", - /* 483 */ "in_op ::= IN", - /* 484 */ "in_op ::= NOT IN", - /* 485 */ "in_predicate_value ::= NK_LP literal_list NK_RP", - /* 486 */ "boolean_value_expression ::= boolean_primary", - /* 487 */ "boolean_value_expression ::= NOT boolean_primary", - /* 488 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 489 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 490 */ "boolean_primary ::= predicate", - /* 491 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 492 */ "common_expression ::= expr_or_subquery", - /* 493 */ "common_expression ::= boolean_value_expression", - /* 494 */ "from_clause_opt ::=", - /* 495 */ "from_clause_opt ::= FROM table_reference_list", - /* 496 */ "table_reference_list ::= table_reference", - /* 497 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 498 */ "table_reference ::= table_primary", - /* 499 */ "table_reference ::= joined_table", - /* 500 */ "table_primary ::= table_name alias_opt", - /* 501 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 502 */ "table_primary ::= subquery alias_opt", - /* 503 */ "table_primary ::= parenthesized_joined_table", - /* 504 */ "alias_opt ::=", - /* 505 */ "alias_opt ::= table_alias", - /* 506 */ "alias_opt ::= AS table_alias", - /* 507 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 508 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 509 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 510 */ "join_type ::=", - /* 511 */ "join_type ::= INNER", - /* 512 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 513 */ "set_quantifier_opt ::=", - /* 514 */ "set_quantifier_opt ::= DISTINCT", - /* 515 */ "set_quantifier_opt ::= ALL", - /* 516 */ "select_list ::= select_item", - /* 517 */ "select_list ::= select_list NK_COMMA select_item", - /* 518 */ "select_item ::= NK_STAR", - /* 519 */ "select_item ::= common_expression", - /* 520 */ "select_item ::= common_expression column_alias", - /* 521 */ "select_item ::= common_expression AS column_alias", - /* 522 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 523 */ "where_clause_opt ::=", - /* 524 */ "where_clause_opt ::= WHERE search_condition", - /* 525 */ "partition_by_clause_opt ::=", - /* 526 */ "partition_by_clause_opt ::= PARTITION BY partition_list", - /* 527 */ "partition_list ::= partition_item", - /* 528 */ "partition_list ::= partition_list NK_COMMA partition_item", - /* 529 */ "partition_item ::= expr_or_subquery", - /* 530 */ "partition_item ::= expr_or_subquery column_alias", - /* 531 */ "partition_item ::= expr_or_subquery AS column_alias", - /* 532 */ "twindow_clause_opt ::=", - /* 533 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 534 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", - /* 535 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 536 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 537 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", - /* 538 */ "sliding_opt ::=", - /* 539 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 540 */ "fill_opt ::=", - /* 541 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 542 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP", - /* 543 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP", - /* 544 */ "fill_mode ::= NONE", - /* 545 */ "fill_mode ::= PREV", - /* 546 */ "fill_mode ::= NULL", - /* 547 */ "fill_mode ::= NULL_F", - /* 548 */ "fill_mode ::= LINEAR", - /* 549 */ "fill_mode ::= NEXT", - /* 550 */ "group_by_clause_opt ::=", - /* 551 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 552 */ "group_by_list ::= expr_or_subquery", - /* 553 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", - /* 554 */ "having_clause_opt ::=", - /* 555 */ "having_clause_opt ::= HAVING search_condition", - /* 556 */ "range_opt ::=", - /* 557 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", - /* 558 */ "every_opt ::=", - /* 559 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", - /* 560 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 561 */ "query_simple ::= query_specification", - /* 562 */ "query_simple ::= union_query_expression", - /* 563 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", - /* 564 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", - /* 565 */ "query_simple_or_subquery ::= query_simple", - /* 566 */ "query_simple_or_subquery ::= subquery", - /* 567 */ "query_or_subquery ::= query_expression", - /* 568 */ "query_or_subquery ::= subquery", - /* 569 */ "order_by_clause_opt ::=", - /* 570 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 571 */ "slimit_clause_opt ::=", - /* 572 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 573 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 574 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 575 */ "limit_clause_opt ::=", - /* 576 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 577 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 578 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 579 */ "subquery ::= NK_LP query_expression NK_RP", - /* 580 */ "subquery ::= NK_LP subquery NK_RP", - /* 581 */ "search_condition ::= common_expression", - /* 582 */ "sort_specification_list ::= sort_specification", - /* 583 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 584 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", - /* 585 */ "ordering_specification_opt ::=", - /* 586 */ "ordering_specification_opt ::= ASC", - /* 587 */ "ordering_specification_opt ::= DESC", - /* 588 */ "null_ordering_opt ::=", - /* 589 */ "null_ordering_opt ::= NULLS FIRST", - /* 590 */ "null_ordering_opt ::= NULLS LAST", + /* 50 */ "cmd ::= DROP DNODE NK_INTEGER unsafe_opt", + /* 51 */ "cmd ::= DROP DNODE dnode_endpoint unsafe_opt", + /* 52 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING", + /* 53 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING", + /* 54 */ "cmd ::= ALTER ALL DNODES NK_STRING", + /* 55 */ "cmd ::= ALTER ALL DNODES NK_STRING NK_STRING", + /* 56 */ "cmd ::= RESTORE DNODE NK_INTEGER", + /* 57 */ "dnode_endpoint ::= NK_STRING", + /* 58 */ "dnode_endpoint ::= NK_ID", + /* 59 */ "dnode_endpoint ::= NK_IPTOKEN", + /* 60 */ "force_opt ::=", + /* 61 */ "force_opt ::= FORCE", + /* 62 */ "unsafe_opt ::= UNSAFE", + /* 63 */ "cmd ::= ALTER LOCAL NK_STRING", + /* 64 */ "cmd ::= ALTER LOCAL NK_STRING NK_STRING", + /* 65 */ "cmd ::= CREATE QNODE ON DNODE NK_INTEGER", + /* 66 */ "cmd ::= DROP QNODE ON DNODE NK_INTEGER", + /* 67 */ "cmd ::= RESTORE QNODE ON DNODE NK_INTEGER", + /* 68 */ "cmd ::= CREATE BNODE ON DNODE NK_INTEGER", + /* 69 */ "cmd ::= DROP BNODE ON DNODE NK_INTEGER", + /* 70 */ "cmd ::= CREATE SNODE ON DNODE NK_INTEGER", + /* 71 */ "cmd ::= DROP SNODE ON DNODE NK_INTEGER", + /* 72 */ "cmd ::= CREATE MNODE ON DNODE NK_INTEGER", + /* 73 */ "cmd ::= DROP MNODE ON DNODE NK_INTEGER", + /* 74 */ "cmd ::= RESTORE MNODE ON DNODE NK_INTEGER", + /* 75 */ "cmd ::= RESTORE VNODE ON DNODE NK_INTEGER", + /* 76 */ "cmd ::= CREATE DATABASE not_exists_opt db_name db_options", + /* 77 */ "cmd ::= DROP DATABASE exists_opt db_name", + /* 78 */ "cmd ::= USE db_name", + /* 79 */ "cmd ::= ALTER DATABASE db_name alter_db_options", + /* 80 */ "cmd ::= FLUSH DATABASE db_name", + /* 81 */ "cmd ::= TRIM DATABASE db_name speed_opt", + /* 82 */ "cmd ::= COMPACT DATABASE db_name start_opt end_opt", + /* 83 */ "not_exists_opt ::= IF NOT EXISTS", + /* 84 */ "not_exists_opt ::=", + /* 85 */ "exists_opt ::= IF EXISTS", + /* 86 */ "exists_opt ::=", + /* 87 */ "db_options ::=", + /* 88 */ "db_options ::= db_options BUFFER NK_INTEGER", + /* 89 */ "db_options ::= db_options CACHEMODEL NK_STRING", + /* 90 */ "db_options ::= db_options CACHESIZE NK_INTEGER", + /* 91 */ "db_options ::= db_options COMP NK_INTEGER", + /* 92 */ "db_options ::= db_options DURATION NK_INTEGER", + /* 93 */ "db_options ::= db_options DURATION NK_VARIABLE", + /* 94 */ "db_options ::= db_options MAXROWS NK_INTEGER", + /* 95 */ "db_options ::= db_options MINROWS NK_INTEGER", + /* 96 */ "db_options ::= db_options KEEP integer_list", + /* 97 */ "db_options ::= db_options KEEP variable_list", + /* 98 */ "db_options ::= db_options PAGES NK_INTEGER", + /* 99 */ "db_options ::= db_options PAGESIZE NK_INTEGER", + /* 100 */ "db_options ::= db_options TSDB_PAGESIZE NK_INTEGER", + /* 101 */ "db_options ::= db_options PRECISION NK_STRING", + /* 102 */ "db_options ::= db_options REPLICA NK_INTEGER", + /* 103 */ "db_options ::= db_options VGROUPS NK_INTEGER", + /* 104 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", + /* 105 */ "db_options ::= db_options RETENTIONS retention_list", + /* 106 */ "db_options ::= db_options SCHEMALESS NK_INTEGER", + /* 107 */ "db_options ::= db_options WAL_LEVEL NK_INTEGER", + /* 108 */ "db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER", + /* 109 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER", + /* 110 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", + /* 111 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER", + /* 112 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", + /* 113 */ "db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER", + /* 114 */ "db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER", + /* 115 */ "db_options ::= db_options STT_TRIGGER NK_INTEGER", + /* 116 */ "db_options ::= db_options TABLE_PREFIX signed", + /* 117 */ "db_options ::= db_options TABLE_SUFFIX signed", + /* 118 */ "alter_db_options ::= alter_db_option", + /* 119 */ "alter_db_options ::= alter_db_options alter_db_option", + /* 120 */ "alter_db_option ::= BUFFER NK_INTEGER", + /* 121 */ "alter_db_option ::= CACHEMODEL NK_STRING", + /* 122 */ "alter_db_option ::= CACHESIZE NK_INTEGER", + /* 123 */ "alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER", + /* 124 */ "alter_db_option ::= KEEP integer_list", + /* 125 */ "alter_db_option ::= KEEP variable_list", + /* 126 */ "alter_db_option ::= PAGES NK_INTEGER", + /* 127 */ "alter_db_option ::= REPLICA NK_INTEGER", + /* 128 */ "alter_db_option ::= WAL_LEVEL NK_INTEGER", + /* 129 */ "alter_db_option ::= STT_TRIGGER NK_INTEGER", + /* 130 */ "alter_db_option ::= MINROWS NK_INTEGER", + /* 131 */ "alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER", + /* 132 */ "alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", + /* 133 */ "alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER", + /* 134 */ "alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", + /* 135 */ "integer_list ::= NK_INTEGER", + /* 136 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", + /* 137 */ "variable_list ::= NK_VARIABLE", + /* 138 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", + /* 139 */ "retention_list ::= retention", + /* 140 */ "retention_list ::= retention_list NK_COMMA retention", + /* 141 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", + /* 142 */ "speed_opt ::=", + /* 143 */ "speed_opt ::= MAX_SPEED NK_INTEGER", + /* 144 */ "start_opt ::=", + /* 145 */ "start_opt ::= START WITH NK_INTEGER", + /* 146 */ "start_opt ::= START WITH NK_STRING", + /* 147 */ "start_opt ::= START WITH TIMESTAMP NK_STRING", + /* 148 */ "end_opt ::=", + /* 149 */ "end_opt ::= END WITH NK_INTEGER", + /* 150 */ "end_opt ::= END WITH NK_STRING", + /* 151 */ "end_opt ::= END WITH TIMESTAMP NK_STRING", + /* 152 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", + /* 153 */ "cmd ::= CREATE TABLE multi_create_clause", + /* 154 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", + /* 155 */ "cmd ::= DROP TABLE multi_drop_clause", + /* 156 */ "cmd ::= DROP STABLE exists_opt full_table_name", + /* 157 */ "cmd ::= ALTER TABLE alter_table_clause", + /* 158 */ "cmd ::= ALTER STABLE alter_table_clause", + /* 159 */ "alter_table_clause ::= full_table_name alter_table_options", + /* 160 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", + /* 161 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", + /* 162 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", + /* 163 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", + /* 164 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", + /* 165 */ "alter_table_clause ::= full_table_name DROP TAG column_name", + /* 166 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", + /* 167 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", + /* 168 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", + /* 169 */ "multi_create_clause ::= create_subtable_clause", + /* 170 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", + /* 171 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options", + /* 172 */ "multi_drop_clause ::= drop_table_clause", + /* 173 */ "multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause", + /* 174 */ "drop_table_clause ::= exists_opt full_table_name", + /* 175 */ "specific_cols_opt ::=", + /* 176 */ "specific_cols_opt ::= NK_LP col_name_list NK_RP", + /* 177 */ "full_table_name ::= table_name", + /* 178 */ "full_table_name ::= db_name NK_DOT table_name", + /* 179 */ "column_def_list ::= column_def", + /* 180 */ "column_def_list ::= column_def_list NK_COMMA column_def", + /* 181 */ "column_def ::= column_name type_name", + /* 182 */ "type_name ::= BOOL", + /* 183 */ "type_name ::= TINYINT", + /* 184 */ "type_name ::= SMALLINT", + /* 185 */ "type_name ::= INT", + /* 186 */ "type_name ::= INTEGER", + /* 187 */ "type_name ::= BIGINT", + /* 188 */ "type_name ::= FLOAT", + /* 189 */ "type_name ::= DOUBLE", + /* 190 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", + /* 191 */ "type_name ::= TIMESTAMP", + /* 192 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", + /* 193 */ "type_name ::= TINYINT UNSIGNED", + /* 194 */ "type_name ::= SMALLINT UNSIGNED", + /* 195 */ "type_name ::= INT UNSIGNED", + /* 196 */ "type_name ::= BIGINT UNSIGNED", + /* 197 */ "type_name ::= JSON", + /* 198 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", + /* 199 */ "type_name ::= MEDIUMBLOB", + /* 200 */ "type_name ::= BLOB", + /* 201 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", + /* 202 */ "type_name ::= GEOMETRY NK_LP NK_INTEGER NK_RP", + /* 203 */ "type_name ::= DECIMAL", + /* 204 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", + /* 205 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", + /* 206 */ "tags_def_opt ::=", + /* 207 */ "tags_def_opt ::= tags_def", + /* 208 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", + /* 209 */ "table_options ::=", + /* 210 */ "table_options ::= table_options COMMENT NK_STRING", + /* 211 */ "table_options ::= table_options MAX_DELAY duration_list", + /* 212 */ "table_options ::= table_options WATERMARK duration_list", + /* 213 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", + /* 214 */ "table_options ::= table_options TTL NK_INTEGER", + /* 215 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", + /* 216 */ "table_options ::= table_options DELETE_MARK duration_list", + /* 217 */ "alter_table_options ::= alter_table_option", + /* 218 */ "alter_table_options ::= alter_table_options alter_table_option", + /* 219 */ "alter_table_option ::= COMMENT NK_STRING", + /* 220 */ "alter_table_option ::= TTL NK_INTEGER", + /* 221 */ "duration_list ::= duration_literal", + /* 222 */ "duration_list ::= duration_list NK_COMMA duration_literal", + /* 223 */ "rollup_func_list ::= rollup_func_name", + /* 224 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", + /* 225 */ "rollup_func_name ::= function_name", + /* 226 */ "rollup_func_name ::= FIRST", + /* 227 */ "rollup_func_name ::= LAST", + /* 228 */ "col_name_list ::= col_name", + /* 229 */ "col_name_list ::= col_name_list NK_COMMA col_name", + /* 230 */ "col_name ::= column_name", + /* 231 */ "cmd ::= SHOW DNODES", + /* 232 */ "cmd ::= SHOW USERS", + /* 233 */ "cmd ::= SHOW USER PRIVILEGES", + /* 234 */ "cmd ::= SHOW DATABASES", + /* 235 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", + /* 236 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", + /* 237 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", + /* 238 */ "cmd ::= SHOW MNODES", + /* 239 */ "cmd ::= SHOW QNODES", + /* 240 */ "cmd ::= SHOW FUNCTIONS", + /* 241 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", + /* 242 */ "cmd ::= SHOW INDEXES FROM db_name NK_DOT table_name", + /* 243 */ "cmd ::= SHOW STREAMS", + /* 244 */ "cmd ::= SHOW ACCOUNTS", + /* 245 */ "cmd ::= SHOW APPS", + /* 246 */ "cmd ::= SHOW CONNECTIONS", + /* 247 */ "cmd ::= SHOW LICENCES", + /* 248 */ "cmd ::= SHOW GRANTS", + /* 249 */ "cmd ::= SHOW CREATE DATABASE db_name", + /* 250 */ "cmd ::= SHOW CREATE TABLE full_table_name", + /* 251 */ "cmd ::= SHOW CREATE STABLE full_table_name", + /* 252 */ "cmd ::= SHOW QUERIES", + /* 253 */ "cmd ::= SHOW SCORES", + /* 254 */ "cmd ::= SHOW TOPICS", + /* 255 */ "cmd ::= SHOW VARIABLES", + /* 256 */ "cmd ::= SHOW CLUSTER VARIABLES", + /* 257 */ "cmd ::= SHOW LOCAL VARIABLES", + /* 258 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt", + /* 259 */ "cmd ::= SHOW BNODES", + /* 260 */ "cmd ::= SHOW SNODES", + /* 261 */ "cmd ::= SHOW CLUSTER", + /* 262 */ "cmd ::= SHOW TRANSACTIONS", + /* 263 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", + /* 264 */ "cmd ::= SHOW CONSUMERS", + /* 265 */ "cmd ::= SHOW SUBSCRIPTIONS", + /* 266 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", + /* 267 */ "cmd ::= SHOW TAGS FROM db_name NK_DOT table_name", + /* 268 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", + /* 269 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM db_name NK_DOT table_name", + /* 270 */ "cmd ::= SHOW VNODES NK_INTEGER", + /* 271 */ "cmd ::= SHOW VNODES NK_STRING", + /* 272 */ "cmd ::= SHOW db_name_cond_opt ALIVE", + /* 273 */ "cmd ::= SHOW CLUSTER ALIVE", + /* 274 */ "db_name_cond_opt ::=", + /* 275 */ "db_name_cond_opt ::= db_name NK_DOT", + /* 276 */ "like_pattern_opt ::=", + /* 277 */ "like_pattern_opt ::= LIKE NK_STRING", + /* 278 */ "table_name_cond ::= table_name", + /* 279 */ "from_db_opt ::=", + /* 280 */ "from_db_opt ::= FROM db_name", + /* 281 */ "tag_list_opt ::=", + /* 282 */ "tag_list_opt ::= tag_item", + /* 283 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", + /* 284 */ "tag_item ::= TBNAME", + /* 285 */ "tag_item ::= QTAGS", + /* 286 */ "tag_item ::= column_name", + /* 287 */ "tag_item ::= column_name column_alias", + /* 288 */ "tag_item ::= column_name AS column_alias", + /* 289 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", + /* 290 */ "cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP", + /* 291 */ "cmd ::= DROP INDEX exists_opt full_index_name", + /* 292 */ "full_index_name ::= index_name", + /* 293 */ "full_index_name ::= db_name NK_DOT index_name", + /* 294 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", + /* 295 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", + /* 296 */ "func_list ::= func", + /* 297 */ "func_list ::= func_list NK_COMMA func", + /* 298 */ "func ::= sma_func_name NK_LP expression_list NK_RP", + /* 299 */ "sma_func_name ::= function_name", + /* 300 */ "sma_func_name ::= COUNT", + /* 301 */ "sma_func_name ::= FIRST", + /* 302 */ "sma_func_name ::= LAST", + /* 303 */ "sma_func_name ::= LAST_ROW", + /* 304 */ "sma_stream_opt ::=", + /* 305 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", + /* 306 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", + /* 307 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", + /* 308 */ "with_meta ::= AS", + /* 309 */ "with_meta ::= WITH META AS", + /* 310 */ "with_meta ::= ONLY META AS", + /* 311 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", + /* 312 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta DATABASE db_name", + /* 313 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta STABLE full_table_name where_clause_opt", + /* 314 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 315 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", + /* 316 */ "cmd ::= DESC full_table_name", + /* 317 */ "cmd ::= DESCRIBE full_table_name", + /* 318 */ "cmd ::= RESET QUERY CACHE", + /* 319 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", + /* 320 */ "cmd ::= EXPLAIN analyze_opt explain_options insert_query", + /* 321 */ "analyze_opt ::=", + /* 322 */ "analyze_opt ::= ANALYZE", + /* 323 */ "explain_options ::=", + /* 324 */ "explain_options ::= explain_options VERBOSE NK_BOOL", + /* 325 */ "explain_options ::= explain_options RATIO NK_FLOAT", + /* 326 */ "cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt", + /* 327 */ "cmd ::= DROP FUNCTION exists_opt function_name", + /* 328 */ "agg_func_opt ::=", + /* 329 */ "agg_func_opt ::= AGGREGATE", + /* 330 */ "bufsize_opt ::=", + /* 331 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", + /* 332 */ "language_opt ::=", + /* 333 */ "language_opt ::= LANGUAGE NK_STRING", + /* 334 */ "or_replace_opt ::=", + /* 335 */ "or_replace_opt ::= OR REPLACE", + /* 336 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery", + /* 337 */ "cmd ::= DROP STREAM exists_opt stream_name", + /* 338 */ "cmd ::= PAUSE STREAM exists_opt stream_name", + /* 339 */ "cmd ::= RESUME STREAM exists_opt ignore_opt stream_name", + /* 340 */ "col_list_opt ::=", + /* 341 */ "col_list_opt ::= NK_LP col_name_list NK_RP", + /* 342 */ "tag_def_or_ref_opt ::=", + /* 343 */ "tag_def_or_ref_opt ::= tags_def", + /* 344 */ "tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP", + /* 345 */ "stream_options ::=", + /* 346 */ "stream_options ::= stream_options TRIGGER AT_ONCE", + /* 347 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", + /* 348 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", + /* 349 */ "stream_options ::= stream_options WATERMARK duration_literal", + /* 350 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", + /* 351 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", + /* 352 */ "stream_options ::= stream_options DELETE_MARK duration_literal", + /* 353 */ "stream_options ::= stream_options IGNORE UPDATE NK_INTEGER", + /* 354 */ "subtable_opt ::=", + /* 355 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", + /* 356 */ "ignore_opt ::=", + /* 357 */ "ignore_opt ::= IGNORE UNTREATED", + /* 358 */ "cmd ::= KILL CONNECTION NK_INTEGER", + /* 359 */ "cmd ::= KILL QUERY NK_STRING", + /* 360 */ "cmd ::= KILL TRANSACTION NK_INTEGER", + /* 361 */ "cmd ::= BALANCE VGROUP", + /* 362 */ "cmd ::= BALANCE VGROUP LEADER", + /* 363 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", + /* 364 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", + /* 365 */ "cmd ::= SPLIT VGROUP NK_INTEGER", + /* 366 */ "dnode_list ::= DNODE NK_INTEGER", + /* 367 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", + /* 368 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", + /* 369 */ "cmd ::= query_or_subquery", + /* 370 */ "cmd ::= insert_query", + /* 371 */ "insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", + /* 372 */ "insert_query ::= INSERT INTO full_table_name query_or_subquery", + /* 373 */ "literal ::= NK_INTEGER", + /* 374 */ "literal ::= NK_FLOAT", + /* 375 */ "literal ::= NK_STRING", + /* 376 */ "literal ::= NK_BOOL", + /* 377 */ "literal ::= TIMESTAMP NK_STRING", + /* 378 */ "literal ::= duration_literal", + /* 379 */ "literal ::= NULL", + /* 380 */ "literal ::= NK_QUESTION", + /* 381 */ "duration_literal ::= NK_VARIABLE", + /* 382 */ "signed ::= NK_INTEGER", + /* 383 */ "signed ::= NK_PLUS NK_INTEGER", + /* 384 */ "signed ::= NK_MINUS NK_INTEGER", + /* 385 */ "signed ::= NK_FLOAT", + /* 386 */ "signed ::= NK_PLUS NK_FLOAT", + /* 387 */ "signed ::= NK_MINUS NK_FLOAT", + /* 388 */ "signed_literal ::= signed", + /* 389 */ "signed_literal ::= NK_STRING", + /* 390 */ "signed_literal ::= NK_BOOL", + /* 391 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 392 */ "signed_literal ::= duration_literal", + /* 393 */ "signed_literal ::= NULL", + /* 394 */ "signed_literal ::= literal_func", + /* 395 */ "signed_literal ::= NK_QUESTION", + /* 396 */ "literal_list ::= signed_literal", + /* 397 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 398 */ "db_name ::= NK_ID", + /* 399 */ "table_name ::= NK_ID", + /* 400 */ "column_name ::= NK_ID", + /* 401 */ "function_name ::= NK_ID", + /* 402 */ "table_alias ::= NK_ID", + /* 403 */ "column_alias ::= NK_ID", + /* 404 */ "user_name ::= NK_ID", + /* 405 */ "topic_name ::= NK_ID", + /* 406 */ "stream_name ::= NK_ID", + /* 407 */ "cgroup_name ::= NK_ID", + /* 408 */ "index_name ::= NK_ID", + /* 409 */ "expr_or_subquery ::= expression", + /* 410 */ "expression ::= literal", + /* 411 */ "expression ::= pseudo_column", + /* 412 */ "expression ::= column_reference", + /* 413 */ "expression ::= function_expression", + /* 414 */ "expression ::= case_when_expression", + /* 415 */ "expression ::= NK_LP expression NK_RP", + /* 416 */ "expression ::= NK_PLUS expr_or_subquery", + /* 417 */ "expression ::= NK_MINUS expr_or_subquery", + /* 418 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", + /* 419 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", + /* 420 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", + /* 421 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", + /* 422 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", + /* 423 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 424 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", + /* 425 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", + /* 426 */ "expression_list ::= expr_or_subquery", + /* 427 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", + /* 428 */ "column_reference ::= column_name", + /* 429 */ "column_reference ::= table_name NK_DOT column_name", + /* 430 */ "pseudo_column ::= ROWTS", + /* 431 */ "pseudo_column ::= TBNAME", + /* 432 */ "pseudo_column ::= table_name NK_DOT TBNAME", + /* 433 */ "pseudo_column ::= QSTART", + /* 434 */ "pseudo_column ::= QEND", + /* 435 */ "pseudo_column ::= QDURATION", + /* 436 */ "pseudo_column ::= WSTART", + /* 437 */ "pseudo_column ::= WEND", + /* 438 */ "pseudo_column ::= WDURATION", + /* 439 */ "pseudo_column ::= IROWTS", + /* 440 */ "pseudo_column ::= ISFILLED", + /* 441 */ "pseudo_column ::= QTAGS", + /* 442 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 443 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 444 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", + /* 445 */ "function_expression ::= literal_func", + /* 446 */ "literal_func ::= noarg_func NK_LP NK_RP", + /* 447 */ "literal_func ::= NOW", + /* 448 */ "noarg_func ::= NOW", + /* 449 */ "noarg_func ::= TODAY", + /* 450 */ "noarg_func ::= TIMEZONE", + /* 451 */ "noarg_func ::= DATABASE", + /* 452 */ "noarg_func ::= CLIENT_VERSION", + /* 453 */ "noarg_func ::= SERVER_VERSION", + /* 454 */ "noarg_func ::= SERVER_STATUS", + /* 455 */ "noarg_func ::= CURRENT_USER", + /* 456 */ "noarg_func ::= USER", + /* 457 */ "star_func ::= COUNT", + /* 458 */ "star_func ::= FIRST", + /* 459 */ "star_func ::= LAST", + /* 460 */ "star_func ::= LAST_ROW", + /* 461 */ "star_func_para_list ::= NK_STAR", + /* 462 */ "star_func_para_list ::= other_para_list", + /* 463 */ "other_para_list ::= star_func_para", + /* 464 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 465 */ "star_func_para ::= expr_or_subquery", + /* 466 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 467 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", + /* 468 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", + /* 469 */ "when_then_list ::= when_then_expr", + /* 470 */ "when_then_list ::= when_then_list when_then_expr", + /* 471 */ "when_then_expr ::= WHEN common_expression THEN common_expression", + /* 472 */ "case_when_else_opt ::=", + /* 473 */ "case_when_else_opt ::= ELSE common_expression", + /* 474 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", + /* 475 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", + /* 476 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", + /* 477 */ "predicate ::= expr_or_subquery IS NULL", + /* 478 */ "predicate ::= expr_or_subquery IS NOT NULL", + /* 479 */ "predicate ::= expr_or_subquery in_op in_predicate_value", + /* 480 */ "compare_op ::= NK_LT", + /* 481 */ "compare_op ::= NK_GT", + /* 482 */ "compare_op ::= NK_LE", + /* 483 */ "compare_op ::= NK_GE", + /* 484 */ "compare_op ::= NK_NE", + /* 485 */ "compare_op ::= NK_EQ", + /* 486 */ "compare_op ::= LIKE", + /* 487 */ "compare_op ::= NOT LIKE", + /* 488 */ "compare_op ::= MATCH", + /* 489 */ "compare_op ::= NMATCH", + /* 490 */ "compare_op ::= CONTAINS", + /* 491 */ "in_op ::= IN", + /* 492 */ "in_op ::= NOT IN", + /* 493 */ "in_predicate_value ::= NK_LP literal_list NK_RP", + /* 494 */ "boolean_value_expression ::= boolean_primary", + /* 495 */ "boolean_value_expression ::= NOT boolean_primary", + /* 496 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 497 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 498 */ "boolean_primary ::= predicate", + /* 499 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 500 */ "common_expression ::= expr_or_subquery", + /* 501 */ "common_expression ::= boolean_value_expression", + /* 502 */ "from_clause_opt ::=", + /* 503 */ "from_clause_opt ::= FROM table_reference_list", + /* 504 */ "table_reference_list ::= table_reference", + /* 505 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 506 */ "table_reference ::= table_primary", + /* 507 */ "table_reference ::= joined_table", + /* 508 */ "table_primary ::= table_name alias_opt", + /* 509 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 510 */ "table_primary ::= subquery alias_opt", + /* 511 */ "table_primary ::= parenthesized_joined_table", + /* 512 */ "alias_opt ::=", + /* 513 */ "alias_opt ::= table_alias", + /* 514 */ "alias_opt ::= AS table_alias", + /* 515 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 516 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 517 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 518 */ "join_type ::=", + /* 519 */ "join_type ::= INNER", + /* 520 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 521 */ "set_quantifier_opt ::=", + /* 522 */ "set_quantifier_opt ::= DISTINCT", + /* 523 */ "set_quantifier_opt ::= ALL", + /* 524 */ "select_list ::= select_item", + /* 525 */ "select_list ::= select_list NK_COMMA select_item", + /* 526 */ "select_item ::= NK_STAR", + /* 527 */ "select_item ::= common_expression", + /* 528 */ "select_item ::= common_expression column_alias", + /* 529 */ "select_item ::= common_expression AS column_alias", + /* 530 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 531 */ "where_clause_opt ::=", + /* 532 */ "where_clause_opt ::= WHERE search_condition", + /* 533 */ "partition_by_clause_opt ::=", + /* 534 */ "partition_by_clause_opt ::= PARTITION BY partition_list", + /* 535 */ "partition_list ::= partition_item", + /* 536 */ "partition_list ::= partition_list NK_COMMA partition_item", + /* 537 */ "partition_item ::= expr_or_subquery", + /* 538 */ "partition_item ::= expr_or_subquery column_alias", + /* 539 */ "partition_item ::= expr_or_subquery AS column_alias", + /* 540 */ "twindow_clause_opt ::=", + /* 541 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 542 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", + /* 543 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 544 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 545 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", + /* 546 */ "sliding_opt ::=", + /* 547 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 548 */ "fill_opt ::=", + /* 549 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 550 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP", + /* 551 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP", + /* 552 */ "fill_mode ::= NONE", + /* 553 */ "fill_mode ::= PREV", + /* 554 */ "fill_mode ::= NULL", + /* 555 */ "fill_mode ::= NULL_F", + /* 556 */ "fill_mode ::= LINEAR", + /* 557 */ "fill_mode ::= NEXT", + /* 558 */ "group_by_clause_opt ::=", + /* 559 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 560 */ "group_by_list ::= expr_or_subquery", + /* 561 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", + /* 562 */ "having_clause_opt ::=", + /* 563 */ "having_clause_opt ::= HAVING search_condition", + /* 564 */ "range_opt ::=", + /* 565 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", + /* 566 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_RP", + /* 567 */ "every_opt ::=", + /* 568 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", + /* 569 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 570 */ "query_simple ::= query_specification", + /* 571 */ "query_simple ::= union_query_expression", + /* 572 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", + /* 573 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", + /* 574 */ "query_simple_or_subquery ::= query_simple", + /* 575 */ "query_simple_or_subquery ::= subquery", + /* 576 */ "query_or_subquery ::= query_expression", + /* 577 */ "query_or_subquery ::= subquery", + /* 578 */ "order_by_clause_opt ::=", + /* 579 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 580 */ "slimit_clause_opt ::=", + /* 581 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 582 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 583 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 584 */ "limit_clause_opt ::=", + /* 585 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 586 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 587 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 588 */ "subquery ::= NK_LP query_expression NK_RP", + /* 589 */ "subquery ::= NK_LP subquery NK_RP", + /* 590 */ "search_condition ::= common_expression", + /* 591 */ "sort_specification_list ::= sort_specification", + /* 592 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 593 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", + /* 594 */ "ordering_specification_opt ::=", + /* 595 */ "ordering_specification_opt ::= ASC", + /* 596 */ "ordering_specification_opt ::= DESC", + /* 597 */ "null_ordering_opt ::=", + /* 598 */ "null_ordering_opt ::= NULLS FIRST", + /* 599 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2703,241 +2697,213 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 335: /* cmd */ - case 338: /* literal */ - case 344: /* with_opt */ - case 350: /* search_condition */ - case 354: /* db_options */ - case 356: /* alter_db_options */ - case 358: /* start_opt */ - case 359: /* end_opt */ - case 363: /* signed */ - case 365: /* retention */ - case 366: /* full_table_name */ - case 369: /* table_options */ - case 373: /* alter_table_clause */ - case 374: /* alter_table_options */ - case 377: /* signed_literal */ - case 378: /* create_subtable_clause */ - case 381: /* drop_table_clause */ - case 383: /* column_def */ - case 387: /* duration_literal */ - case 388: /* rollup_func_name */ - case 390: /* col_name */ - case 391: /* db_name_cond_opt */ - case 392: /* like_pattern_opt */ - case 393: /* table_name_cond */ - case 394: /* from_db_opt */ - case 396: /* tag_item */ - case 398: /* full_index_name */ - case 399: /* index_options */ - case 402: /* sliding_opt */ - case 403: /* sma_stream_opt */ - case 404: /* func */ - case 406: /* query_or_subquery */ - case 409: /* explain_options */ - case 410: /* insert_query */ - case 416: /* stream_options */ - case 419: /* subtable_opt */ - case 421: /* expression */ - case 423: /* where_clause_opt */ - case 424: /* literal_func */ - case 427: /* expr_or_subquery */ - case 428: /* pseudo_column */ - case 429: /* column_reference */ - case 430: /* function_expression */ - case 431: /* case_when_expression */ - case 436: /* star_func_para */ - case 438: /* case_when_else_opt */ - case 439: /* common_expression */ - case 440: /* when_then_expr */ - case 441: /* predicate */ - case 444: /* in_predicate_value */ - case 445: /* boolean_value_expression */ - case 446: /* boolean_primary */ - case 447: /* from_clause_opt */ - case 448: /* table_reference_list */ - case 449: /* table_reference */ - case 450: /* table_primary */ - case 451: /* joined_table */ - case 453: /* subquery */ - case 454: /* parenthesized_joined_table */ - case 456: /* query_specification */ - case 460: /* range_opt */ - case 461: /* every_opt */ - case 462: /* fill_opt */ - case 463: /* twindow_clause_opt */ - case 465: /* having_clause_opt */ - case 466: /* select_item */ - case 468: /* partition_item */ - case 471: /* query_expression */ - case 472: /* query_simple */ - case 474: /* slimit_clause_opt */ - case 475: /* limit_clause_opt */ - case 476: /* union_query_expression */ - case 477: /* query_simple_or_subquery */ - case 479: /* sort_specification */ + case 338: /* cmd */ + case 341: /* literal */ + case 347: /* with_opt */ + case 353: /* search_condition */ + case 358: /* db_options */ + case 360: /* alter_db_options */ + case 362: /* start_opt */ + case 363: /* end_opt */ + case 367: /* signed */ + case 369: /* retention */ + case 370: /* full_table_name */ + case 373: /* table_options */ + case 377: /* alter_table_clause */ + case 378: /* alter_table_options */ + case 381: /* signed_literal */ + case 382: /* create_subtable_clause */ + case 385: /* drop_table_clause */ + case 387: /* column_def */ + case 391: /* duration_literal */ + case 392: /* rollup_func_name */ + case 394: /* col_name */ + case 395: /* db_name_cond_opt */ + case 396: /* like_pattern_opt */ + case 397: /* table_name_cond */ + case 398: /* from_db_opt */ + case 400: /* tag_item */ + case 402: /* full_index_name */ + case 403: /* index_options */ + case 406: /* sliding_opt */ + case 407: /* sma_stream_opt */ + case 408: /* func */ + case 411: /* query_or_subquery */ + case 412: /* where_clause_opt */ + case 415: /* explain_options */ + case 416: /* insert_query */ + case 422: /* stream_options */ + case 425: /* subtable_opt */ + case 427: /* expression */ + case 429: /* literal_func */ + case 432: /* expr_or_subquery */ + case 433: /* pseudo_column */ + case 434: /* column_reference */ + case 435: /* function_expression */ + case 436: /* case_when_expression */ + case 441: /* star_func_para */ + case 443: /* case_when_else_opt */ + case 444: /* common_expression */ + case 445: /* when_then_expr */ + case 446: /* predicate */ + case 449: /* in_predicate_value */ + case 450: /* boolean_value_expression */ + case 451: /* boolean_primary */ + case 452: /* from_clause_opt */ + case 453: /* table_reference_list */ + case 454: /* table_reference */ + case 455: /* table_primary */ + case 456: /* joined_table */ + case 458: /* subquery */ + case 459: /* parenthesized_joined_table */ + case 461: /* query_specification */ + case 465: /* range_opt */ + case 466: /* every_opt */ + case 467: /* fill_opt */ + case 468: /* twindow_clause_opt */ + case 470: /* having_clause_opt */ + case 471: /* select_item */ + case 473: /* partition_item */ + case 476: /* query_expression */ + case 477: /* query_simple */ + case 479: /* slimit_clause_opt */ + case 480: /* limit_clause_opt */ + case 481: /* union_query_expression */ + case 482: /* query_simple_or_subquery */ + case 484: /* sort_specification */ { -#line 7 "sql.y" - nodesDestroyNode((yypminor->yy164)); -#line 2784 "sql.c" + nodesDestroyNode((yypminor->yy452)); } break; - case 336: /* account_options */ - case 337: /* alter_account_options */ - case 339: /* alter_account_option */ - case 357: /* speed_opt */ - case 413: /* bufsize_opt */ + case 339: /* account_options */ + case 340: /* alter_account_options */ + case 342: /* alter_account_option */ + case 361: /* speed_opt */ + case 410: /* with_meta */ + case 419: /* bufsize_opt */ { -#line 54 "sql.y" -#line 2795 "sql.c" } break; - case 340: /* user_name */ - case 347: /* db_name */ - case 348: /* table_name */ - case 349: /* topic_name */ - case 351: /* dnode_endpoint */ - case 375: /* column_name */ - case 389: /* function_name */ - case 397: /* column_alias */ - case 400: /* index_name */ - case 405: /* sma_func_name */ - case 407: /* cgroup_name */ - case 414: /* language_opt */ - case 415: /* stream_name */ - case 426: /* table_alias */ - case 432: /* star_func */ - case 434: /* noarg_func */ - case 452: /* alias_opt */ + case 343: /* user_name */ + case 350: /* db_name */ + case 351: /* table_name */ + case 352: /* topic_name */ + case 354: /* dnode_endpoint */ + case 379: /* column_name */ + case 393: /* function_name */ + case 401: /* column_alias */ + case 404: /* index_name */ + case 409: /* sma_func_name */ + case 413: /* cgroup_name */ + case 420: /* language_opt */ + case 421: /* stream_name */ + case 431: /* table_alias */ + case 437: /* star_func */ + case 439: /* noarg_func */ + case 457: /* alias_opt */ { -#line 728 "sql.y" -#line 2818 "sql.c" } break; - case 341: /* sysinfo_opt */ + case 344: /* sysinfo_opt */ { -#line 92 "sql.y" -#line 2825 "sql.c" } break; - case 342: /* privileges */ - case 345: /* priv_type_list */ - case 346: /* priv_type */ + case 345: /* privileges */ + case 348: /* priv_type_list */ + case 349: /* priv_type */ { -#line 101 "sql.y" -#line 2834 "sql.c" } break; - case 343: /* priv_level */ + case 346: /* priv_level */ { -#line 117 "sql.y" -#line 2841 "sql.c" } break; - case 352: /* force_opt */ - case 353: /* not_exists_opt */ - case 355: /* exists_opt */ - case 408: /* analyze_opt */ - case 411: /* or_replace_opt */ - case 412: /* agg_func_opt */ - case 420: /* ignore_opt */ - case 457: /* set_quantifier_opt */ + case 355: /* force_opt */ + case 356: /* unsafe_opt */ + case 357: /* not_exists_opt */ + case 359: /* exists_opt */ + case 414: /* analyze_opt */ + case 417: /* or_replace_opt */ + case 418: /* agg_func_opt */ + case 426: /* ignore_opt */ + case 462: /* set_quantifier_opt */ { -#line 144 "sql.y" -#line 2855 "sql.c" } break; - case 360: /* integer_list */ - case 361: /* variable_list */ - case 362: /* retention_list */ - case 367: /* column_def_list */ - case 368: /* tags_def_opt */ - case 370: /* multi_create_clause */ - case 371: /* tags_def */ - case 372: /* multi_drop_clause */ - case 379: /* specific_cols_opt */ - case 380: /* expression_list */ - case 382: /* col_name_list */ - case 384: /* duration_list */ - case 385: /* rollup_func_list */ - case 395: /* tag_list_opt */ - case 401: /* func_list */ - case 417: /* col_list_opt */ - case 418: /* tag_def_or_ref_opt */ - case 422: /* dnode_list */ - case 425: /* literal_list */ - case 433: /* star_func_para_list */ - case 435: /* other_para_list */ - case 437: /* when_then_list */ - case 458: /* select_list */ - case 459: /* partition_by_clause_opt */ - case 464: /* group_by_clause_opt */ - case 467: /* partition_list */ - case 470: /* group_by_list */ - case 473: /* order_by_clause_opt */ - case 478: /* sort_specification_list */ + case 364: /* integer_list */ + case 365: /* variable_list */ + case 366: /* retention_list */ + case 371: /* column_def_list */ + case 372: /* tags_def_opt */ + case 374: /* multi_create_clause */ + case 375: /* tags_def */ + case 376: /* multi_drop_clause */ + case 383: /* specific_cols_opt */ + case 384: /* expression_list */ + case 386: /* col_name_list */ + case 388: /* duration_list */ + case 389: /* rollup_func_list */ + case 399: /* tag_list_opt */ + case 405: /* func_list */ + case 423: /* col_list_opt */ + case 424: /* tag_def_or_ref_opt */ + case 428: /* dnode_list */ + case 430: /* literal_list */ + case 438: /* star_func_para_list */ + case 440: /* other_para_list */ + case 442: /* when_then_list */ + case 463: /* select_list */ + case 464: /* partition_by_clause_opt */ + case 469: /* group_by_clause_opt */ + case 472: /* partition_list */ + case 475: /* group_by_list */ + case 478: /* order_by_clause_opt */ + case 483: /* sort_specification_list */ { -#line 264 "sql.y" - nodesDestroyList((yypminor->yy72)); -#line 2890 "sql.c" + nodesDestroyList((yypminor->yy812)); } break; - case 364: /* alter_db_option */ - case 386: /* alter_table_option */ + case 368: /* alter_db_option */ + case 390: /* alter_table_option */ { -#line 237 "sql.y" -#line 2898 "sql.c" } break; - case 376: /* type_name */ + case 380: /* type_name */ { -#line 358 "sql.y" -#line 2905 "sql.c" } break; - case 442: /* compare_op */ - case 443: /* in_op */ + case 447: /* compare_op */ + case 448: /* in_op */ { -#line 916 "sql.y" -#line 2913 "sql.c" } break; - case 455: /* join_type */ + case 460: /* join_type */ { -#line 992 "sql.y" -#line 2920 "sql.c" } break; - case 469: /* fill_mode */ + case 474: /* fill_mode */ { -#line 1067 "sql.y" -#line 2927 "sql.c" } break; - case 480: /* ordering_specification_opt */ + case 485: /* ordering_specification_opt */ { -#line 1150 "sql.y" -#line 2934 "sql.c" } break; - case 481: /* null_ordering_opt */ + case 486: /* null_ordering_opt */ { -#line 1156 "sql.y" -#line 2941 "sql.c" } break; /********* End destructor definitions *****************************************/ @@ -3226,597 +3192,606 @@ static void yy_shift( /* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side ** of that rule */ static const YYCODETYPE yyRuleInfoLhs[] = { - 335, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ - 335, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ - 336, /* (2) account_options ::= */ - 336, /* (3) account_options ::= account_options PPS literal */ - 336, /* (4) account_options ::= account_options TSERIES literal */ - 336, /* (5) account_options ::= account_options STORAGE literal */ - 336, /* (6) account_options ::= account_options STREAMS literal */ - 336, /* (7) account_options ::= account_options QTIME literal */ - 336, /* (8) account_options ::= account_options DBS literal */ - 336, /* (9) account_options ::= account_options USERS literal */ - 336, /* (10) account_options ::= account_options CONNS literal */ - 336, /* (11) account_options ::= account_options STATE literal */ - 337, /* (12) alter_account_options ::= alter_account_option */ - 337, /* (13) alter_account_options ::= alter_account_options alter_account_option */ - 339, /* (14) alter_account_option ::= PASS literal */ - 339, /* (15) alter_account_option ::= PPS literal */ - 339, /* (16) alter_account_option ::= TSERIES literal */ - 339, /* (17) alter_account_option ::= STORAGE literal */ - 339, /* (18) alter_account_option ::= STREAMS literal */ - 339, /* (19) alter_account_option ::= QTIME literal */ - 339, /* (20) alter_account_option ::= DBS literal */ - 339, /* (21) alter_account_option ::= USERS literal */ - 339, /* (22) alter_account_option ::= CONNS literal */ - 339, /* (23) alter_account_option ::= STATE literal */ - 335, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ - 335, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ - 335, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ - 335, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ - 335, /* (28) cmd ::= DROP USER user_name */ - 341, /* (29) sysinfo_opt ::= */ - 341, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ - 335, /* (31) cmd ::= GRANT privileges ON priv_level with_opt TO user_name */ - 335, /* (32) cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name */ - 342, /* (33) privileges ::= ALL */ - 342, /* (34) privileges ::= priv_type_list */ - 342, /* (35) privileges ::= SUBSCRIBE */ - 345, /* (36) priv_type_list ::= priv_type */ - 345, /* (37) priv_type_list ::= priv_type_list NK_COMMA priv_type */ - 346, /* (38) priv_type ::= READ */ - 346, /* (39) priv_type ::= WRITE */ - 343, /* (40) priv_level ::= NK_STAR NK_DOT NK_STAR */ - 343, /* (41) priv_level ::= db_name NK_DOT NK_STAR */ - 343, /* (42) priv_level ::= db_name NK_DOT table_name */ - 343, /* (43) priv_level ::= topic_name */ - 344, /* (44) with_opt ::= */ - 344, /* (45) with_opt ::= WITH search_condition */ - 335, /* (46) cmd ::= CREATE DNODE dnode_endpoint */ - 335, /* (47) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ - 335, /* (48) cmd ::= DROP DNODE NK_INTEGER force_opt */ - 335, /* (49) cmd ::= DROP DNODE dnode_endpoint force_opt */ - 335, /* (50) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ - 335, /* (51) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ - 335, /* (52) cmd ::= ALTER ALL DNODES NK_STRING */ - 335, /* (53) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ - 335, /* (54) cmd ::= RESTORE DNODE NK_INTEGER */ - 351, /* (55) dnode_endpoint ::= NK_STRING */ - 351, /* (56) dnode_endpoint ::= NK_ID */ - 351, /* (57) dnode_endpoint ::= NK_IPTOKEN */ - 352, /* (58) force_opt ::= */ - 352, /* (59) force_opt ::= FORCE */ - 335, /* (60) cmd ::= ALTER LOCAL NK_STRING */ - 335, /* (61) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ - 335, /* (62) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ - 335, /* (63) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ - 335, /* (64) cmd ::= RESTORE QNODE ON DNODE NK_INTEGER */ - 335, /* (65) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ - 335, /* (66) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ - 335, /* (67) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ - 335, /* (68) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ - 335, /* (69) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ - 335, /* (70) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ - 335, /* (71) cmd ::= RESTORE MNODE ON DNODE NK_INTEGER */ - 335, /* (72) cmd ::= RESTORE VNODE ON DNODE NK_INTEGER */ - 335, /* (73) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - 335, /* (74) cmd ::= DROP DATABASE exists_opt db_name */ - 335, /* (75) cmd ::= USE db_name */ - 335, /* (76) cmd ::= ALTER DATABASE db_name alter_db_options */ - 335, /* (77) cmd ::= FLUSH DATABASE db_name */ - 335, /* (78) cmd ::= TRIM DATABASE db_name speed_opt */ - 335, /* (79) cmd ::= COMPACT DATABASE db_name start_opt end_opt */ - 353, /* (80) not_exists_opt ::= IF NOT EXISTS */ - 353, /* (81) not_exists_opt ::= */ - 355, /* (82) exists_opt ::= IF EXISTS */ - 355, /* (83) exists_opt ::= */ - 354, /* (84) db_options ::= */ - 354, /* (85) db_options ::= db_options BUFFER NK_INTEGER */ - 354, /* (86) db_options ::= db_options CACHEMODEL NK_STRING */ - 354, /* (87) db_options ::= db_options CACHESIZE NK_INTEGER */ - 354, /* (88) db_options ::= db_options COMP NK_INTEGER */ - 354, /* (89) db_options ::= db_options DURATION NK_INTEGER */ - 354, /* (90) db_options ::= db_options DURATION NK_VARIABLE */ - 354, /* (91) db_options ::= db_options MAXROWS NK_INTEGER */ - 354, /* (92) db_options ::= db_options MINROWS NK_INTEGER */ - 354, /* (93) db_options ::= db_options KEEP integer_list */ - 354, /* (94) db_options ::= db_options KEEP variable_list */ - 354, /* (95) db_options ::= db_options PAGES NK_INTEGER */ - 354, /* (96) db_options ::= db_options PAGESIZE NK_INTEGER */ - 354, /* (97) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ - 354, /* (98) db_options ::= db_options PRECISION NK_STRING */ - 354, /* (99) db_options ::= db_options REPLICA NK_INTEGER */ - 354, /* (100) db_options ::= db_options VGROUPS NK_INTEGER */ - 354, /* (101) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - 354, /* (102) db_options ::= db_options RETENTIONS retention_list */ - 354, /* (103) db_options ::= db_options SCHEMALESS NK_INTEGER */ - 354, /* (104) db_options ::= db_options WAL_LEVEL NK_INTEGER */ - 354, /* (105) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ - 354, /* (106) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ - 354, /* (107) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ - 354, /* (108) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ - 354, /* (109) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ - 354, /* (110) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ - 354, /* (111) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ - 354, /* (112) db_options ::= db_options STT_TRIGGER NK_INTEGER */ - 354, /* (113) db_options ::= db_options TABLE_PREFIX signed */ - 354, /* (114) db_options ::= db_options TABLE_SUFFIX signed */ - 356, /* (115) alter_db_options ::= alter_db_option */ - 356, /* (116) alter_db_options ::= alter_db_options alter_db_option */ - 364, /* (117) alter_db_option ::= BUFFER NK_INTEGER */ - 364, /* (118) alter_db_option ::= CACHEMODEL NK_STRING */ - 364, /* (119) alter_db_option ::= CACHESIZE NK_INTEGER */ - 364, /* (120) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ - 364, /* (121) alter_db_option ::= KEEP integer_list */ - 364, /* (122) alter_db_option ::= KEEP variable_list */ - 364, /* (123) alter_db_option ::= PAGES NK_INTEGER */ - 364, /* (124) alter_db_option ::= REPLICA NK_INTEGER */ - 364, /* (125) alter_db_option ::= WAL_LEVEL NK_INTEGER */ - 364, /* (126) alter_db_option ::= STT_TRIGGER NK_INTEGER */ - 364, /* (127) alter_db_option ::= MINROWS NK_INTEGER */ - 364, /* (128) alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ - 364, /* (129) alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ - 364, /* (130) alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ - 364, /* (131) alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ - 360, /* (132) integer_list ::= NK_INTEGER */ - 360, /* (133) integer_list ::= integer_list NK_COMMA NK_INTEGER */ - 361, /* (134) variable_list ::= NK_VARIABLE */ - 361, /* (135) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ - 362, /* (136) retention_list ::= retention */ - 362, /* (137) retention_list ::= retention_list NK_COMMA retention */ - 365, /* (138) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ - 357, /* (139) speed_opt ::= */ - 357, /* (140) speed_opt ::= MAX_SPEED NK_INTEGER */ - 358, /* (141) start_opt ::= */ - 358, /* (142) start_opt ::= START WITH NK_INTEGER */ - 358, /* (143) start_opt ::= START WITH NK_STRING */ - 358, /* (144) start_opt ::= START WITH TIMESTAMP NK_STRING */ - 359, /* (145) end_opt ::= */ - 359, /* (146) end_opt ::= END WITH NK_INTEGER */ - 359, /* (147) end_opt ::= END WITH NK_STRING */ - 359, /* (148) end_opt ::= END WITH TIMESTAMP NK_STRING */ - 335, /* (149) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - 335, /* (150) cmd ::= CREATE TABLE multi_create_clause */ - 335, /* (151) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - 335, /* (152) cmd ::= DROP TABLE multi_drop_clause */ - 335, /* (153) cmd ::= DROP STABLE exists_opt full_table_name */ - 335, /* (154) cmd ::= ALTER TABLE alter_table_clause */ - 335, /* (155) cmd ::= ALTER STABLE alter_table_clause */ - 373, /* (156) alter_table_clause ::= full_table_name alter_table_options */ - 373, /* (157) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ - 373, /* (158) alter_table_clause ::= full_table_name DROP COLUMN column_name */ - 373, /* (159) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ - 373, /* (160) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ - 373, /* (161) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ - 373, /* (162) alter_table_clause ::= full_table_name DROP TAG column_name */ - 373, /* (163) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ - 373, /* (164) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ - 373, /* (165) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ - 370, /* (166) multi_create_clause ::= create_subtable_clause */ - 370, /* (167) multi_create_clause ::= multi_create_clause create_subtable_clause */ - 378, /* (168) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ - 372, /* (169) multi_drop_clause ::= drop_table_clause */ - 372, /* (170) multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ - 381, /* (171) drop_table_clause ::= exists_opt full_table_name */ - 379, /* (172) specific_cols_opt ::= */ - 379, /* (173) specific_cols_opt ::= NK_LP col_name_list NK_RP */ - 366, /* (174) full_table_name ::= table_name */ - 366, /* (175) full_table_name ::= db_name NK_DOT table_name */ - 367, /* (176) column_def_list ::= column_def */ - 367, /* (177) column_def_list ::= column_def_list NK_COMMA column_def */ - 383, /* (178) column_def ::= column_name type_name */ - 376, /* (179) type_name ::= BOOL */ - 376, /* (180) type_name ::= TINYINT */ - 376, /* (181) type_name ::= SMALLINT */ - 376, /* (182) type_name ::= INT */ - 376, /* (183) type_name ::= INTEGER */ - 376, /* (184) type_name ::= BIGINT */ - 376, /* (185) type_name ::= FLOAT */ - 376, /* (186) type_name ::= DOUBLE */ - 376, /* (187) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - 376, /* (188) type_name ::= TIMESTAMP */ - 376, /* (189) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - 376, /* (190) type_name ::= TINYINT UNSIGNED */ - 376, /* (191) type_name ::= SMALLINT UNSIGNED */ - 376, /* (192) type_name ::= INT UNSIGNED */ - 376, /* (193) type_name ::= BIGINT UNSIGNED */ - 376, /* (194) type_name ::= JSON */ - 376, /* (195) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - 376, /* (196) type_name ::= MEDIUMBLOB */ - 376, /* (197) type_name ::= BLOB */ - 376, /* (198) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - 376, /* (199) type_name ::= DECIMAL */ - 376, /* (200) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - 376, /* (201) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - 368, /* (202) tags_def_opt ::= */ - 368, /* (203) tags_def_opt ::= tags_def */ - 371, /* (204) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - 369, /* (205) table_options ::= */ - 369, /* (206) table_options ::= table_options COMMENT NK_STRING */ - 369, /* (207) table_options ::= table_options MAX_DELAY duration_list */ - 369, /* (208) table_options ::= table_options WATERMARK duration_list */ - 369, /* (209) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ - 369, /* (210) table_options ::= table_options TTL NK_INTEGER */ - 369, /* (211) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - 369, /* (212) table_options ::= table_options DELETE_MARK duration_list */ - 374, /* (213) alter_table_options ::= alter_table_option */ - 374, /* (214) alter_table_options ::= alter_table_options alter_table_option */ - 386, /* (215) alter_table_option ::= COMMENT NK_STRING */ - 386, /* (216) alter_table_option ::= TTL NK_INTEGER */ - 384, /* (217) duration_list ::= duration_literal */ - 384, /* (218) duration_list ::= duration_list NK_COMMA duration_literal */ - 385, /* (219) rollup_func_list ::= rollup_func_name */ - 385, /* (220) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ - 388, /* (221) rollup_func_name ::= function_name */ - 388, /* (222) rollup_func_name ::= FIRST */ - 388, /* (223) rollup_func_name ::= LAST */ - 382, /* (224) col_name_list ::= col_name */ - 382, /* (225) col_name_list ::= col_name_list NK_COMMA col_name */ - 390, /* (226) col_name ::= column_name */ - 335, /* (227) cmd ::= SHOW DNODES */ - 335, /* (228) cmd ::= SHOW USERS */ - 335, /* (229) cmd ::= SHOW USER PRIVILEGES */ - 335, /* (230) cmd ::= SHOW DATABASES */ - 335, /* (231) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ - 335, /* (232) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ - 335, /* (233) cmd ::= SHOW db_name_cond_opt VGROUPS */ - 335, /* (234) cmd ::= SHOW MNODES */ - 335, /* (235) cmd ::= SHOW QNODES */ - 335, /* (236) cmd ::= SHOW FUNCTIONS */ - 335, /* (237) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - 335, /* (238) cmd ::= SHOW STREAMS */ - 335, /* (239) cmd ::= SHOW ACCOUNTS */ - 335, /* (240) cmd ::= SHOW APPS */ - 335, /* (241) cmd ::= SHOW CONNECTIONS */ - 335, /* (242) cmd ::= SHOW LICENCES */ - 335, /* (243) cmd ::= SHOW GRANTS */ - 335, /* (244) cmd ::= SHOW CREATE DATABASE db_name */ - 335, /* (245) cmd ::= SHOW CREATE TABLE full_table_name */ - 335, /* (246) cmd ::= SHOW CREATE STABLE full_table_name */ - 335, /* (247) cmd ::= SHOW QUERIES */ - 335, /* (248) cmd ::= SHOW SCORES */ - 335, /* (249) cmd ::= SHOW TOPICS */ - 335, /* (250) cmd ::= SHOW VARIABLES */ - 335, /* (251) cmd ::= SHOW CLUSTER VARIABLES */ - 335, /* (252) cmd ::= SHOW LOCAL VARIABLES */ - 335, /* (253) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ - 335, /* (254) cmd ::= SHOW BNODES */ - 335, /* (255) cmd ::= SHOW SNODES */ - 335, /* (256) cmd ::= SHOW CLUSTER */ - 335, /* (257) cmd ::= SHOW TRANSACTIONS */ - 335, /* (258) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ - 335, /* (259) cmd ::= SHOW CONSUMERS */ - 335, /* (260) cmd ::= SHOW SUBSCRIPTIONS */ - 335, /* (261) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ - 335, /* (262) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ - 335, /* (263) cmd ::= SHOW VNODES NK_INTEGER */ - 335, /* (264) cmd ::= SHOW VNODES NK_STRING */ - 335, /* (265) cmd ::= SHOW db_name_cond_opt ALIVE */ - 335, /* (266) cmd ::= SHOW CLUSTER ALIVE */ - 391, /* (267) db_name_cond_opt ::= */ - 391, /* (268) db_name_cond_opt ::= db_name NK_DOT */ - 392, /* (269) like_pattern_opt ::= */ - 392, /* (270) like_pattern_opt ::= LIKE NK_STRING */ - 393, /* (271) table_name_cond ::= table_name */ - 394, /* (272) from_db_opt ::= */ - 394, /* (273) from_db_opt ::= FROM db_name */ - 395, /* (274) tag_list_opt ::= */ - 395, /* (275) tag_list_opt ::= tag_item */ - 395, /* (276) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ - 396, /* (277) tag_item ::= TBNAME */ - 396, /* (278) tag_item ::= QTAGS */ - 396, /* (279) tag_item ::= column_name */ - 396, /* (280) tag_item ::= column_name column_alias */ - 396, /* (281) tag_item ::= column_name AS column_alias */ - 335, /* (282) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ - 335, /* (283) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ - 335, /* (284) cmd ::= DROP INDEX exists_opt full_index_name */ - 398, /* (285) full_index_name ::= index_name */ - 398, /* (286) full_index_name ::= db_name NK_DOT index_name */ - 399, /* (287) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ - 399, /* (288) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ - 401, /* (289) func_list ::= func */ - 401, /* (290) func_list ::= func_list NK_COMMA func */ - 404, /* (291) func ::= sma_func_name NK_LP expression_list NK_RP */ - 405, /* (292) sma_func_name ::= function_name */ - 405, /* (293) sma_func_name ::= COUNT */ - 405, /* (294) sma_func_name ::= FIRST */ - 405, /* (295) sma_func_name ::= LAST */ - 405, /* (296) sma_func_name ::= LAST_ROW */ - 403, /* (297) sma_stream_opt ::= */ - 403, /* (298) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ - 403, /* (299) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ - 403, /* (300) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ - 335, /* (301) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ - 335, /* (302) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ - 335, /* (303) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - 335, /* (304) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ - 335, /* (305) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ - 335, /* (306) cmd ::= DROP TOPIC exists_opt topic_name */ - 335, /* (307) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ - 335, /* (308) cmd ::= DESC full_table_name */ - 335, /* (309) cmd ::= DESCRIBE full_table_name */ - 335, /* (310) cmd ::= RESET QUERY CACHE */ - 335, /* (311) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - 335, /* (312) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ - 408, /* (313) analyze_opt ::= */ - 408, /* (314) analyze_opt ::= ANALYZE */ - 409, /* (315) explain_options ::= */ - 409, /* (316) explain_options ::= explain_options VERBOSE NK_BOOL */ - 409, /* (317) explain_options ::= explain_options RATIO NK_FLOAT */ - 335, /* (318) cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ - 335, /* (319) cmd ::= DROP FUNCTION exists_opt function_name */ - 412, /* (320) agg_func_opt ::= */ - 412, /* (321) agg_func_opt ::= AGGREGATE */ - 413, /* (322) bufsize_opt ::= */ - 413, /* (323) bufsize_opt ::= BUFSIZE NK_INTEGER */ - 414, /* (324) language_opt ::= */ - 414, /* (325) language_opt ::= LANGUAGE NK_STRING */ - 411, /* (326) or_replace_opt ::= */ - 411, /* (327) or_replace_opt ::= OR REPLACE */ - 335, /* (328) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ - 335, /* (329) cmd ::= DROP STREAM exists_opt stream_name */ - 335, /* (330) cmd ::= PAUSE STREAM exists_opt stream_name */ - 335, /* (331) cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */ - 417, /* (332) col_list_opt ::= */ - 417, /* (333) col_list_opt ::= NK_LP col_name_list NK_RP */ - 418, /* (334) tag_def_or_ref_opt ::= */ - 418, /* (335) tag_def_or_ref_opt ::= tags_def */ - 418, /* (336) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ - 416, /* (337) stream_options ::= */ - 416, /* (338) stream_options ::= stream_options TRIGGER AT_ONCE */ - 416, /* (339) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - 416, /* (340) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - 416, /* (341) stream_options ::= stream_options WATERMARK duration_literal */ - 416, /* (342) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ - 416, /* (343) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ - 416, /* (344) stream_options ::= stream_options DELETE_MARK duration_literal */ - 416, /* (345) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ - 419, /* (346) subtable_opt ::= */ - 419, /* (347) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - 420, /* (348) ignore_opt ::= */ - 420, /* (349) ignore_opt ::= IGNORE UNTREATED */ - 335, /* (350) cmd ::= KILL CONNECTION NK_INTEGER */ - 335, /* (351) cmd ::= KILL QUERY NK_STRING */ - 335, /* (352) cmd ::= KILL TRANSACTION NK_INTEGER */ - 335, /* (353) cmd ::= BALANCE VGROUP */ - 335, /* (354) cmd ::= BALANCE VGROUP LEADER */ - 335, /* (355) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ - 335, /* (356) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ - 335, /* (357) cmd ::= SPLIT VGROUP NK_INTEGER */ - 422, /* (358) dnode_list ::= DNODE NK_INTEGER */ - 422, /* (359) dnode_list ::= dnode_list DNODE NK_INTEGER */ - 335, /* (360) cmd ::= DELETE FROM full_table_name where_clause_opt */ - 335, /* (361) cmd ::= query_or_subquery */ - 335, /* (362) cmd ::= insert_query */ - 410, /* (363) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ - 410, /* (364) insert_query ::= INSERT INTO full_table_name query_or_subquery */ - 338, /* (365) literal ::= NK_INTEGER */ - 338, /* (366) literal ::= NK_FLOAT */ - 338, /* (367) literal ::= NK_STRING */ - 338, /* (368) literal ::= NK_BOOL */ - 338, /* (369) literal ::= TIMESTAMP NK_STRING */ - 338, /* (370) literal ::= duration_literal */ - 338, /* (371) literal ::= NULL */ - 338, /* (372) literal ::= NK_QUESTION */ - 387, /* (373) duration_literal ::= NK_VARIABLE */ - 363, /* (374) signed ::= NK_INTEGER */ - 363, /* (375) signed ::= NK_PLUS NK_INTEGER */ - 363, /* (376) signed ::= NK_MINUS NK_INTEGER */ - 363, /* (377) signed ::= NK_FLOAT */ - 363, /* (378) signed ::= NK_PLUS NK_FLOAT */ - 363, /* (379) signed ::= NK_MINUS NK_FLOAT */ - 377, /* (380) signed_literal ::= signed */ - 377, /* (381) signed_literal ::= NK_STRING */ - 377, /* (382) signed_literal ::= NK_BOOL */ - 377, /* (383) signed_literal ::= TIMESTAMP NK_STRING */ - 377, /* (384) signed_literal ::= duration_literal */ - 377, /* (385) signed_literal ::= NULL */ - 377, /* (386) signed_literal ::= literal_func */ - 377, /* (387) signed_literal ::= NK_QUESTION */ - 425, /* (388) literal_list ::= signed_literal */ - 425, /* (389) literal_list ::= literal_list NK_COMMA signed_literal */ - 347, /* (390) db_name ::= NK_ID */ - 348, /* (391) table_name ::= NK_ID */ - 375, /* (392) column_name ::= NK_ID */ - 389, /* (393) function_name ::= NK_ID */ - 426, /* (394) table_alias ::= NK_ID */ - 397, /* (395) column_alias ::= NK_ID */ - 340, /* (396) user_name ::= NK_ID */ - 349, /* (397) topic_name ::= NK_ID */ - 415, /* (398) stream_name ::= NK_ID */ - 407, /* (399) cgroup_name ::= NK_ID */ - 400, /* (400) index_name ::= NK_ID */ - 427, /* (401) expr_or_subquery ::= expression */ - 421, /* (402) expression ::= literal */ - 421, /* (403) expression ::= pseudo_column */ - 421, /* (404) expression ::= column_reference */ - 421, /* (405) expression ::= function_expression */ - 421, /* (406) expression ::= case_when_expression */ - 421, /* (407) expression ::= NK_LP expression NK_RP */ - 421, /* (408) expression ::= NK_PLUS expr_or_subquery */ - 421, /* (409) expression ::= NK_MINUS expr_or_subquery */ - 421, /* (410) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ - 421, /* (411) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ - 421, /* (412) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ - 421, /* (413) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ - 421, /* (414) expression ::= expr_or_subquery NK_REM expr_or_subquery */ - 421, /* (415) expression ::= column_reference NK_ARROW NK_STRING */ - 421, /* (416) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ - 421, /* (417) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ - 380, /* (418) expression_list ::= expr_or_subquery */ - 380, /* (419) expression_list ::= expression_list NK_COMMA expr_or_subquery */ - 429, /* (420) column_reference ::= column_name */ - 429, /* (421) column_reference ::= table_name NK_DOT column_name */ - 428, /* (422) pseudo_column ::= ROWTS */ - 428, /* (423) pseudo_column ::= TBNAME */ - 428, /* (424) pseudo_column ::= table_name NK_DOT TBNAME */ - 428, /* (425) pseudo_column ::= QSTART */ - 428, /* (426) pseudo_column ::= QEND */ - 428, /* (427) pseudo_column ::= QDURATION */ - 428, /* (428) pseudo_column ::= WSTART */ - 428, /* (429) pseudo_column ::= WEND */ - 428, /* (430) pseudo_column ::= WDURATION */ - 428, /* (431) pseudo_column ::= IROWTS */ - 428, /* (432) pseudo_column ::= ISFILLED */ - 428, /* (433) pseudo_column ::= QTAGS */ - 430, /* (434) function_expression ::= function_name NK_LP expression_list NK_RP */ - 430, /* (435) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - 430, /* (436) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ - 430, /* (437) function_expression ::= literal_func */ - 424, /* (438) literal_func ::= noarg_func NK_LP NK_RP */ - 424, /* (439) literal_func ::= NOW */ - 434, /* (440) noarg_func ::= NOW */ - 434, /* (441) noarg_func ::= TODAY */ - 434, /* (442) noarg_func ::= TIMEZONE */ - 434, /* (443) noarg_func ::= DATABASE */ - 434, /* (444) noarg_func ::= CLIENT_VERSION */ - 434, /* (445) noarg_func ::= SERVER_VERSION */ - 434, /* (446) noarg_func ::= SERVER_STATUS */ - 434, /* (447) noarg_func ::= CURRENT_USER */ - 434, /* (448) noarg_func ::= USER */ - 432, /* (449) star_func ::= COUNT */ - 432, /* (450) star_func ::= FIRST */ - 432, /* (451) star_func ::= LAST */ - 432, /* (452) star_func ::= LAST_ROW */ - 433, /* (453) star_func_para_list ::= NK_STAR */ - 433, /* (454) star_func_para_list ::= other_para_list */ - 435, /* (455) other_para_list ::= star_func_para */ - 435, /* (456) other_para_list ::= other_para_list NK_COMMA star_func_para */ - 436, /* (457) star_func_para ::= expr_or_subquery */ - 436, /* (458) star_func_para ::= table_name NK_DOT NK_STAR */ - 431, /* (459) case_when_expression ::= CASE when_then_list case_when_else_opt END */ - 431, /* (460) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ - 437, /* (461) when_then_list ::= when_then_expr */ - 437, /* (462) when_then_list ::= when_then_list when_then_expr */ - 440, /* (463) when_then_expr ::= WHEN common_expression THEN common_expression */ - 438, /* (464) case_when_else_opt ::= */ - 438, /* (465) case_when_else_opt ::= ELSE common_expression */ - 441, /* (466) predicate ::= expr_or_subquery compare_op expr_or_subquery */ - 441, /* (467) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ - 441, /* (468) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ - 441, /* (469) predicate ::= expr_or_subquery IS NULL */ - 441, /* (470) predicate ::= expr_or_subquery IS NOT NULL */ - 441, /* (471) predicate ::= expr_or_subquery in_op in_predicate_value */ - 442, /* (472) compare_op ::= NK_LT */ - 442, /* (473) compare_op ::= NK_GT */ - 442, /* (474) compare_op ::= NK_LE */ - 442, /* (475) compare_op ::= NK_GE */ - 442, /* (476) compare_op ::= NK_NE */ - 442, /* (477) compare_op ::= NK_EQ */ - 442, /* (478) compare_op ::= LIKE */ - 442, /* (479) compare_op ::= NOT LIKE */ - 442, /* (480) compare_op ::= MATCH */ - 442, /* (481) compare_op ::= NMATCH */ - 442, /* (482) compare_op ::= CONTAINS */ - 443, /* (483) in_op ::= IN */ - 443, /* (484) in_op ::= NOT IN */ - 444, /* (485) in_predicate_value ::= NK_LP literal_list NK_RP */ - 445, /* (486) boolean_value_expression ::= boolean_primary */ - 445, /* (487) boolean_value_expression ::= NOT boolean_primary */ - 445, /* (488) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - 445, /* (489) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - 446, /* (490) boolean_primary ::= predicate */ - 446, /* (491) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - 439, /* (492) common_expression ::= expr_or_subquery */ - 439, /* (493) common_expression ::= boolean_value_expression */ - 447, /* (494) from_clause_opt ::= */ - 447, /* (495) from_clause_opt ::= FROM table_reference_list */ - 448, /* (496) table_reference_list ::= table_reference */ - 448, /* (497) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - 449, /* (498) table_reference ::= table_primary */ - 449, /* (499) table_reference ::= joined_table */ - 450, /* (500) table_primary ::= table_name alias_opt */ - 450, /* (501) table_primary ::= db_name NK_DOT table_name alias_opt */ - 450, /* (502) table_primary ::= subquery alias_opt */ - 450, /* (503) table_primary ::= parenthesized_joined_table */ - 452, /* (504) alias_opt ::= */ - 452, /* (505) alias_opt ::= table_alias */ - 452, /* (506) alias_opt ::= AS table_alias */ - 454, /* (507) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - 454, /* (508) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - 451, /* (509) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - 455, /* (510) join_type ::= */ - 455, /* (511) join_type ::= INNER */ - 456, /* (512) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - 457, /* (513) set_quantifier_opt ::= */ - 457, /* (514) set_quantifier_opt ::= DISTINCT */ - 457, /* (515) set_quantifier_opt ::= ALL */ - 458, /* (516) select_list ::= select_item */ - 458, /* (517) select_list ::= select_list NK_COMMA select_item */ - 466, /* (518) select_item ::= NK_STAR */ - 466, /* (519) select_item ::= common_expression */ - 466, /* (520) select_item ::= common_expression column_alias */ - 466, /* (521) select_item ::= common_expression AS column_alias */ - 466, /* (522) select_item ::= table_name NK_DOT NK_STAR */ - 423, /* (523) where_clause_opt ::= */ - 423, /* (524) where_clause_opt ::= WHERE search_condition */ - 459, /* (525) partition_by_clause_opt ::= */ - 459, /* (526) partition_by_clause_opt ::= PARTITION BY partition_list */ - 467, /* (527) partition_list ::= partition_item */ - 467, /* (528) partition_list ::= partition_list NK_COMMA partition_item */ - 468, /* (529) partition_item ::= expr_or_subquery */ - 468, /* (530) partition_item ::= expr_or_subquery column_alias */ - 468, /* (531) partition_item ::= expr_or_subquery AS column_alias */ - 463, /* (532) twindow_clause_opt ::= */ - 463, /* (533) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - 463, /* (534) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ - 463, /* (535) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - 463, /* (536) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - 463, /* (537) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ - 402, /* (538) sliding_opt ::= */ - 402, /* (539) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - 462, /* (540) fill_opt ::= */ - 462, /* (541) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - 462, /* (542) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ - 462, /* (543) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ - 469, /* (544) fill_mode ::= NONE */ - 469, /* (545) fill_mode ::= PREV */ - 469, /* (546) fill_mode ::= NULL */ - 469, /* (547) fill_mode ::= NULL_F */ - 469, /* (548) fill_mode ::= LINEAR */ - 469, /* (549) fill_mode ::= NEXT */ - 464, /* (550) group_by_clause_opt ::= */ - 464, /* (551) group_by_clause_opt ::= GROUP BY group_by_list */ - 470, /* (552) group_by_list ::= expr_or_subquery */ - 470, /* (553) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ - 465, /* (554) having_clause_opt ::= */ - 465, /* (555) having_clause_opt ::= HAVING search_condition */ - 460, /* (556) range_opt ::= */ - 460, /* (557) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ - 461, /* (558) every_opt ::= */ - 461, /* (559) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - 471, /* (560) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ - 472, /* (561) query_simple ::= query_specification */ - 472, /* (562) query_simple ::= union_query_expression */ - 476, /* (563) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ - 476, /* (564) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ - 477, /* (565) query_simple_or_subquery ::= query_simple */ - 477, /* (566) query_simple_or_subquery ::= subquery */ - 406, /* (567) query_or_subquery ::= query_expression */ - 406, /* (568) query_or_subquery ::= subquery */ - 473, /* (569) order_by_clause_opt ::= */ - 473, /* (570) order_by_clause_opt ::= ORDER BY sort_specification_list */ - 474, /* (571) slimit_clause_opt ::= */ - 474, /* (572) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - 474, /* (573) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - 474, /* (574) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - 475, /* (575) limit_clause_opt ::= */ - 475, /* (576) limit_clause_opt ::= LIMIT NK_INTEGER */ - 475, /* (577) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - 475, /* (578) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - 453, /* (579) subquery ::= NK_LP query_expression NK_RP */ - 453, /* (580) subquery ::= NK_LP subquery NK_RP */ - 350, /* (581) search_condition ::= common_expression */ - 478, /* (582) sort_specification_list ::= sort_specification */ - 478, /* (583) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - 479, /* (584) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ - 480, /* (585) ordering_specification_opt ::= */ - 480, /* (586) ordering_specification_opt ::= ASC */ - 480, /* (587) ordering_specification_opt ::= DESC */ - 481, /* (588) null_ordering_opt ::= */ - 481, /* (589) null_ordering_opt ::= NULLS FIRST */ - 481, /* (590) null_ordering_opt ::= NULLS LAST */ + 338, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ + 338, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ + 339, /* (2) account_options ::= */ + 339, /* (3) account_options ::= account_options PPS literal */ + 339, /* (4) account_options ::= account_options TSERIES literal */ + 339, /* (5) account_options ::= account_options STORAGE literal */ + 339, /* (6) account_options ::= account_options STREAMS literal */ + 339, /* (7) account_options ::= account_options QTIME literal */ + 339, /* (8) account_options ::= account_options DBS literal */ + 339, /* (9) account_options ::= account_options USERS literal */ + 339, /* (10) account_options ::= account_options CONNS literal */ + 339, /* (11) account_options ::= account_options STATE literal */ + 340, /* (12) alter_account_options ::= alter_account_option */ + 340, /* (13) alter_account_options ::= alter_account_options alter_account_option */ + 342, /* (14) alter_account_option ::= PASS literal */ + 342, /* (15) alter_account_option ::= PPS literal */ + 342, /* (16) alter_account_option ::= TSERIES literal */ + 342, /* (17) alter_account_option ::= STORAGE literal */ + 342, /* (18) alter_account_option ::= STREAMS literal */ + 342, /* (19) alter_account_option ::= QTIME literal */ + 342, /* (20) alter_account_option ::= DBS literal */ + 342, /* (21) alter_account_option ::= USERS literal */ + 342, /* (22) alter_account_option ::= CONNS literal */ + 342, /* (23) alter_account_option ::= STATE literal */ + 338, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ + 338, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ + 338, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ + 338, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ + 338, /* (28) cmd ::= DROP USER user_name */ + 344, /* (29) sysinfo_opt ::= */ + 344, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ + 338, /* (31) cmd ::= GRANT privileges ON priv_level with_opt TO user_name */ + 338, /* (32) cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name */ + 345, /* (33) privileges ::= ALL */ + 345, /* (34) privileges ::= priv_type_list */ + 345, /* (35) privileges ::= SUBSCRIBE */ + 348, /* (36) priv_type_list ::= priv_type */ + 348, /* (37) priv_type_list ::= priv_type_list NK_COMMA priv_type */ + 349, /* (38) priv_type ::= READ */ + 349, /* (39) priv_type ::= WRITE */ + 346, /* (40) priv_level ::= NK_STAR NK_DOT NK_STAR */ + 346, /* (41) priv_level ::= db_name NK_DOT NK_STAR */ + 346, /* (42) priv_level ::= db_name NK_DOT table_name */ + 346, /* (43) priv_level ::= topic_name */ + 347, /* (44) with_opt ::= */ + 347, /* (45) with_opt ::= WITH search_condition */ + 338, /* (46) cmd ::= CREATE DNODE dnode_endpoint */ + 338, /* (47) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ + 338, /* (48) cmd ::= DROP DNODE NK_INTEGER force_opt */ + 338, /* (49) cmd ::= DROP DNODE dnode_endpoint force_opt */ + 338, /* (50) cmd ::= DROP DNODE NK_INTEGER unsafe_opt */ + 338, /* (51) cmd ::= DROP DNODE dnode_endpoint unsafe_opt */ + 338, /* (52) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + 338, /* (53) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + 338, /* (54) cmd ::= ALTER ALL DNODES NK_STRING */ + 338, /* (55) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + 338, /* (56) cmd ::= RESTORE DNODE NK_INTEGER */ + 354, /* (57) dnode_endpoint ::= NK_STRING */ + 354, /* (58) dnode_endpoint ::= NK_ID */ + 354, /* (59) dnode_endpoint ::= NK_IPTOKEN */ + 355, /* (60) force_opt ::= */ + 355, /* (61) force_opt ::= FORCE */ + 356, /* (62) unsafe_opt ::= UNSAFE */ + 338, /* (63) cmd ::= ALTER LOCAL NK_STRING */ + 338, /* (64) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + 338, /* (65) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + 338, /* (66) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + 338, /* (67) cmd ::= RESTORE QNODE ON DNODE NK_INTEGER */ + 338, /* (68) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + 338, /* (69) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + 338, /* (70) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + 338, /* (71) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + 338, /* (72) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + 338, /* (73) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + 338, /* (74) cmd ::= RESTORE MNODE ON DNODE NK_INTEGER */ + 338, /* (75) cmd ::= RESTORE VNODE ON DNODE NK_INTEGER */ + 338, /* (76) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + 338, /* (77) cmd ::= DROP DATABASE exists_opt db_name */ + 338, /* (78) cmd ::= USE db_name */ + 338, /* (79) cmd ::= ALTER DATABASE db_name alter_db_options */ + 338, /* (80) cmd ::= FLUSH DATABASE db_name */ + 338, /* (81) cmd ::= TRIM DATABASE db_name speed_opt */ + 338, /* (82) cmd ::= COMPACT DATABASE db_name start_opt end_opt */ + 357, /* (83) not_exists_opt ::= IF NOT EXISTS */ + 357, /* (84) not_exists_opt ::= */ + 359, /* (85) exists_opt ::= IF EXISTS */ + 359, /* (86) exists_opt ::= */ + 358, /* (87) db_options ::= */ + 358, /* (88) db_options ::= db_options BUFFER NK_INTEGER */ + 358, /* (89) db_options ::= db_options CACHEMODEL NK_STRING */ + 358, /* (90) db_options ::= db_options CACHESIZE NK_INTEGER */ + 358, /* (91) db_options ::= db_options COMP NK_INTEGER */ + 358, /* (92) db_options ::= db_options DURATION NK_INTEGER */ + 358, /* (93) db_options ::= db_options DURATION NK_VARIABLE */ + 358, /* (94) db_options ::= db_options MAXROWS NK_INTEGER */ + 358, /* (95) db_options ::= db_options MINROWS NK_INTEGER */ + 358, /* (96) db_options ::= db_options KEEP integer_list */ + 358, /* (97) db_options ::= db_options KEEP variable_list */ + 358, /* (98) db_options ::= db_options PAGES NK_INTEGER */ + 358, /* (99) db_options ::= db_options PAGESIZE NK_INTEGER */ + 358, /* (100) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ + 358, /* (101) db_options ::= db_options PRECISION NK_STRING */ + 358, /* (102) db_options ::= db_options REPLICA NK_INTEGER */ + 358, /* (103) db_options ::= db_options VGROUPS NK_INTEGER */ + 358, /* (104) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + 358, /* (105) db_options ::= db_options RETENTIONS retention_list */ + 358, /* (106) db_options ::= db_options SCHEMALESS NK_INTEGER */ + 358, /* (107) db_options ::= db_options WAL_LEVEL NK_INTEGER */ + 358, /* (108) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ + 358, /* (109) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ + 358, /* (110) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + 358, /* (111) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ + 358, /* (112) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + 358, /* (113) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ + 358, /* (114) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ + 358, /* (115) db_options ::= db_options STT_TRIGGER NK_INTEGER */ + 358, /* (116) db_options ::= db_options TABLE_PREFIX signed */ + 358, /* (117) db_options ::= db_options TABLE_SUFFIX signed */ + 360, /* (118) alter_db_options ::= alter_db_option */ + 360, /* (119) alter_db_options ::= alter_db_options alter_db_option */ + 368, /* (120) alter_db_option ::= BUFFER NK_INTEGER */ + 368, /* (121) alter_db_option ::= CACHEMODEL NK_STRING */ + 368, /* (122) alter_db_option ::= CACHESIZE NK_INTEGER */ + 368, /* (123) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ + 368, /* (124) alter_db_option ::= KEEP integer_list */ + 368, /* (125) alter_db_option ::= KEEP variable_list */ + 368, /* (126) alter_db_option ::= PAGES NK_INTEGER */ + 368, /* (127) alter_db_option ::= REPLICA NK_INTEGER */ + 368, /* (128) alter_db_option ::= WAL_LEVEL NK_INTEGER */ + 368, /* (129) alter_db_option ::= STT_TRIGGER NK_INTEGER */ + 368, /* (130) alter_db_option ::= MINROWS NK_INTEGER */ + 368, /* (131) alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ + 368, /* (132) alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + 368, /* (133) alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ + 368, /* (134) alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + 364, /* (135) integer_list ::= NK_INTEGER */ + 364, /* (136) integer_list ::= integer_list NK_COMMA NK_INTEGER */ + 365, /* (137) variable_list ::= NK_VARIABLE */ + 365, /* (138) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ + 366, /* (139) retention_list ::= retention */ + 366, /* (140) retention_list ::= retention_list NK_COMMA retention */ + 369, /* (141) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ + 361, /* (142) speed_opt ::= */ + 361, /* (143) speed_opt ::= MAX_SPEED NK_INTEGER */ + 362, /* (144) start_opt ::= */ + 362, /* (145) start_opt ::= START WITH NK_INTEGER */ + 362, /* (146) start_opt ::= START WITH NK_STRING */ + 362, /* (147) start_opt ::= START WITH TIMESTAMP NK_STRING */ + 363, /* (148) end_opt ::= */ + 363, /* (149) end_opt ::= END WITH NK_INTEGER */ + 363, /* (150) end_opt ::= END WITH NK_STRING */ + 363, /* (151) end_opt ::= END WITH TIMESTAMP NK_STRING */ + 338, /* (152) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + 338, /* (153) cmd ::= CREATE TABLE multi_create_clause */ + 338, /* (154) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + 338, /* (155) cmd ::= DROP TABLE multi_drop_clause */ + 338, /* (156) cmd ::= DROP STABLE exists_opt full_table_name */ + 338, /* (157) cmd ::= ALTER TABLE alter_table_clause */ + 338, /* (158) cmd ::= ALTER STABLE alter_table_clause */ + 377, /* (159) alter_table_clause ::= full_table_name alter_table_options */ + 377, /* (160) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + 377, /* (161) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + 377, /* (162) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + 377, /* (163) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + 377, /* (164) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + 377, /* (165) alter_table_clause ::= full_table_name DROP TAG column_name */ + 377, /* (166) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + 377, /* (167) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + 377, /* (168) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ + 374, /* (169) multi_create_clause ::= create_subtable_clause */ + 374, /* (170) multi_create_clause ::= multi_create_clause create_subtable_clause */ + 382, /* (171) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ + 376, /* (172) multi_drop_clause ::= drop_table_clause */ + 376, /* (173) multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ + 385, /* (174) drop_table_clause ::= exists_opt full_table_name */ + 383, /* (175) specific_cols_opt ::= */ + 383, /* (176) specific_cols_opt ::= NK_LP col_name_list NK_RP */ + 370, /* (177) full_table_name ::= table_name */ + 370, /* (178) full_table_name ::= db_name NK_DOT table_name */ + 371, /* (179) column_def_list ::= column_def */ + 371, /* (180) column_def_list ::= column_def_list NK_COMMA column_def */ + 387, /* (181) column_def ::= column_name type_name */ + 380, /* (182) type_name ::= BOOL */ + 380, /* (183) type_name ::= TINYINT */ + 380, /* (184) type_name ::= SMALLINT */ + 380, /* (185) type_name ::= INT */ + 380, /* (186) type_name ::= INTEGER */ + 380, /* (187) type_name ::= BIGINT */ + 380, /* (188) type_name ::= FLOAT */ + 380, /* (189) type_name ::= DOUBLE */ + 380, /* (190) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + 380, /* (191) type_name ::= TIMESTAMP */ + 380, /* (192) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + 380, /* (193) type_name ::= TINYINT UNSIGNED */ + 380, /* (194) type_name ::= SMALLINT UNSIGNED */ + 380, /* (195) type_name ::= INT UNSIGNED */ + 380, /* (196) type_name ::= BIGINT UNSIGNED */ + 380, /* (197) type_name ::= JSON */ + 380, /* (198) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + 380, /* (199) type_name ::= MEDIUMBLOB */ + 380, /* (200) type_name ::= BLOB */ + 380, /* (201) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + 380, /* (202) type_name ::= GEOMETRY NK_LP NK_INTEGER NK_RP */ + 380, /* (203) type_name ::= DECIMAL */ + 380, /* (204) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + 380, /* (205) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + 372, /* (206) tags_def_opt ::= */ + 372, /* (207) tags_def_opt ::= tags_def */ + 375, /* (208) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + 373, /* (209) table_options ::= */ + 373, /* (210) table_options ::= table_options COMMENT NK_STRING */ + 373, /* (211) table_options ::= table_options MAX_DELAY duration_list */ + 373, /* (212) table_options ::= table_options WATERMARK duration_list */ + 373, /* (213) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ + 373, /* (214) table_options ::= table_options TTL NK_INTEGER */ + 373, /* (215) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + 373, /* (216) table_options ::= table_options DELETE_MARK duration_list */ + 378, /* (217) alter_table_options ::= alter_table_option */ + 378, /* (218) alter_table_options ::= alter_table_options alter_table_option */ + 390, /* (219) alter_table_option ::= COMMENT NK_STRING */ + 390, /* (220) alter_table_option ::= TTL NK_INTEGER */ + 388, /* (221) duration_list ::= duration_literal */ + 388, /* (222) duration_list ::= duration_list NK_COMMA duration_literal */ + 389, /* (223) rollup_func_list ::= rollup_func_name */ + 389, /* (224) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ + 392, /* (225) rollup_func_name ::= function_name */ + 392, /* (226) rollup_func_name ::= FIRST */ + 392, /* (227) rollup_func_name ::= LAST */ + 386, /* (228) col_name_list ::= col_name */ + 386, /* (229) col_name_list ::= col_name_list NK_COMMA col_name */ + 394, /* (230) col_name ::= column_name */ + 338, /* (231) cmd ::= SHOW DNODES */ + 338, /* (232) cmd ::= SHOW USERS */ + 338, /* (233) cmd ::= SHOW USER PRIVILEGES */ + 338, /* (234) cmd ::= SHOW DATABASES */ + 338, /* (235) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + 338, /* (236) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + 338, /* (237) cmd ::= SHOW db_name_cond_opt VGROUPS */ + 338, /* (238) cmd ::= SHOW MNODES */ + 338, /* (239) cmd ::= SHOW QNODES */ + 338, /* (240) cmd ::= SHOW FUNCTIONS */ + 338, /* (241) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + 338, /* (242) cmd ::= SHOW INDEXES FROM db_name NK_DOT table_name */ + 338, /* (243) cmd ::= SHOW STREAMS */ + 338, /* (244) cmd ::= SHOW ACCOUNTS */ + 338, /* (245) cmd ::= SHOW APPS */ + 338, /* (246) cmd ::= SHOW CONNECTIONS */ + 338, /* (247) cmd ::= SHOW LICENCES */ + 338, /* (248) cmd ::= SHOW GRANTS */ + 338, /* (249) cmd ::= SHOW CREATE DATABASE db_name */ + 338, /* (250) cmd ::= SHOW CREATE TABLE full_table_name */ + 338, /* (251) cmd ::= SHOW CREATE STABLE full_table_name */ + 338, /* (252) cmd ::= SHOW QUERIES */ + 338, /* (253) cmd ::= SHOW SCORES */ + 338, /* (254) cmd ::= SHOW TOPICS */ + 338, /* (255) cmd ::= SHOW VARIABLES */ + 338, /* (256) cmd ::= SHOW CLUSTER VARIABLES */ + 338, /* (257) cmd ::= SHOW LOCAL VARIABLES */ + 338, /* (258) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ + 338, /* (259) cmd ::= SHOW BNODES */ + 338, /* (260) cmd ::= SHOW SNODES */ + 338, /* (261) cmd ::= SHOW CLUSTER */ + 338, /* (262) cmd ::= SHOW TRANSACTIONS */ + 338, /* (263) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ + 338, /* (264) cmd ::= SHOW CONSUMERS */ + 338, /* (265) cmd ::= SHOW SUBSCRIPTIONS */ + 338, /* (266) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ + 338, /* (267) cmd ::= SHOW TAGS FROM db_name NK_DOT table_name */ + 338, /* (268) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ + 338, /* (269) cmd ::= SHOW TABLE TAGS tag_list_opt FROM db_name NK_DOT table_name */ + 338, /* (270) cmd ::= SHOW VNODES NK_INTEGER */ + 338, /* (271) cmd ::= SHOW VNODES NK_STRING */ + 338, /* (272) cmd ::= SHOW db_name_cond_opt ALIVE */ + 338, /* (273) cmd ::= SHOW CLUSTER ALIVE */ + 395, /* (274) db_name_cond_opt ::= */ + 395, /* (275) db_name_cond_opt ::= db_name NK_DOT */ + 396, /* (276) like_pattern_opt ::= */ + 396, /* (277) like_pattern_opt ::= LIKE NK_STRING */ + 397, /* (278) table_name_cond ::= table_name */ + 398, /* (279) from_db_opt ::= */ + 398, /* (280) from_db_opt ::= FROM db_name */ + 399, /* (281) tag_list_opt ::= */ + 399, /* (282) tag_list_opt ::= tag_item */ + 399, /* (283) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ + 400, /* (284) tag_item ::= TBNAME */ + 400, /* (285) tag_item ::= QTAGS */ + 400, /* (286) tag_item ::= column_name */ + 400, /* (287) tag_item ::= column_name column_alias */ + 400, /* (288) tag_item ::= column_name AS column_alias */ + 338, /* (289) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ + 338, /* (290) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ + 338, /* (291) cmd ::= DROP INDEX exists_opt full_index_name */ + 402, /* (292) full_index_name ::= index_name */ + 402, /* (293) full_index_name ::= db_name NK_DOT index_name */ + 403, /* (294) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ + 403, /* (295) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ + 405, /* (296) func_list ::= func */ + 405, /* (297) func_list ::= func_list NK_COMMA func */ + 408, /* (298) func ::= sma_func_name NK_LP expression_list NK_RP */ + 409, /* (299) sma_func_name ::= function_name */ + 409, /* (300) sma_func_name ::= COUNT */ + 409, /* (301) sma_func_name ::= FIRST */ + 409, /* (302) sma_func_name ::= LAST */ + 409, /* (303) sma_func_name ::= LAST_ROW */ + 407, /* (304) sma_stream_opt ::= */ + 407, /* (305) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + 407, /* (306) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ + 407, /* (307) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ + 410, /* (308) with_meta ::= AS */ + 410, /* (309) with_meta ::= WITH META AS */ + 410, /* (310) with_meta ::= ONLY META AS */ + 338, /* (311) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ + 338, /* (312) cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta DATABASE db_name */ + 338, /* (313) cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta STABLE full_table_name where_clause_opt */ + 338, /* (314) cmd ::= DROP TOPIC exists_opt topic_name */ + 338, /* (315) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ + 338, /* (316) cmd ::= DESC full_table_name */ + 338, /* (317) cmd ::= DESCRIBE full_table_name */ + 338, /* (318) cmd ::= RESET QUERY CACHE */ + 338, /* (319) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + 338, /* (320) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ + 414, /* (321) analyze_opt ::= */ + 414, /* (322) analyze_opt ::= ANALYZE */ + 415, /* (323) explain_options ::= */ + 415, /* (324) explain_options ::= explain_options VERBOSE NK_BOOL */ + 415, /* (325) explain_options ::= explain_options RATIO NK_FLOAT */ + 338, /* (326) cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ + 338, /* (327) cmd ::= DROP FUNCTION exists_opt function_name */ + 418, /* (328) agg_func_opt ::= */ + 418, /* (329) agg_func_opt ::= AGGREGATE */ + 419, /* (330) bufsize_opt ::= */ + 419, /* (331) bufsize_opt ::= BUFSIZE NK_INTEGER */ + 420, /* (332) language_opt ::= */ + 420, /* (333) language_opt ::= LANGUAGE NK_STRING */ + 417, /* (334) or_replace_opt ::= */ + 417, /* (335) or_replace_opt ::= OR REPLACE */ + 338, /* (336) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ + 338, /* (337) cmd ::= DROP STREAM exists_opt stream_name */ + 338, /* (338) cmd ::= PAUSE STREAM exists_opt stream_name */ + 338, /* (339) cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */ + 423, /* (340) col_list_opt ::= */ + 423, /* (341) col_list_opt ::= NK_LP col_name_list NK_RP */ + 424, /* (342) tag_def_or_ref_opt ::= */ + 424, /* (343) tag_def_or_ref_opt ::= tags_def */ + 424, /* (344) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ + 422, /* (345) stream_options ::= */ + 422, /* (346) stream_options ::= stream_options TRIGGER AT_ONCE */ + 422, /* (347) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + 422, /* (348) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + 422, /* (349) stream_options ::= stream_options WATERMARK duration_literal */ + 422, /* (350) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ + 422, /* (351) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ + 422, /* (352) stream_options ::= stream_options DELETE_MARK duration_literal */ + 422, /* (353) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ + 425, /* (354) subtable_opt ::= */ + 425, /* (355) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + 426, /* (356) ignore_opt ::= */ + 426, /* (357) ignore_opt ::= IGNORE UNTREATED */ + 338, /* (358) cmd ::= KILL CONNECTION NK_INTEGER */ + 338, /* (359) cmd ::= KILL QUERY NK_STRING */ + 338, /* (360) cmd ::= KILL TRANSACTION NK_INTEGER */ + 338, /* (361) cmd ::= BALANCE VGROUP */ + 338, /* (362) cmd ::= BALANCE VGROUP LEADER */ + 338, /* (363) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + 338, /* (364) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + 338, /* (365) cmd ::= SPLIT VGROUP NK_INTEGER */ + 428, /* (366) dnode_list ::= DNODE NK_INTEGER */ + 428, /* (367) dnode_list ::= dnode_list DNODE NK_INTEGER */ + 338, /* (368) cmd ::= DELETE FROM full_table_name where_clause_opt */ + 338, /* (369) cmd ::= query_or_subquery */ + 338, /* (370) cmd ::= insert_query */ + 416, /* (371) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ + 416, /* (372) insert_query ::= INSERT INTO full_table_name query_or_subquery */ + 341, /* (373) literal ::= NK_INTEGER */ + 341, /* (374) literal ::= NK_FLOAT */ + 341, /* (375) literal ::= NK_STRING */ + 341, /* (376) literal ::= NK_BOOL */ + 341, /* (377) literal ::= TIMESTAMP NK_STRING */ + 341, /* (378) literal ::= duration_literal */ + 341, /* (379) literal ::= NULL */ + 341, /* (380) literal ::= NK_QUESTION */ + 391, /* (381) duration_literal ::= NK_VARIABLE */ + 367, /* (382) signed ::= NK_INTEGER */ + 367, /* (383) signed ::= NK_PLUS NK_INTEGER */ + 367, /* (384) signed ::= NK_MINUS NK_INTEGER */ + 367, /* (385) signed ::= NK_FLOAT */ + 367, /* (386) signed ::= NK_PLUS NK_FLOAT */ + 367, /* (387) signed ::= NK_MINUS NK_FLOAT */ + 381, /* (388) signed_literal ::= signed */ + 381, /* (389) signed_literal ::= NK_STRING */ + 381, /* (390) signed_literal ::= NK_BOOL */ + 381, /* (391) signed_literal ::= TIMESTAMP NK_STRING */ + 381, /* (392) signed_literal ::= duration_literal */ + 381, /* (393) signed_literal ::= NULL */ + 381, /* (394) signed_literal ::= literal_func */ + 381, /* (395) signed_literal ::= NK_QUESTION */ + 430, /* (396) literal_list ::= signed_literal */ + 430, /* (397) literal_list ::= literal_list NK_COMMA signed_literal */ + 350, /* (398) db_name ::= NK_ID */ + 351, /* (399) table_name ::= NK_ID */ + 379, /* (400) column_name ::= NK_ID */ + 393, /* (401) function_name ::= NK_ID */ + 431, /* (402) table_alias ::= NK_ID */ + 401, /* (403) column_alias ::= NK_ID */ + 343, /* (404) user_name ::= NK_ID */ + 352, /* (405) topic_name ::= NK_ID */ + 421, /* (406) stream_name ::= NK_ID */ + 413, /* (407) cgroup_name ::= NK_ID */ + 404, /* (408) index_name ::= NK_ID */ + 432, /* (409) expr_or_subquery ::= expression */ + 427, /* (410) expression ::= literal */ + 427, /* (411) expression ::= pseudo_column */ + 427, /* (412) expression ::= column_reference */ + 427, /* (413) expression ::= function_expression */ + 427, /* (414) expression ::= case_when_expression */ + 427, /* (415) expression ::= NK_LP expression NK_RP */ + 427, /* (416) expression ::= NK_PLUS expr_or_subquery */ + 427, /* (417) expression ::= NK_MINUS expr_or_subquery */ + 427, /* (418) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + 427, /* (419) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + 427, /* (420) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + 427, /* (421) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + 427, /* (422) expression ::= expr_or_subquery NK_REM expr_or_subquery */ + 427, /* (423) expression ::= column_reference NK_ARROW NK_STRING */ + 427, /* (424) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + 427, /* (425) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + 384, /* (426) expression_list ::= expr_or_subquery */ + 384, /* (427) expression_list ::= expression_list NK_COMMA expr_or_subquery */ + 434, /* (428) column_reference ::= column_name */ + 434, /* (429) column_reference ::= table_name NK_DOT column_name */ + 433, /* (430) pseudo_column ::= ROWTS */ + 433, /* (431) pseudo_column ::= TBNAME */ + 433, /* (432) pseudo_column ::= table_name NK_DOT TBNAME */ + 433, /* (433) pseudo_column ::= QSTART */ + 433, /* (434) pseudo_column ::= QEND */ + 433, /* (435) pseudo_column ::= QDURATION */ + 433, /* (436) pseudo_column ::= WSTART */ + 433, /* (437) pseudo_column ::= WEND */ + 433, /* (438) pseudo_column ::= WDURATION */ + 433, /* (439) pseudo_column ::= IROWTS */ + 433, /* (440) pseudo_column ::= ISFILLED */ + 433, /* (441) pseudo_column ::= QTAGS */ + 435, /* (442) function_expression ::= function_name NK_LP expression_list NK_RP */ + 435, /* (443) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + 435, /* (444) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ + 435, /* (445) function_expression ::= literal_func */ + 429, /* (446) literal_func ::= noarg_func NK_LP NK_RP */ + 429, /* (447) literal_func ::= NOW */ + 439, /* (448) noarg_func ::= NOW */ + 439, /* (449) noarg_func ::= TODAY */ + 439, /* (450) noarg_func ::= TIMEZONE */ + 439, /* (451) noarg_func ::= DATABASE */ + 439, /* (452) noarg_func ::= CLIENT_VERSION */ + 439, /* (453) noarg_func ::= SERVER_VERSION */ + 439, /* (454) noarg_func ::= SERVER_STATUS */ + 439, /* (455) noarg_func ::= CURRENT_USER */ + 439, /* (456) noarg_func ::= USER */ + 437, /* (457) star_func ::= COUNT */ + 437, /* (458) star_func ::= FIRST */ + 437, /* (459) star_func ::= LAST */ + 437, /* (460) star_func ::= LAST_ROW */ + 438, /* (461) star_func_para_list ::= NK_STAR */ + 438, /* (462) star_func_para_list ::= other_para_list */ + 440, /* (463) other_para_list ::= star_func_para */ + 440, /* (464) other_para_list ::= other_para_list NK_COMMA star_func_para */ + 441, /* (465) star_func_para ::= expr_or_subquery */ + 441, /* (466) star_func_para ::= table_name NK_DOT NK_STAR */ + 436, /* (467) case_when_expression ::= CASE when_then_list case_when_else_opt END */ + 436, /* (468) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ + 442, /* (469) when_then_list ::= when_then_expr */ + 442, /* (470) when_then_list ::= when_then_list when_then_expr */ + 445, /* (471) when_then_expr ::= WHEN common_expression THEN common_expression */ + 443, /* (472) case_when_else_opt ::= */ + 443, /* (473) case_when_else_opt ::= ELSE common_expression */ + 446, /* (474) predicate ::= expr_or_subquery compare_op expr_or_subquery */ + 446, /* (475) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + 446, /* (476) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + 446, /* (477) predicate ::= expr_or_subquery IS NULL */ + 446, /* (478) predicate ::= expr_or_subquery IS NOT NULL */ + 446, /* (479) predicate ::= expr_or_subquery in_op in_predicate_value */ + 447, /* (480) compare_op ::= NK_LT */ + 447, /* (481) compare_op ::= NK_GT */ + 447, /* (482) compare_op ::= NK_LE */ + 447, /* (483) compare_op ::= NK_GE */ + 447, /* (484) compare_op ::= NK_NE */ + 447, /* (485) compare_op ::= NK_EQ */ + 447, /* (486) compare_op ::= LIKE */ + 447, /* (487) compare_op ::= NOT LIKE */ + 447, /* (488) compare_op ::= MATCH */ + 447, /* (489) compare_op ::= NMATCH */ + 447, /* (490) compare_op ::= CONTAINS */ + 448, /* (491) in_op ::= IN */ + 448, /* (492) in_op ::= NOT IN */ + 449, /* (493) in_predicate_value ::= NK_LP literal_list NK_RP */ + 450, /* (494) boolean_value_expression ::= boolean_primary */ + 450, /* (495) boolean_value_expression ::= NOT boolean_primary */ + 450, /* (496) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + 450, /* (497) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + 451, /* (498) boolean_primary ::= predicate */ + 451, /* (499) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + 444, /* (500) common_expression ::= expr_or_subquery */ + 444, /* (501) common_expression ::= boolean_value_expression */ + 452, /* (502) from_clause_opt ::= */ + 452, /* (503) from_clause_opt ::= FROM table_reference_list */ + 453, /* (504) table_reference_list ::= table_reference */ + 453, /* (505) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + 454, /* (506) table_reference ::= table_primary */ + 454, /* (507) table_reference ::= joined_table */ + 455, /* (508) table_primary ::= table_name alias_opt */ + 455, /* (509) table_primary ::= db_name NK_DOT table_name alias_opt */ + 455, /* (510) table_primary ::= subquery alias_opt */ + 455, /* (511) table_primary ::= parenthesized_joined_table */ + 457, /* (512) alias_opt ::= */ + 457, /* (513) alias_opt ::= table_alias */ + 457, /* (514) alias_opt ::= AS table_alias */ + 459, /* (515) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + 459, /* (516) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + 456, /* (517) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + 460, /* (518) join_type ::= */ + 460, /* (519) join_type ::= INNER */ + 461, /* (520) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + 462, /* (521) set_quantifier_opt ::= */ + 462, /* (522) set_quantifier_opt ::= DISTINCT */ + 462, /* (523) set_quantifier_opt ::= ALL */ + 463, /* (524) select_list ::= select_item */ + 463, /* (525) select_list ::= select_list NK_COMMA select_item */ + 471, /* (526) select_item ::= NK_STAR */ + 471, /* (527) select_item ::= common_expression */ + 471, /* (528) select_item ::= common_expression column_alias */ + 471, /* (529) select_item ::= common_expression AS column_alias */ + 471, /* (530) select_item ::= table_name NK_DOT NK_STAR */ + 412, /* (531) where_clause_opt ::= */ + 412, /* (532) where_clause_opt ::= WHERE search_condition */ + 464, /* (533) partition_by_clause_opt ::= */ + 464, /* (534) partition_by_clause_opt ::= PARTITION BY partition_list */ + 472, /* (535) partition_list ::= partition_item */ + 472, /* (536) partition_list ::= partition_list NK_COMMA partition_item */ + 473, /* (537) partition_item ::= expr_or_subquery */ + 473, /* (538) partition_item ::= expr_or_subquery column_alias */ + 473, /* (539) partition_item ::= expr_or_subquery AS column_alias */ + 468, /* (540) twindow_clause_opt ::= */ + 468, /* (541) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + 468, /* (542) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ + 468, /* (543) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + 468, /* (544) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + 468, /* (545) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ + 406, /* (546) sliding_opt ::= */ + 406, /* (547) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + 467, /* (548) fill_opt ::= */ + 467, /* (549) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + 467, /* (550) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ + 467, /* (551) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ + 474, /* (552) fill_mode ::= NONE */ + 474, /* (553) fill_mode ::= PREV */ + 474, /* (554) fill_mode ::= NULL */ + 474, /* (555) fill_mode ::= NULL_F */ + 474, /* (556) fill_mode ::= LINEAR */ + 474, /* (557) fill_mode ::= NEXT */ + 469, /* (558) group_by_clause_opt ::= */ + 469, /* (559) group_by_clause_opt ::= GROUP BY group_by_list */ + 475, /* (560) group_by_list ::= expr_or_subquery */ + 475, /* (561) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ + 470, /* (562) having_clause_opt ::= */ + 470, /* (563) having_clause_opt ::= HAVING search_condition */ + 465, /* (564) range_opt ::= */ + 465, /* (565) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ + 465, /* (566) range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ + 466, /* (567) every_opt ::= */ + 466, /* (568) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + 476, /* (569) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + 477, /* (570) query_simple ::= query_specification */ + 477, /* (571) query_simple ::= union_query_expression */ + 481, /* (572) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + 481, /* (573) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + 482, /* (574) query_simple_or_subquery ::= query_simple */ + 482, /* (575) query_simple_or_subquery ::= subquery */ + 411, /* (576) query_or_subquery ::= query_expression */ + 411, /* (577) query_or_subquery ::= subquery */ + 478, /* (578) order_by_clause_opt ::= */ + 478, /* (579) order_by_clause_opt ::= ORDER BY sort_specification_list */ + 479, /* (580) slimit_clause_opt ::= */ + 479, /* (581) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + 479, /* (582) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + 479, /* (583) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + 480, /* (584) limit_clause_opt ::= */ + 480, /* (585) limit_clause_opt ::= LIMIT NK_INTEGER */ + 480, /* (586) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + 480, /* (587) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + 458, /* (588) subquery ::= NK_LP query_expression NK_RP */ + 458, /* (589) subquery ::= NK_LP subquery NK_RP */ + 353, /* (590) search_condition ::= common_expression */ + 483, /* (591) sort_specification_list ::= sort_specification */ + 483, /* (592) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + 484, /* (593) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + 485, /* (594) ordering_specification_opt ::= */ + 485, /* (595) ordering_specification_opt ::= ASC */ + 485, /* (596) ordering_specification_opt ::= DESC */ + 486, /* (597) null_ordering_opt ::= */ + 486, /* (598) null_ordering_opt ::= NULLS FIRST */ + 486, /* (599) null_ordering_opt ::= NULLS LAST */ }; /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number @@ -3872,547 +3847,556 @@ static const signed char yyRuleInfoNRhs[] = { -5, /* (47) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ -4, /* (48) cmd ::= DROP DNODE NK_INTEGER force_opt */ -4, /* (49) cmd ::= DROP DNODE dnode_endpoint force_opt */ - -4, /* (50) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ - -5, /* (51) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ - -4, /* (52) cmd ::= ALTER ALL DNODES NK_STRING */ - -5, /* (53) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ - -3, /* (54) cmd ::= RESTORE DNODE NK_INTEGER */ - -1, /* (55) dnode_endpoint ::= NK_STRING */ - -1, /* (56) dnode_endpoint ::= NK_ID */ - -1, /* (57) dnode_endpoint ::= NK_IPTOKEN */ - 0, /* (58) force_opt ::= */ - -1, /* (59) force_opt ::= FORCE */ - -3, /* (60) cmd ::= ALTER LOCAL NK_STRING */ - -4, /* (61) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ - -5, /* (62) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ - -5, /* (63) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ - -5, /* (64) cmd ::= RESTORE QNODE ON DNODE NK_INTEGER */ - -5, /* (65) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ - -5, /* (66) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ - -5, /* (67) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ - -5, /* (68) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ - -5, /* (69) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ - -5, /* (70) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ - -5, /* (71) cmd ::= RESTORE MNODE ON DNODE NK_INTEGER */ - -5, /* (72) cmd ::= RESTORE VNODE ON DNODE NK_INTEGER */ - -5, /* (73) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - -4, /* (74) cmd ::= DROP DATABASE exists_opt db_name */ - -2, /* (75) cmd ::= USE db_name */ - -4, /* (76) cmd ::= ALTER DATABASE db_name alter_db_options */ - -3, /* (77) cmd ::= FLUSH DATABASE db_name */ - -4, /* (78) cmd ::= TRIM DATABASE db_name speed_opt */ - -5, /* (79) cmd ::= COMPACT DATABASE db_name start_opt end_opt */ - -3, /* (80) not_exists_opt ::= IF NOT EXISTS */ - 0, /* (81) not_exists_opt ::= */ - -2, /* (82) exists_opt ::= IF EXISTS */ - 0, /* (83) exists_opt ::= */ - 0, /* (84) db_options ::= */ - -3, /* (85) db_options ::= db_options BUFFER NK_INTEGER */ - -3, /* (86) db_options ::= db_options CACHEMODEL NK_STRING */ - -3, /* (87) db_options ::= db_options CACHESIZE NK_INTEGER */ - -3, /* (88) db_options ::= db_options COMP NK_INTEGER */ - -3, /* (89) db_options ::= db_options DURATION NK_INTEGER */ - -3, /* (90) db_options ::= db_options DURATION NK_VARIABLE */ - -3, /* (91) db_options ::= db_options MAXROWS NK_INTEGER */ - -3, /* (92) db_options ::= db_options MINROWS NK_INTEGER */ - -3, /* (93) db_options ::= db_options KEEP integer_list */ - -3, /* (94) db_options ::= db_options KEEP variable_list */ - -3, /* (95) db_options ::= db_options PAGES NK_INTEGER */ - -3, /* (96) db_options ::= db_options PAGESIZE NK_INTEGER */ - -3, /* (97) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ - -3, /* (98) db_options ::= db_options PRECISION NK_STRING */ - -3, /* (99) db_options ::= db_options REPLICA NK_INTEGER */ - -3, /* (100) db_options ::= db_options VGROUPS NK_INTEGER */ - -3, /* (101) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - -3, /* (102) db_options ::= db_options RETENTIONS retention_list */ - -3, /* (103) db_options ::= db_options SCHEMALESS NK_INTEGER */ - -3, /* (104) db_options ::= db_options WAL_LEVEL NK_INTEGER */ - -3, /* (105) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ - -3, /* (106) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ - -4, /* (107) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ - -3, /* (108) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ - -4, /* (109) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ - -3, /* (110) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ - -3, /* (111) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ - -3, /* (112) db_options ::= db_options STT_TRIGGER NK_INTEGER */ - -3, /* (113) db_options ::= db_options TABLE_PREFIX signed */ - -3, /* (114) db_options ::= db_options TABLE_SUFFIX signed */ - -1, /* (115) alter_db_options ::= alter_db_option */ - -2, /* (116) alter_db_options ::= alter_db_options alter_db_option */ - -2, /* (117) alter_db_option ::= BUFFER NK_INTEGER */ - -2, /* (118) alter_db_option ::= CACHEMODEL NK_STRING */ - -2, /* (119) alter_db_option ::= CACHESIZE NK_INTEGER */ - -2, /* (120) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ - -2, /* (121) alter_db_option ::= KEEP integer_list */ - -2, /* (122) alter_db_option ::= KEEP variable_list */ - -2, /* (123) alter_db_option ::= PAGES NK_INTEGER */ - -2, /* (124) alter_db_option ::= REPLICA NK_INTEGER */ - -2, /* (125) alter_db_option ::= WAL_LEVEL NK_INTEGER */ - -2, /* (126) alter_db_option ::= STT_TRIGGER NK_INTEGER */ - -2, /* (127) alter_db_option ::= MINROWS NK_INTEGER */ - -2, /* (128) alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ - -3, /* (129) alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ - -2, /* (130) alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ - -3, /* (131) alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ - -1, /* (132) integer_list ::= NK_INTEGER */ - -3, /* (133) integer_list ::= integer_list NK_COMMA NK_INTEGER */ - -1, /* (134) variable_list ::= NK_VARIABLE */ - -3, /* (135) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ - -1, /* (136) retention_list ::= retention */ - -3, /* (137) retention_list ::= retention_list NK_COMMA retention */ - -3, /* (138) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ - 0, /* (139) speed_opt ::= */ - -2, /* (140) speed_opt ::= MAX_SPEED NK_INTEGER */ - 0, /* (141) start_opt ::= */ - -3, /* (142) start_opt ::= START WITH NK_INTEGER */ - -3, /* (143) start_opt ::= START WITH NK_STRING */ - -4, /* (144) start_opt ::= START WITH TIMESTAMP NK_STRING */ - 0, /* (145) end_opt ::= */ - -3, /* (146) end_opt ::= END WITH NK_INTEGER */ - -3, /* (147) end_opt ::= END WITH NK_STRING */ - -4, /* (148) end_opt ::= END WITH TIMESTAMP NK_STRING */ - -9, /* (149) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - -3, /* (150) cmd ::= CREATE TABLE multi_create_clause */ - -9, /* (151) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - -3, /* (152) cmd ::= DROP TABLE multi_drop_clause */ - -4, /* (153) cmd ::= DROP STABLE exists_opt full_table_name */ - -3, /* (154) cmd ::= ALTER TABLE alter_table_clause */ - -3, /* (155) cmd ::= ALTER STABLE alter_table_clause */ - -2, /* (156) alter_table_clause ::= full_table_name alter_table_options */ - -5, /* (157) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ - -4, /* (158) alter_table_clause ::= full_table_name DROP COLUMN column_name */ - -5, /* (159) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ - -5, /* (160) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ - -5, /* (161) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ - -4, /* (162) alter_table_clause ::= full_table_name DROP TAG column_name */ - -5, /* (163) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ - -5, /* (164) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ - -6, /* (165) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ - -1, /* (166) multi_create_clause ::= create_subtable_clause */ - -2, /* (167) multi_create_clause ::= multi_create_clause create_subtable_clause */ - -10, /* (168) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ - -1, /* (169) multi_drop_clause ::= drop_table_clause */ - -3, /* (170) multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ - -2, /* (171) drop_table_clause ::= exists_opt full_table_name */ - 0, /* (172) specific_cols_opt ::= */ - -3, /* (173) specific_cols_opt ::= NK_LP col_name_list NK_RP */ - -1, /* (174) full_table_name ::= table_name */ - -3, /* (175) full_table_name ::= db_name NK_DOT table_name */ - -1, /* (176) column_def_list ::= column_def */ - -3, /* (177) column_def_list ::= column_def_list NK_COMMA column_def */ - -2, /* (178) column_def ::= column_name type_name */ - -1, /* (179) type_name ::= BOOL */ - -1, /* (180) type_name ::= TINYINT */ - -1, /* (181) type_name ::= SMALLINT */ - -1, /* (182) type_name ::= INT */ - -1, /* (183) type_name ::= INTEGER */ - -1, /* (184) type_name ::= BIGINT */ - -1, /* (185) type_name ::= FLOAT */ - -1, /* (186) type_name ::= DOUBLE */ - -4, /* (187) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - -1, /* (188) type_name ::= TIMESTAMP */ - -4, /* (189) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - -2, /* (190) type_name ::= TINYINT UNSIGNED */ - -2, /* (191) type_name ::= SMALLINT UNSIGNED */ - -2, /* (192) type_name ::= INT UNSIGNED */ - -2, /* (193) type_name ::= BIGINT UNSIGNED */ - -1, /* (194) type_name ::= JSON */ - -4, /* (195) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - -1, /* (196) type_name ::= MEDIUMBLOB */ - -1, /* (197) type_name ::= BLOB */ - -4, /* (198) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - -1, /* (199) type_name ::= DECIMAL */ - -4, /* (200) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - -6, /* (201) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - 0, /* (202) tags_def_opt ::= */ - -1, /* (203) tags_def_opt ::= tags_def */ - -4, /* (204) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - 0, /* (205) table_options ::= */ - -3, /* (206) table_options ::= table_options COMMENT NK_STRING */ - -3, /* (207) table_options ::= table_options MAX_DELAY duration_list */ - -3, /* (208) table_options ::= table_options WATERMARK duration_list */ - -5, /* (209) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ - -3, /* (210) table_options ::= table_options TTL NK_INTEGER */ - -5, /* (211) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - -3, /* (212) table_options ::= table_options DELETE_MARK duration_list */ - -1, /* (213) alter_table_options ::= alter_table_option */ - -2, /* (214) alter_table_options ::= alter_table_options alter_table_option */ - -2, /* (215) alter_table_option ::= COMMENT NK_STRING */ - -2, /* (216) alter_table_option ::= TTL NK_INTEGER */ - -1, /* (217) duration_list ::= duration_literal */ - -3, /* (218) duration_list ::= duration_list NK_COMMA duration_literal */ - -1, /* (219) rollup_func_list ::= rollup_func_name */ - -3, /* (220) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ - -1, /* (221) rollup_func_name ::= function_name */ - -1, /* (222) rollup_func_name ::= FIRST */ - -1, /* (223) rollup_func_name ::= LAST */ - -1, /* (224) col_name_list ::= col_name */ - -3, /* (225) col_name_list ::= col_name_list NK_COMMA col_name */ - -1, /* (226) col_name ::= column_name */ - -2, /* (227) cmd ::= SHOW DNODES */ - -2, /* (228) cmd ::= SHOW USERS */ - -3, /* (229) cmd ::= SHOW USER PRIVILEGES */ - -2, /* (230) cmd ::= SHOW DATABASES */ - -4, /* (231) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ - -4, /* (232) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ - -3, /* (233) cmd ::= SHOW db_name_cond_opt VGROUPS */ - -2, /* (234) cmd ::= SHOW MNODES */ - -2, /* (235) cmd ::= SHOW QNODES */ - -2, /* (236) cmd ::= SHOW FUNCTIONS */ - -5, /* (237) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - -2, /* (238) cmd ::= SHOW STREAMS */ - -2, /* (239) cmd ::= SHOW ACCOUNTS */ - -2, /* (240) cmd ::= SHOW APPS */ - -2, /* (241) cmd ::= SHOW CONNECTIONS */ - -2, /* (242) cmd ::= SHOW LICENCES */ - -2, /* (243) cmd ::= SHOW GRANTS */ - -4, /* (244) cmd ::= SHOW CREATE DATABASE db_name */ - -4, /* (245) cmd ::= SHOW CREATE TABLE full_table_name */ - -4, /* (246) cmd ::= SHOW CREATE STABLE full_table_name */ - -2, /* (247) cmd ::= SHOW QUERIES */ - -2, /* (248) cmd ::= SHOW SCORES */ - -2, /* (249) cmd ::= SHOW TOPICS */ - -2, /* (250) cmd ::= SHOW VARIABLES */ - -3, /* (251) cmd ::= SHOW CLUSTER VARIABLES */ - -3, /* (252) cmd ::= SHOW LOCAL VARIABLES */ - -5, /* (253) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ - -2, /* (254) cmd ::= SHOW BNODES */ - -2, /* (255) cmd ::= SHOW SNODES */ - -2, /* (256) cmd ::= SHOW CLUSTER */ - -2, /* (257) cmd ::= SHOW TRANSACTIONS */ - -4, /* (258) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ - -2, /* (259) cmd ::= SHOW CONSUMERS */ - -2, /* (260) cmd ::= SHOW SUBSCRIPTIONS */ - -5, /* (261) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ - -7, /* (262) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ - -3, /* (263) cmd ::= SHOW VNODES NK_INTEGER */ - -3, /* (264) cmd ::= SHOW VNODES NK_STRING */ - -3, /* (265) cmd ::= SHOW db_name_cond_opt ALIVE */ - -3, /* (266) cmd ::= SHOW CLUSTER ALIVE */ - 0, /* (267) db_name_cond_opt ::= */ - -2, /* (268) db_name_cond_opt ::= db_name NK_DOT */ - 0, /* (269) like_pattern_opt ::= */ - -2, /* (270) like_pattern_opt ::= LIKE NK_STRING */ - -1, /* (271) table_name_cond ::= table_name */ - 0, /* (272) from_db_opt ::= */ - -2, /* (273) from_db_opt ::= FROM db_name */ - 0, /* (274) tag_list_opt ::= */ - -1, /* (275) tag_list_opt ::= tag_item */ - -3, /* (276) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ - -1, /* (277) tag_item ::= TBNAME */ - -1, /* (278) tag_item ::= QTAGS */ - -1, /* (279) tag_item ::= column_name */ - -2, /* (280) tag_item ::= column_name column_alias */ - -3, /* (281) tag_item ::= column_name AS column_alias */ - -8, /* (282) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ - -9, /* (283) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ - -4, /* (284) cmd ::= DROP INDEX exists_opt full_index_name */ - -1, /* (285) full_index_name ::= index_name */ - -3, /* (286) full_index_name ::= db_name NK_DOT index_name */ - -10, /* (287) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ - -12, /* (288) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ - -1, /* (289) func_list ::= func */ - -3, /* (290) func_list ::= func_list NK_COMMA func */ - -4, /* (291) func ::= sma_func_name NK_LP expression_list NK_RP */ - -1, /* (292) sma_func_name ::= function_name */ - -1, /* (293) sma_func_name ::= COUNT */ - -1, /* (294) sma_func_name ::= FIRST */ - -1, /* (295) sma_func_name ::= LAST */ - -1, /* (296) sma_func_name ::= LAST_ROW */ - 0, /* (297) sma_stream_opt ::= */ - -3, /* (298) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ - -3, /* (299) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ - -3, /* (300) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ - -6, /* (301) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ - -7, /* (302) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ - -9, /* (303) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - -7, /* (304) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ - -9, /* (305) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ - -4, /* (306) cmd ::= DROP TOPIC exists_opt topic_name */ - -7, /* (307) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ - -2, /* (308) cmd ::= DESC full_table_name */ - -2, /* (309) cmd ::= DESCRIBE full_table_name */ - -3, /* (310) cmd ::= RESET QUERY CACHE */ - -4, /* (311) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - -4, /* (312) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ - 0, /* (313) analyze_opt ::= */ - -1, /* (314) analyze_opt ::= ANALYZE */ - 0, /* (315) explain_options ::= */ - -3, /* (316) explain_options ::= explain_options VERBOSE NK_BOOL */ - -3, /* (317) explain_options ::= explain_options RATIO NK_FLOAT */ - -12, /* (318) cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ - -4, /* (319) cmd ::= DROP FUNCTION exists_opt function_name */ - 0, /* (320) agg_func_opt ::= */ - -1, /* (321) agg_func_opt ::= AGGREGATE */ - 0, /* (322) bufsize_opt ::= */ - -2, /* (323) bufsize_opt ::= BUFSIZE NK_INTEGER */ - 0, /* (324) language_opt ::= */ - -2, /* (325) language_opt ::= LANGUAGE NK_STRING */ - 0, /* (326) or_replace_opt ::= */ - -2, /* (327) or_replace_opt ::= OR REPLACE */ - -12, /* (328) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ - -4, /* (329) cmd ::= DROP STREAM exists_opt stream_name */ - -4, /* (330) cmd ::= PAUSE STREAM exists_opt stream_name */ - -5, /* (331) cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */ - 0, /* (332) col_list_opt ::= */ - -3, /* (333) col_list_opt ::= NK_LP col_name_list NK_RP */ - 0, /* (334) tag_def_or_ref_opt ::= */ - -1, /* (335) tag_def_or_ref_opt ::= tags_def */ - -4, /* (336) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ - 0, /* (337) stream_options ::= */ - -3, /* (338) stream_options ::= stream_options TRIGGER AT_ONCE */ - -3, /* (339) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - -4, /* (340) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - -3, /* (341) stream_options ::= stream_options WATERMARK duration_literal */ - -4, /* (342) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ - -3, /* (343) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ - -3, /* (344) stream_options ::= stream_options DELETE_MARK duration_literal */ - -4, /* (345) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ - 0, /* (346) subtable_opt ::= */ - -4, /* (347) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - 0, /* (348) ignore_opt ::= */ - -2, /* (349) ignore_opt ::= IGNORE UNTREATED */ - -3, /* (350) cmd ::= KILL CONNECTION NK_INTEGER */ - -3, /* (351) cmd ::= KILL QUERY NK_STRING */ - -3, /* (352) cmd ::= KILL TRANSACTION NK_INTEGER */ - -2, /* (353) cmd ::= BALANCE VGROUP */ - -3, /* (354) cmd ::= BALANCE VGROUP LEADER */ - -4, /* (355) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ - -4, /* (356) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ - -3, /* (357) cmd ::= SPLIT VGROUP NK_INTEGER */ - -2, /* (358) dnode_list ::= DNODE NK_INTEGER */ - -3, /* (359) dnode_list ::= dnode_list DNODE NK_INTEGER */ - -4, /* (360) cmd ::= DELETE FROM full_table_name where_clause_opt */ - -1, /* (361) cmd ::= query_or_subquery */ - -1, /* (362) cmd ::= insert_query */ - -7, /* (363) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ - -4, /* (364) insert_query ::= INSERT INTO full_table_name query_or_subquery */ - -1, /* (365) literal ::= NK_INTEGER */ - -1, /* (366) literal ::= NK_FLOAT */ - -1, /* (367) literal ::= NK_STRING */ - -1, /* (368) literal ::= NK_BOOL */ - -2, /* (369) literal ::= TIMESTAMP NK_STRING */ - -1, /* (370) literal ::= duration_literal */ - -1, /* (371) literal ::= NULL */ - -1, /* (372) literal ::= NK_QUESTION */ - -1, /* (373) duration_literal ::= NK_VARIABLE */ - -1, /* (374) signed ::= NK_INTEGER */ - -2, /* (375) signed ::= NK_PLUS NK_INTEGER */ - -2, /* (376) signed ::= NK_MINUS NK_INTEGER */ - -1, /* (377) signed ::= NK_FLOAT */ - -2, /* (378) signed ::= NK_PLUS NK_FLOAT */ - -2, /* (379) signed ::= NK_MINUS NK_FLOAT */ - -1, /* (380) signed_literal ::= signed */ - -1, /* (381) signed_literal ::= NK_STRING */ - -1, /* (382) signed_literal ::= NK_BOOL */ - -2, /* (383) signed_literal ::= TIMESTAMP NK_STRING */ - -1, /* (384) signed_literal ::= duration_literal */ - -1, /* (385) signed_literal ::= NULL */ - -1, /* (386) signed_literal ::= literal_func */ - -1, /* (387) signed_literal ::= NK_QUESTION */ - -1, /* (388) literal_list ::= signed_literal */ - -3, /* (389) literal_list ::= literal_list NK_COMMA signed_literal */ - -1, /* (390) db_name ::= NK_ID */ - -1, /* (391) table_name ::= NK_ID */ - -1, /* (392) column_name ::= NK_ID */ - -1, /* (393) function_name ::= NK_ID */ - -1, /* (394) table_alias ::= NK_ID */ - -1, /* (395) column_alias ::= NK_ID */ - -1, /* (396) user_name ::= NK_ID */ - -1, /* (397) topic_name ::= NK_ID */ - -1, /* (398) stream_name ::= NK_ID */ - -1, /* (399) cgroup_name ::= NK_ID */ - -1, /* (400) index_name ::= NK_ID */ - -1, /* (401) expr_or_subquery ::= expression */ - -1, /* (402) expression ::= literal */ - -1, /* (403) expression ::= pseudo_column */ - -1, /* (404) expression ::= column_reference */ - -1, /* (405) expression ::= function_expression */ - -1, /* (406) expression ::= case_when_expression */ - -3, /* (407) expression ::= NK_LP expression NK_RP */ - -2, /* (408) expression ::= NK_PLUS expr_or_subquery */ - -2, /* (409) expression ::= NK_MINUS expr_or_subquery */ - -3, /* (410) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ - -3, /* (411) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ - -3, /* (412) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ - -3, /* (413) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ - -3, /* (414) expression ::= expr_or_subquery NK_REM expr_or_subquery */ - -3, /* (415) expression ::= column_reference NK_ARROW NK_STRING */ - -3, /* (416) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ - -3, /* (417) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ - -1, /* (418) expression_list ::= expr_or_subquery */ - -3, /* (419) expression_list ::= expression_list NK_COMMA expr_or_subquery */ - -1, /* (420) column_reference ::= column_name */ - -3, /* (421) column_reference ::= table_name NK_DOT column_name */ - -1, /* (422) pseudo_column ::= ROWTS */ - -1, /* (423) pseudo_column ::= TBNAME */ - -3, /* (424) pseudo_column ::= table_name NK_DOT TBNAME */ - -1, /* (425) pseudo_column ::= QSTART */ - -1, /* (426) pseudo_column ::= QEND */ - -1, /* (427) pseudo_column ::= QDURATION */ - -1, /* (428) pseudo_column ::= WSTART */ - -1, /* (429) pseudo_column ::= WEND */ - -1, /* (430) pseudo_column ::= WDURATION */ - -1, /* (431) pseudo_column ::= IROWTS */ - -1, /* (432) pseudo_column ::= ISFILLED */ - -1, /* (433) pseudo_column ::= QTAGS */ - -4, /* (434) function_expression ::= function_name NK_LP expression_list NK_RP */ - -4, /* (435) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - -6, /* (436) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ - -1, /* (437) function_expression ::= literal_func */ - -3, /* (438) literal_func ::= noarg_func NK_LP NK_RP */ - -1, /* (439) literal_func ::= NOW */ - -1, /* (440) noarg_func ::= NOW */ - -1, /* (441) noarg_func ::= TODAY */ - -1, /* (442) noarg_func ::= TIMEZONE */ - -1, /* (443) noarg_func ::= DATABASE */ - -1, /* (444) noarg_func ::= CLIENT_VERSION */ - -1, /* (445) noarg_func ::= SERVER_VERSION */ - -1, /* (446) noarg_func ::= SERVER_STATUS */ - -1, /* (447) noarg_func ::= CURRENT_USER */ - -1, /* (448) noarg_func ::= USER */ - -1, /* (449) star_func ::= COUNT */ - -1, /* (450) star_func ::= FIRST */ - -1, /* (451) star_func ::= LAST */ - -1, /* (452) star_func ::= LAST_ROW */ - -1, /* (453) star_func_para_list ::= NK_STAR */ - -1, /* (454) star_func_para_list ::= other_para_list */ - -1, /* (455) other_para_list ::= star_func_para */ - -3, /* (456) other_para_list ::= other_para_list NK_COMMA star_func_para */ - -1, /* (457) star_func_para ::= expr_or_subquery */ - -3, /* (458) star_func_para ::= table_name NK_DOT NK_STAR */ - -4, /* (459) case_when_expression ::= CASE when_then_list case_when_else_opt END */ - -5, /* (460) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ - -1, /* (461) when_then_list ::= when_then_expr */ - -2, /* (462) when_then_list ::= when_then_list when_then_expr */ - -4, /* (463) when_then_expr ::= WHEN common_expression THEN common_expression */ - 0, /* (464) case_when_else_opt ::= */ - -2, /* (465) case_when_else_opt ::= ELSE common_expression */ - -3, /* (466) predicate ::= expr_or_subquery compare_op expr_or_subquery */ - -5, /* (467) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ - -6, /* (468) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ - -3, /* (469) predicate ::= expr_or_subquery IS NULL */ - -4, /* (470) predicate ::= expr_or_subquery IS NOT NULL */ - -3, /* (471) predicate ::= expr_or_subquery in_op in_predicate_value */ - -1, /* (472) compare_op ::= NK_LT */ - -1, /* (473) compare_op ::= NK_GT */ - -1, /* (474) compare_op ::= NK_LE */ - -1, /* (475) compare_op ::= NK_GE */ - -1, /* (476) compare_op ::= NK_NE */ - -1, /* (477) compare_op ::= NK_EQ */ - -1, /* (478) compare_op ::= LIKE */ - -2, /* (479) compare_op ::= NOT LIKE */ - -1, /* (480) compare_op ::= MATCH */ - -1, /* (481) compare_op ::= NMATCH */ - -1, /* (482) compare_op ::= CONTAINS */ - -1, /* (483) in_op ::= IN */ - -2, /* (484) in_op ::= NOT IN */ - -3, /* (485) in_predicate_value ::= NK_LP literal_list NK_RP */ - -1, /* (486) boolean_value_expression ::= boolean_primary */ - -2, /* (487) boolean_value_expression ::= NOT boolean_primary */ - -3, /* (488) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - -3, /* (489) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - -1, /* (490) boolean_primary ::= predicate */ - -3, /* (491) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - -1, /* (492) common_expression ::= expr_or_subquery */ - -1, /* (493) common_expression ::= boolean_value_expression */ - 0, /* (494) from_clause_opt ::= */ - -2, /* (495) from_clause_opt ::= FROM table_reference_list */ - -1, /* (496) table_reference_list ::= table_reference */ - -3, /* (497) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - -1, /* (498) table_reference ::= table_primary */ - -1, /* (499) table_reference ::= joined_table */ - -2, /* (500) table_primary ::= table_name alias_opt */ - -4, /* (501) table_primary ::= db_name NK_DOT table_name alias_opt */ - -2, /* (502) table_primary ::= subquery alias_opt */ - -1, /* (503) table_primary ::= parenthesized_joined_table */ - 0, /* (504) alias_opt ::= */ - -1, /* (505) alias_opt ::= table_alias */ - -2, /* (506) alias_opt ::= AS table_alias */ - -3, /* (507) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - -3, /* (508) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - -6, /* (509) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - 0, /* (510) join_type ::= */ - -1, /* (511) join_type ::= INNER */ - -12, /* (512) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - 0, /* (513) set_quantifier_opt ::= */ - -1, /* (514) set_quantifier_opt ::= DISTINCT */ - -1, /* (515) set_quantifier_opt ::= ALL */ - -1, /* (516) select_list ::= select_item */ - -3, /* (517) select_list ::= select_list NK_COMMA select_item */ - -1, /* (518) select_item ::= NK_STAR */ - -1, /* (519) select_item ::= common_expression */ - -2, /* (520) select_item ::= common_expression column_alias */ - -3, /* (521) select_item ::= common_expression AS column_alias */ - -3, /* (522) select_item ::= table_name NK_DOT NK_STAR */ - 0, /* (523) where_clause_opt ::= */ - -2, /* (524) where_clause_opt ::= WHERE search_condition */ - 0, /* (525) partition_by_clause_opt ::= */ - -3, /* (526) partition_by_clause_opt ::= PARTITION BY partition_list */ - -1, /* (527) partition_list ::= partition_item */ - -3, /* (528) partition_list ::= partition_list NK_COMMA partition_item */ - -1, /* (529) partition_item ::= expr_or_subquery */ - -2, /* (530) partition_item ::= expr_or_subquery column_alias */ - -3, /* (531) partition_item ::= expr_or_subquery AS column_alias */ - 0, /* (532) twindow_clause_opt ::= */ - -6, /* (533) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - -4, /* (534) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ - -6, /* (535) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - -8, /* (536) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - -7, /* (537) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ - 0, /* (538) sliding_opt ::= */ - -4, /* (539) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - 0, /* (540) fill_opt ::= */ - -4, /* (541) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - -6, /* (542) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ - -6, /* (543) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ - -1, /* (544) fill_mode ::= NONE */ - -1, /* (545) fill_mode ::= PREV */ - -1, /* (546) fill_mode ::= NULL */ - -1, /* (547) fill_mode ::= NULL_F */ - -1, /* (548) fill_mode ::= LINEAR */ - -1, /* (549) fill_mode ::= NEXT */ - 0, /* (550) group_by_clause_opt ::= */ - -3, /* (551) group_by_clause_opt ::= GROUP BY group_by_list */ - -1, /* (552) group_by_list ::= expr_or_subquery */ - -3, /* (553) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ - 0, /* (554) having_clause_opt ::= */ - -2, /* (555) having_clause_opt ::= HAVING search_condition */ - 0, /* (556) range_opt ::= */ - -6, /* (557) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ - 0, /* (558) every_opt ::= */ - -4, /* (559) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - -4, /* (560) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ - -1, /* (561) query_simple ::= query_specification */ - -1, /* (562) query_simple ::= union_query_expression */ - -4, /* (563) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ - -3, /* (564) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ - -1, /* (565) query_simple_or_subquery ::= query_simple */ - -1, /* (566) query_simple_or_subquery ::= subquery */ - -1, /* (567) query_or_subquery ::= query_expression */ - -1, /* (568) query_or_subquery ::= subquery */ - 0, /* (569) order_by_clause_opt ::= */ - -3, /* (570) order_by_clause_opt ::= ORDER BY sort_specification_list */ - 0, /* (571) slimit_clause_opt ::= */ - -2, /* (572) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - -4, /* (573) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - -4, /* (574) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - 0, /* (575) limit_clause_opt ::= */ - -2, /* (576) limit_clause_opt ::= LIMIT NK_INTEGER */ - -4, /* (577) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - -4, /* (578) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - -3, /* (579) subquery ::= NK_LP query_expression NK_RP */ - -3, /* (580) subquery ::= NK_LP subquery NK_RP */ - -1, /* (581) search_condition ::= common_expression */ - -1, /* (582) sort_specification_list ::= sort_specification */ - -3, /* (583) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - -3, /* (584) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ - 0, /* (585) ordering_specification_opt ::= */ - -1, /* (586) ordering_specification_opt ::= ASC */ - -1, /* (587) ordering_specification_opt ::= DESC */ - 0, /* (588) null_ordering_opt ::= */ - -2, /* (589) null_ordering_opt ::= NULLS FIRST */ - -2, /* (590) null_ordering_opt ::= NULLS LAST */ + -4, /* (50) cmd ::= DROP DNODE NK_INTEGER unsafe_opt */ + -4, /* (51) cmd ::= DROP DNODE dnode_endpoint unsafe_opt */ + -4, /* (52) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + -5, /* (53) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + -4, /* (54) cmd ::= ALTER ALL DNODES NK_STRING */ + -5, /* (55) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + -3, /* (56) cmd ::= RESTORE DNODE NK_INTEGER */ + -1, /* (57) dnode_endpoint ::= NK_STRING */ + -1, /* (58) dnode_endpoint ::= NK_ID */ + -1, /* (59) dnode_endpoint ::= NK_IPTOKEN */ + 0, /* (60) force_opt ::= */ + -1, /* (61) force_opt ::= FORCE */ + -1, /* (62) unsafe_opt ::= UNSAFE */ + -3, /* (63) cmd ::= ALTER LOCAL NK_STRING */ + -4, /* (64) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + -5, /* (65) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + -5, /* (66) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + -5, /* (67) cmd ::= RESTORE QNODE ON DNODE NK_INTEGER */ + -5, /* (68) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + -5, /* (69) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + -5, /* (70) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + -5, /* (71) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + -5, /* (72) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + -5, /* (73) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + -5, /* (74) cmd ::= RESTORE MNODE ON DNODE NK_INTEGER */ + -5, /* (75) cmd ::= RESTORE VNODE ON DNODE NK_INTEGER */ + -5, /* (76) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + -4, /* (77) cmd ::= DROP DATABASE exists_opt db_name */ + -2, /* (78) cmd ::= USE db_name */ + -4, /* (79) cmd ::= ALTER DATABASE db_name alter_db_options */ + -3, /* (80) cmd ::= FLUSH DATABASE db_name */ + -4, /* (81) cmd ::= TRIM DATABASE db_name speed_opt */ + -5, /* (82) cmd ::= COMPACT DATABASE db_name start_opt end_opt */ + -3, /* (83) not_exists_opt ::= IF NOT EXISTS */ + 0, /* (84) not_exists_opt ::= */ + -2, /* (85) exists_opt ::= IF EXISTS */ + 0, /* (86) exists_opt ::= */ + 0, /* (87) db_options ::= */ + -3, /* (88) db_options ::= db_options BUFFER NK_INTEGER */ + -3, /* (89) db_options ::= db_options CACHEMODEL NK_STRING */ + -3, /* (90) db_options ::= db_options CACHESIZE NK_INTEGER */ + -3, /* (91) db_options ::= db_options COMP NK_INTEGER */ + -3, /* (92) db_options ::= db_options DURATION NK_INTEGER */ + -3, /* (93) db_options ::= db_options DURATION NK_VARIABLE */ + -3, /* (94) db_options ::= db_options MAXROWS NK_INTEGER */ + -3, /* (95) db_options ::= db_options MINROWS NK_INTEGER */ + -3, /* (96) db_options ::= db_options KEEP integer_list */ + -3, /* (97) db_options ::= db_options KEEP variable_list */ + -3, /* (98) db_options ::= db_options PAGES NK_INTEGER */ + -3, /* (99) db_options ::= db_options PAGESIZE NK_INTEGER */ + -3, /* (100) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ + -3, /* (101) db_options ::= db_options PRECISION NK_STRING */ + -3, /* (102) db_options ::= db_options REPLICA NK_INTEGER */ + -3, /* (103) db_options ::= db_options VGROUPS NK_INTEGER */ + -3, /* (104) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + -3, /* (105) db_options ::= db_options RETENTIONS retention_list */ + -3, /* (106) db_options ::= db_options SCHEMALESS NK_INTEGER */ + -3, /* (107) db_options ::= db_options WAL_LEVEL NK_INTEGER */ + -3, /* (108) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ + -3, /* (109) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ + -4, /* (110) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + -3, /* (111) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ + -4, /* (112) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + -3, /* (113) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ + -3, /* (114) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ + -3, /* (115) db_options ::= db_options STT_TRIGGER NK_INTEGER */ + -3, /* (116) db_options ::= db_options TABLE_PREFIX signed */ + -3, /* (117) db_options ::= db_options TABLE_SUFFIX signed */ + -1, /* (118) alter_db_options ::= alter_db_option */ + -2, /* (119) alter_db_options ::= alter_db_options alter_db_option */ + -2, /* (120) alter_db_option ::= BUFFER NK_INTEGER */ + -2, /* (121) alter_db_option ::= CACHEMODEL NK_STRING */ + -2, /* (122) alter_db_option ::= CACHESIZE NK_INTEGER */ + -2, /* (123) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ + -2, /* (124) alter_db_option ::= KEEP integer_list */ + -2, /* (125) alter_db_option ::= KEEP variable_list */ + -2, /* (126) alter_db_option ::= PAGES NK_INTEGER */ + -2, /* (127) alter_db_option ::= REPLICA NK_INTEGER */ + -2, /* (128) alter_db_option ::= WAL_LEVEL NK_INTEGER */ + -2, /* (129) alter_db_option ::= STT_TRIGGER NK_INTEGER */ + -2, /* (130) alter_db_option ::= MINROWS NK_INTEGER */ + -2, /* (131) alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ + -3, /* (132) alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + -2, /* (133) alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ + -3, /* (134) alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + -1, /* (135) integer_list ::= NK_INTEGER */ + -3, /* (136) integer_list ::= integer_list NK_COMMA NK_INTEGER */ + -1, /* (137) variable_list ::= NK_VARIABLE */ + -3, /* (138) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ + -1, /* (139) retention_list ::= retention */ + -3, /* (140) retention_list ::= retention_list NK_COMMA retention */ + -3, /* (141) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ + 0, /* (142) speed_opt ::= */ + -2, /* (143) speed_opt ::= MAX_SPEED NK_INTEGER */ + 0, /* (144) start_opt ::= */ + -3, /* (145) start_opt ::= START WITH NK_INTEGER */ + -3, /* (146) start_opt ::= START WITH NK_STRING */ + -4, /* (147) start_opt ::= START WITH TIMESTAMP NK_STRING */ + 0, /* (148) end_opt ::= */ + -3, /* (149) end_opt ::= END WITH NK_INTEGER */ + -3, /* (150) end_opt ::= END WITH NK_STRING */ + -4, /* (151) end_opt ::= END WITH TIMESTAMP NK_STRING */ + -9, /* (152) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + -3, /* (153) cmd ::= CREATE TABLE multi_create_clause */ + -9, /* (154) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + -3, /* (155) cmd ::= DROP TABLE multi_drop_clause */ + -4, /* (156) cmd ::= DROP STABLE exists_opt full_table_name */ + -3, /* (157) cmd ::= ALTER TABLE alter_table_clause */ + -3, /* (158) cmd ::= ALTER STABLE alter_table_clause */ + -2, /* (159) alter_table_clause ::= full_table_name alter_table_options */ + -5, /* (160) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + -4, /* (161) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + -5, /* (162) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + -5, /* (163) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + -5, /* (164) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + -4, /* (165) alter_table_clause ::= full_table_name DROP TAG column_name */ + -5, /* (166) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + -5, /* (167) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + -6, /* (168) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ + -1, /* (169) multi_create_clause ::= create_subtable_clause */ + -2, /* (170) multi_create_clause ::= multi_create_clause create_subtable_clause */ + -10, /* (171) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ + -1, /* (172) multi_drop_clause ::= drop_table_clause */ + -3, /* (173) multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ + -2, /* (174) drop_table_clause ::= exists_opt full_table_name */ + 0, /* (175) specific_cols_opt ::= */ + -3, /* (176) specific_cols_opt ::= NK_LP col_name_list NK_RP */ + -1, /* (177) full_table_name ::= table_name */ + -3, /* (178) full_table_name ::= db_name NK_DOT table_name */ + -1, /* (179) column_def_list ::= column_def */ + -3, /* (180) column_def_list ::= column_def_list NK_COMMA column_def */ + -2, /* (181) column_def ::= column_name type_name */ + -1, /* (182) type_name ::= BOOL */ + -1, /* (183) type_name ::= TINYINT */ + -1, /* (184) type_name ::= SMALLINT */ + -1, /* (185) type_name ::= INT */ + -1, /* (186) type_name ::= INTEGER */ + -1, /* (187) type_name ::= BIGINT */ + -1, /* (188) type_name ::= FLOAT */ + -1, /* (189) type_name ::= DOUBLE */ + -4, /* (190) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + -1, /* (191) type_name ::= TIMESTAMP */ + -4, /* (192) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + -2, /* (193) type_name ::= TINYINT UNSIGNED */ + -2, /* (194) type_name ::= SMALLINT UNSIGNED */ + -2, /* (195) type_name ::= INT UNSIGNED */ + -2, /* (196) type_name ::= BIGINT UNSIGNED */ + -1, /* (197) type_name ::= JSON */ + -4, /* (198) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + -1, /* (199) type_name ::= MEDIUMBLOB */ + -1, /* (200) type_name ::= BLOB */ + -4, /* (201) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + -4, /* (202) type_name ::= GEOMETRY NK_LP NK_INTEGER NK_RP */ + -1, /* (203) type_name ::= DECIMAL */ + -4, /* (204) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + -6, /* (205) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + 0, /* (206) tags_def_opt ::= */ + -1, /* (207) tags_def_opt ::= tags_def */ + -4, /* (208) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + 0, /* (209) table_options ::= */ + -3, /* (210) table_options ::= table_options COMMENT NK_STRING */ + -3, /* (211) table_options ::= table_options MAX_DELAY duration_list */ + -3, /* (212) table_options ::= table_options WATERMARK duration_list */ + -5, /* (213) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ + -3, /* (214) table_options ::= table_options TTL NK_INTEGER */ + -5, /* (215) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + -3, /* (216) table_options ::= table_options DELETE_MARK duration_list */ + -1, /* (217) alter_table_options ::= alter_table_option */ + -2, /* (218) alter_table_options ::= alter_table_options alter_table_option */ + -2, /* (219) alter_table_option ::= COMMENT NK_STRING */ + -2, /* (220) alter_table_option ::= TTL NK_INTEGER */ + -1, /* (221) duration_list ::= duration_literal */ + -3, /* (222) duration_list ::= duration_list NK_COMMA duration_literal */ + -1, /* (223) rollup_func_list ::= rollup_func_name */ + -3, /* (224) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ + -1, /* (225) rollup_func_name ::= function_name */ + -1, /* (226) rollup_func_name ::= FIRST */ + -1, /* (227) rollup_func_name ::= LAST */ + -1, /* (228) col_name_list ::= col_name */ + -3, /* (229) col_name_list ::= col_name_list NK_COMMA col_name */ + -1, /* (230) col_name ::= column_name */ + -2, /* (231) cmd ::= SHOW DNODES */ + -2, /* (232) cmd ::= SHOW USERS */ + -3, /* (233) cmd ::= SHOW USER PRIVILEGES */ + -2, /* (234) cmd ::= SHOW DATABASES */ + -4, /* (235) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + -4, /* (236) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + -3, /* (237) cmd ::= SHOW db_name_cond_opt VGROUPS */ + -2, /* (238) cmd ::= SHOW MNODES */ + -2, /* (239) cmd ::= SHOW QNODES */ + -2, /* (240) cmd ::= SHOW FUNCTIONS */ + -5, /* (241) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + -6, /* (242) cmd ::= SHOW INDEXES FROM db_name NK_DOT table_name */ + -2, /* (243) cmd ::= SHOW STREAMS */ + -2, /* (244) cmd ::= SHOW ACCOUNTS */ + -2, /* (245) cmd ::= SHOW APPS */ + -2, /* (246) cmd ::= SHOW CONNECTIONS */ + -2, /* (247) cmd ::= SHOW LICENCES */ + -2, /* (248) cmd ::= SHOW GRANTS */ + -4, /* (249) cmd ::= SHOW CREATE DATABASE db_name */ + -4, /* (250) cmd ::= SHOW CREATE TABLE full_table_name */ + -4, /* (251) cmd ::= SHOW CREATE STABLE full_table_name */ + -2, /* (252) cmd ::= SHOW QUERIES */ + -2, /* (253) cmd ::= SHOW SCORES */ + -2, /* (254) cmd ::= SHOW TOPICS */ + -2, /* (255) cmd ::= SHOW VARIABLES */ + -3, /* (256) cmd ::= SHOW CLUSTER VARIABLES */ + -3, /* (257) cmd ::= SHOW LOCAL VARIABLES */ + -5, /* (258) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ + -2, /* (259) cmd ::= SHOW BNODES */ + -2, /* (260) cmd ::= SHOW SNODES */ + -2, /* (261) cmd ::= SHOW CLUSTER */ + -2, /* (262) cmd ::= SHOW TRANSACTIONS */ + -4, /* (263) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ + -2, /* (264) cmd ::= SHOW CONSUMERS */ + -2, /* (265) cmd ::= SHOW SUBSCRIPTIONS */ + -5, /* (266) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ + -6, /* (267) cmd ::= SHOW TAGS FROM db_name NK_DOT table_name */ + -7, /* (268) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ + -8, /* (269) cmd ::= SHOW TABLE TAGS tag_list_opt FROM db_name NK_DOT table_name */ + -3, /* (270) cmd ::= SHOW VNODES NK_INTEGER */ + -3, /* (271) cmd ::= SHOW VNODES NK_STRING */ + -3, /* (272) cmd ::= SHOW db_name_cond_opt ALIVE */ + -3, /* (273) cmd ::= SHOW CLUSTER ALIVE */ + 0, /* (274) db_name_cond_opt ::= */ + -2, /* (275) db_name_cond_opt ::= db_name NK_DOT */ + 0, /* (276) like_pattern_opt ::= */ + -2, /* (277) like_pattern_opt ::= LIKE NK_STRING */ + -1, /* (278) table_name_cond ::= table_name */ + 0, /* (279) from_db_opt ::= */ + -2, /* (280) from_db_opt ::= FROM db_name */ + 0, /* (281) tag_list_opt ::= */ + -1, /* (282) tag_list_opt ::= tag_item */ + -3, /* (283) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ + -1, /* (284) tag_item ::= TBNAME */ + -1, /* (285) tag_item ::= QTAGS */ + -1, /* (286) tag_item ::= column_name */ + -2, /* (287) tag_item ::= column_name column_alias */ + -3, /* (288) tag_item ::= column_name AS column_alias */ + -8, /* (289) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ + -9, /* (290) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ + -4, /* (291) cmd ::= DROP INDEX exists_opt full_index_name */ + -1, /* (292) full_index_name ::= index_name */ + -3, /* (293) full_index_name ::= db_name NK_DOT index_name */ + -10, /* (294) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ + -12, /* (295) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ + -1, /* (296) func_list ::= func */ + -3, /* (297) func_list ::= func_list NK_COMMA func */ + -4, /* (298) func ::= sma_func_name NK_LP expression_list NK_RP */ + -1, /* (299) sma_func_name ::= function_name */ + -1, /* (300) sma_func_name ::= COUNT */ + -1, /* (301) sma_func_name ::= FIRST */ + -1, /* (302) sma_func_name ::= LAST */ + -1, /* (303) sma_func_name ::= LAST_ROW */ + 0, /* (304) sma_stream_opt ::= */ + -3, /* (305) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + -3, /* (306) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ + -3, /* (307) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ + -1, /* (308) with_meta ::= AS */ + -3, /* (309) with_meta ::= WITH META AS */ + -3, /* (310) with_meta ::= ONLY META AS */ + -6, /* (311) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ + -7, /* (312) cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta DATABASE db_name */ + -8, /* (313) cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta STABLE full_table_name where_clause_opt */ + -4, /* (314) cmd ::= DROP TOPIC exists_opt topic_name */ + -7, /* (315) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ + -2, /* (316) cmd ::= DESC full_table_name */ + -2, /* (317) cmd ::= DESCRIBE full_table_name */ + -3, /* (318) cmd ::= RESET QUERY CACHE */ + -4, /* (319) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + -4, /* (320) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ + 0, /* (321) analyze_opt ::= */ + -1, /* (322) analyze_opt ::= ANALYZE */ + 0, /* (323) explain_options ::= */ + -3, /* (324) explain_options ::= explain_options VERBOSE NK_BOOL */ + -3, /* (325) explain_options ::= explain_options RATIO NK_FLOAT */ + -12, /* (326) cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ + -4, /* (327) cmd ::= DROP FUNCTION exists_opt function_name */ + 0, /* (328) agg_func_opt ::= */ + -1, /* (329) agg_func_opt ::= AGGREGATE */ + 0, /* (330) bufsize_opt ::= */ + -2, /* (331) bufsize_opt ::= BUFSIZE NK_INTEGER */ + 0, /* (332) language_opt ::= */ + -2, /* (333) language_opt ::= LANGUAGE NK_STRING */ + 0, /* (334) or_replace_opt ::= */ + -2, /* (335) or_replace_opt ::= OR REPLACE */ + -12, /* (336) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ + -4, /* (337) cmd ::= DROP STREAM exists_opt stream_name */ + -4, /* (338) cmd ::= PAUSE STREAM exists_opt stream_name */ + -5, /* (339) cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */ + 0, /* (340) col_list_opt ::= */ + -3, /* (341) col_list_opt ::= NK_LP col_name_list NK_RP */ + 0, /* (342) tag_def_or_ref_opt ::= */ + -1, /* (343) tag_def_or_ref_opt ::= tags_def */ + -4, /* (344) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ + 0, /* (345) stream_options ::= */ + -3, /* (346) stream_options ::= stream_options TRIGGER AT_ONCE */ + -3, /* (347) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + -4, /* (348) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + -3, /* (349) stream_options ::= stream_options WATERMARK duration_literal */ + -4, /* (350) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ + -3, /* (351) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ + -3, /* (352) stream_options ::= stream_options DELETE_MARK duration_literal */ + -4, /* (353) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ + 0, /* (354) subtable_opt ::= */ + -4, /* (355) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + 0, /* (356) ignore_opt ::= */ + -2, /* (357) ignore_opt ::= IGNORE UNTREATED */ + -3, /* (358) cmd ::= KILL CONNECTION NK_INTEGER */ + -3, /* (359) cmd ::= KILL QUERY NK_STRING */ + -3, /* (360) cmd ::= KILL TRANSACTION NK_INTEGER */ + -2, /* (361) cmd ::= BALANCE VGROUP */ + -3, /* (362) cmd ::= BALANCE VGROUP LEADER */ + -4, /* (363) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + -4, /* (364) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + -3, /* (365) cmd ::= SPLIT VGROUP NK_INTEGER */ + -2, /* (366) dnode_list ::= DNODE NK_INTEGER */ + -3, /* (367) dnode_list ::= dnode_list DNODE NK_INTEGER */ + -4, /* (368) cmd ::= DELETE FROM full_table_name where_clause_opt */ + -1, /* (369) cmd ::= query_or_subquery */ + -1, /* (370) cmd ::= insert_query */ + -7, /* (371) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ + -4, /* (372) insert_query ::= INSERT INTO full_table_name query_or_subquery */ + -1, /* (373) literal ::= NK_INTEGER */ + -1, /* (374) literal ::= NK_FLOAT */ + -1, /* (375) literal ::= NK_STRING */ + -1, /* (376) literal ::= NK_BOOL */ + -2, /* (377) literal ::= TIMESTAMP NK_STRING */ + -1, /* (378) literal ::= duration_literal */ + -1, /* (379) literal ::= NULL */ + -1, /* (380) literal ::= NK_QUESTION */ + -1, /* (381) duration_literal ::= NK_VARIABLE */ + -1, /* (382) signed ::= NK_INTEGER */ + -2, /* (383) signed ::= NK_PLUS NK_INTEGER */ + -2, /* (384) signed ::= NK_MINUS NK_INTEGER */ + -1, /* (385) signed ::= NK_FLOAT */ + -2, /* (386) signed ::= NK_PLUS NK_FLOAT */ + -2, /* (387) signed ::= NK_MINUS NK_FLOAT */ + -1, /* (388) signed_literal ::= signed */ + -1, /* (389) signed_literal ::= NK_STRING */ + -1, /* (390) signed_literal ::= NK_BOOL */ + -2, /* (391) signed_literal ::= TIMESTAMP NK_STRING */ + -1, /* (392) signed_literal ::= duration_literal */ + -1, /* (393) signed_literal ::= NULL */ + -1, /* (394) signed_literal ::= literal_func */ + -1, /* (395) signed_literal ::= NK_QUESTION */ + -1, /* (396) literal_list ::= signed_literal */ + -3, /* (397) literal_list ::= literal_list NK_COMMA signed_literal */ + -1, /* (398) db_name ::= NK_ID */ + -1, /* (399) table_name ::= NK_ID */ + -1, /* (400) column_name ::= NK_ID */ + -1, /* (401) function_name ::= NK_ID */ + -1, /* (402) table_alias ::= NK_ID */ + -1, /* (403) column_alias ::= NK_ID */ + -1, /* (404) user_name ::= NK_ID */ + -1, /* (405) topic_name ::= NK_ID */ + -1, /* (406) stream_name ::= NK_ID */ + -1, /* (407) cgroup_name ::= NK_ID */ + -1, /* (408) index_name ::= NK_ID */ + -1, /* (409) expr_or_subquery ::= expression */ + -1, /* (410) expression ::= literal */ + -1, /* (411) expression ::= pseudo_column */ + -1, /* (412) expression ::= column_reference */ + -1, /* (413) expression ::= function_expression */ + -1, /* (414) expression ::= case_when_expression */ + -3, /* (415) expression ::= NK_LP expression NK_RP */ + -2, /* (416) expression ::= NK_PLUS expr_or_subquery */ + -2, /* (417) expression ::= NK_MINUS expr_or_subquery */ + -3, /* (418) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + -3, /* (419) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + -3, /* (420) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + -3, /* (421) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + -3, /* (422) expression ::= expr_or_subquery NK_REM expr_or_subquery */ + -3, /* (423) expression ::= column_reference NK_ARROW NK_STRING */ + -3, /* (424) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + -3, /* (425) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + -1, /* (426) expression_list ::= expr_or_subquery */ + -3, /* (427) expression_list ::= expression_list NK_COMMA expr_or_subquery */ + -1, /* (428) column_reference ::= column_name */ + -3, /* (429) column_reference ::= table_name NK_DOT column_name */ + -1, /* (430) pseudo_column ::= ROWTS */ + -1, /* (431) pseudo_column ::= TBNAME */ + -3, /* (432) pseudo_column ::= table_name NK_DOT TBNAME */ + -1, /* (433) pseudo_column ::= QSTART */ + -1, /* (434) pseudo_column ::= QEND */ + -1, /* (435) pseudo_column ::= QDURATION */ + -1, /* (436) pseudo_column ::= WSTART */ + -1, /* (437) pseudo_column ::= WEND */ + -1, /* (438) pseudo_column ::= WDURATION */ + -1, /* (439) pseudo_column ::= IROWTS */ + -1, /* (440) pseudo_column ::= ISFILLED */ + -1, /* (441) pseudo_column ::= QTAGS */ + -4, /* (442) function_expression ::= function_name NK_LP expression_list NK_RP */ + -4, /* (443) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + -6, /* (444) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ + -1, /* (445) function_expression ::= literal_func */ + -3, /* (446) literal_func ::= noarg_func NK_LP NK_RP */ + -1, /* (447) literal_func ::= NOW */ + -1, /* (448) noarg_func ::= NOW */ + -1, /* (449) noarg_func ::= TODAY */ + -1, /* (450) noarg_func ::= TIMEZONE */ + -1, /* (451) noarg_func ::= DATABASE */ + -1, /* (452) noarg_func ::= CLIENT_VERSION */ + -1, /* (453) noarg_func ::= SERVER_VERSION */ + -1, /* (454) noarg_func ::= SERVER_STATUS */ + -1, /* (455) noarg_func ::= CURRENT_USER */ + -1, /* (456) noarg_func ::= USER */ + -1, /* (457) star_func ::= COUNT */ + -1, /* (458) star_func ::= FIRST */ + -1, /* (459) star_func ::= LAST */ + -1, /* (460) star_func ::= LAST_ROW */ + -1, /* (461) star_func_para_list ::= NK_STAR */ + -1, /* (462) star_func_para_list ::= other_para_list */ + -1, /* (463) other_para_list ::= star_func_para */ + -3, /* (464) other_para_list ::= other_para_list NK_COMMA star_func_para */ + -1, /* (465) star_func_para ::= expr_or_subquery */ + -3, /* (466) star_func_para ::= table_name NK_DOT NK_STAR */ + -4, /* (467) case_when_expression ::= CASE when_then_list case_when_else_opt END */ + -5, /* (468) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ + -1, /* (469) when_then_list ::= when_then_expr */ + -2, /* (470) when_then_list ::= when_then_list when_then_expr */ + -4, /* (471) when_then_expr ::= WHEN common_expression THEN common_expression */ + 0, /* (472) case_when_else_opt ::= */ + -2, /* (473) case_when_else_opt ::= ELSE common_expression */ + -3, /* (474) predicate ::= expr_or_subquery compare_op expr_or_subquery */ + -5, /* (475) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + -6, /* (476) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + -3, /* (477) predicate ::= expr_or_subquery IS NULL */ + -4, /* (478) predicate ::= expr_or_subquery IS NOT NULL */ + -3, /* (479) predicate ::= expr_or_subquery in_op in_predicate_value */ + -1, /* (480) compare_op ::= NK_LT */ + -1, /* (481) compare_op ::= NK_GT */ + -1, /* (482) compare_op ::= NK_LE */ + -1, /* (483) compare_op ::= NK_GE */ + -1, /* (484) compare_op ::= NK_NE */ + -1, /* (485) compare_op ::= NK_EQ */ + -1, /* (486) compare_op ::= LIKE */ + -2, /* (487) compare_op ::= NOT LIKE */ + -1, /* (488) compare_op ::= MATCH */ + -1, /* (489) compare_op ::= NMATCH */ + -1, /* (490) compare_op ::= CONTAINS */ + -1, /* (491) in_op ::= IN */ + -2, /* (492) in_op ::= NOT IN */ + -3, /* (493) in_predicate_value ::= NK_LP literal_list NK_RP */ + -1, /* (494) boolean_value_expression ::= boolean_primary */ + -2, /* (495) boolean_value_expression ::= NOT boolean_primary */ + -3, /* (496) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + -3, /* (497) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + -1, /* (498) boolean_primary ::= predicate */ + -3, /* (499) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + -1, /* (500) common_expression ::= expr_or_subquery */ + -1, /* (501) common_expression ::= boolean_value_expression */ + 0, /* (502) from_clause_opt ::= */ + -2, /* (503) from_clause_opt ::= FROM table_reference_list */ + -1, /* (504) table_reference_list ::= table_reference */ + -3, /* (505) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + -1, /* (506) table_reference ::= table_primary */ + -1, /* (507) table_reference ::= joined_table */ + -2, /* (508) table_primary ::= table_name alias_opt */ + -4, /* (509) table_primary ::= db_name NK_DOT table_name alias_opt */ + -2, /* (510) table_primary ::= subquery alias_opt */ + -1, /* (511) table_primary ::= parenthesized_joined_table */ + 0, /* (512) alias_opt ::= */ + -1, /* (513) alias_opt ::= table_alias */ + -2, /* (514) alias_opt ::= AS table_alias */ + -3, /* (515) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + -3, /* (516) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + -6, /* (517) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + 0, /* (518) join_type ::= */ + -1, /* (519) join_type ::= INNER */ + -12, /* (520) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + 0, /* (521) set_quantifier_opt ::= */ + -1, /* (522) set_quantifier_opt ::= DISTINCT */ + -1, /* (523) set_quantifier_opt ::= ALL */ + -1, /* (524) select_list ::= select_item */ + -3, /* (525) select_list ::= select_list NK_COMMA select_item */ + -1, /* (526) select_item ::= NK_STAR */ + -1, /* (527) select_item ::= common_expression */ + -2, /* (528) select_item ::= common_expression column_alias */ + -3, /* (529) select_item ::= common_expression AS column_alias */ + -3, /* (530) select_item ::= table_name NK_DOT NK_STAR */ + 0, /* (531) where_clause_opt ::= */ + -2, /* (532) where_clause_opt ::= WHERE search_condition */ + 0, /* (533) partition_by_clause_opt ::= */ + -3, /* (534) partition_by_clause_opt ::= PARTITION BY partition_list */ + -1, /* (535) partition_list ::= partition_item */ + -3, /* (536) partition_list ::= partition_list NK_COMMA partition_item */ + -1, /* (537) partition_item ::= expr_or_subquery */ + -2, /* (538) partition_item ::= expr_or_subquery column_alias */ + -3, /* (539) partition_item ::= expr_or_subquery AS column_alias */ + 0, /* (540) twindow_clause_opt ::= */ + -6, /* (541) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + -4, /* (542) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ + -6, /* (543) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + -8, /* (544) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + -7, /* (545) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ + 0, /* (546) sliding_opt ::= */ + -4, /* (547) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + 0, /* (548) fill_opt ::= */ + -4, /* (549) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + -6, /* (550) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ + -6, /* (551) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ + -1, /* (552) fill_mode ::= NONE */ + -1, /* (553) fill_mode ::= PREV */ + -1, /* (554) fill_mode ::= NULL */ + -1, /* (555) fill_mode ::= NULL_F */ + -1, /* (556) fill_mode ::= LINEAR */ + -1, /* (557) fill_mode ::= NEXT */ + 0, /* (558) group_by_clause_opt ::= */ + -3, /* (559) group_by_clause_opt ::= GROUP BY group_by_list */ + -1, /* (560) group_by_list ::= expr_or_subquery */ + -3, /* (561) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ + 0, /* (562) having_clause_opt ::= */ + -2, /* (563) having_clause_opt ::= HAVING search_condition */ + 0, /* (564) range_opt ::= */ + -6, /* (565) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ + -4, /* (566) range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ + 0, /* (567) every_opt ::= */ + -4, /* (568) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + -4, /* (569) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + -1, /* (570) query_simple ::= query_specification */ + -1, /* (571) query_simple ::= union_query_expression */ + -4, /* (572) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + -3, /* (573) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + -1, /* (574) query_simple_or_subquery ::= query_simple */ + -1, /* (575) query_simple_or_subquery ::= subquery */ + -1, /* (576) query_or_subquery ::= query_expression */ + -1, /* (577) query_or_subquery ::= subquery */ + 0, /* (578) order_by_clause_opt ::= */ + -3, /* (579) order_by_clause_opt ::= ORDER BY sort_specification_list */ + 0, /* (580) slimit_clause_opt ::= */ + -2, /* (581) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + -4, /* (582) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + -4, /* (583) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + 0, /* (584) limit_clause_opt ::= */ + -2, /* (585) limit_clause_opt ::= LIMIT NK_INTEGER */ + -4, /* (586) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + -4, /* (587) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + -3, /* (588) subquery ::= NK_LP query_expression NK_RP */ + -3, /* (589) subquery ::= NK_LP subquery NK_RP */ + -1, /* (590) search_condition ::= common_expression */ + -1, /* (591) sort_specification_list ::= sort_specification */ + -3, /* (592) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + -3, /* (593) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + 0, /* (594) ordering_specification_opt ::= */ + -1, /* (595) ordering_specification_opt ::= ASC */ + -1, /* (596) ordering_specification_opt ::= DESC */ + 0, /* (597) null_ordering_opt ::= */ + -2, /* (598) null_ordering_opt ::= NULLS FIRST */ + -2, /* (599) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -4503,21 +4487,15 @@ static YYACTIONTYPE yy_reduce( /********** Begin reduce actions **********************************************/ YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ -#line 50 "sql.y" { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } -#line 4509 "sql.c" - yy_destructor(yypParser,336,&yymsp[0].minor); + yy_destructor(yypParser,339,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ -#line 51 "sql.y" { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } -#line 4515 "sql.c" - yy_destructor(yypParser,337,&yymsp[0].minor); + yy_destructor(yypParser,340,&yymsp[0].minor); break; case 2: /* account_options ::= */ -#line 55 "sql.y" { } -#line 4521 "sql.c" break; case 3: /* account_options ::= account_options PPS literal */ case 4: /* account_options ::= account_options TSERIES literal */ yytestcase(yyruleno==4); @@ -4528,26 +4506,20 @@ static YYACTIONTYPE yy_reduce( case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); -{ yy_destructor(yypParser,336,&yymsp[-2].minor); -#line 56 "sql.y" +{ yy_destructor(yypParser,339,&yymsp[-2].minor); { } -#line 4535 "sql.c" - yy_destructor(yypParser,338,&yymsp[0].minor); + yy_destructor(yypParser,341,&yymsp[0].minor); } break; case 12: /* alter_account_options ::= alter_account_option */ -{ yy_destructor(yypParser,339,&yymsp[0].minor); -#line 68 "sql.y" +{ yy_destructor(yypParser,342,&yymsp[0].minor); { } -#line 4543 "sql.c" } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ -{ yy_destructor(yypParser,337,&yymsp[-1].minor); -#line 69 "sql.y" +{ yy_destructor(yypParser,340,&yymsp[-1].minor); { } -#line 4550 "sql.c" - yy_destructor(yypParser,339,&yymsp[0].minor); + yy_destructor(yypParser,342,&yymsp[0].minor); } break; case 14: /* alter_account_option ::= PASS literal */ @@ -4560,2405 +4532,1646 @@ static YYACTIONTYPE yy_reduce( case 21: /* alter_account_option ::= USERS literal */ yytestcase(yyruleno==21); case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); -#line 73 "sql.y" { } -#line 4566 "sql.c" - yy_destructor(yypParser,338,&yymsp[0].minor); + yy_destructor(yypParser,341,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ -#line 85 "sql.y" -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy497, &yymsp[-1].minor.yy0, yymsp[0].minor.yy563); } -#line 4572 "sql.c" +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy371, &yymsp[-1].minor.yy0, yymsp[0].minor.yy475); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -#line 86 "sql.y" -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy497, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } -#line 4577 "sql.c" +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy371, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ -#line 87 "sql.y" -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy497, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } -#line 4582 "sql.c" +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy371, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ -#line 88 "sql.y" -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy497, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } -#line 4587 "sql.c" +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy371, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } break; case 28: /* cmd ::= DROP USER user_name */ -#line 89 "sql.y" -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy497); } -#line 4592 "sql.c" +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy371); } break; case 29: /* sysinfo_opt ::= */ -#line 93 "sql.y" -{ yymsp[1].minor.yy563 = 1; } -#line 4597 "sql.c" +{ yymsp[1].minor.yy475 = 1; } break; case 30: /* sysinfo_opt ::= SYSINFO NK_INTEGER */ -#line 94 "sql.y" -{ yymsp[-1].minor.yy563 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } -#line 4602 "sql.c" +{ yymsp[-1].minor.yy475 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } break; case 31: /* cmd ::= GRANT privileges ON priv_level with_opt TO user_name */ -#line 97 "sql.y" -{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-5].minor.yy693, &yymsp[-3].minor.yy953, &yymsp[0].minor.yy497, yymsp[-2].minor.yy164); } -#line 4607 "sql.c" +{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-5].minor.yy729, &yymsp[-3].minor.yy347, &yymsp[0].minor.yy371, yymsp[-2].minor.yy452); } break; case 32: /* cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name */ -#line 98 "sql.y" -{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-5].minor.yy693, &yymsp[-3].minor.yy953, &yymsp[0].minor.yy497, yymsp[-2].minor.yy164); } -#line 4612 "sql.c" +{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-5].minor.yy729, &yymsp[-3].minor.yy347, &yymsp[0].minor.yy371, yymsp[-2].minor.yy452); } break; case 33: /* privileges ::= ALL */ -#line 102 "sql.y" -{ yymsp[0].minor.yy693 = PRIVILEGE_TYPE_ALL; } -#line 4617 "sql.c" +{ yymsp[0].minor.yy729 = PRIVILEGE_TYPE_ALL; } break; case 34: /* privileges ::= priv_type_list */ case 36: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==36); -#line 103 "sql.y" -{ yylhsminor.yy693 = yymsp[0].minor.yy693; } -#line 4623 "sql.c" - yymsp[0].minor.yy693 = yylhsminor.yy693; +{ yylhsminor.yy729 = yymsp[0].minor.yy729; } + yymsp[0].minor.yy729 = yylhsminor.yy729; break; case 35: /* privileges ::= SUBSCRIBE */ -#line 104 "sql.y" -{ yymsp[0].minor.yy693 = PRIVILEGE_TYPE_SUBSCRIBE; } -#line 4629 "sql.c" +{ yymsp[0].minor.yy729 = PRIVILEGE_TYPE_SUBSCRIBE; } break; case 37: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ -#line 109 "sql.y" -{ yylhsminor.yy693 = yymsp[-2].minor.yy693 | yymsp[0].minor.yy693; } -#line 4634 "sql.c" - yymsp[-2].minor.yy693 = yylhsminor.yy693; +{ yylhsminor.yy729 = yymsp[-2].minor.yy729 | yymsp[0].minor.yy729; } + yymsp[-2].minor.yy729 = yylhsminor.yy729; break; case 38: /* priv_type ::= READ */ -#line 113 "sql.y" -{ yymsp[0].minor.yy693 = PRIVILEGE_TYPE_READ; } -#line 4640 "sql.c" +{ yymsp[0].minor.yy729 = PRIVILEGE_TYPE_READ; } break; case 39: /* priv_type ::= WRITE */ -#line 114 "sql.y" -{ yymsp[0].minor.yy693 = PRIVILEGE_TYPE_WRITE; } -#line 4645 "sql.c" +{ yymsp[0].minor.yy729 = PRIVILEGE_TYPE_WRITE; } break; case 40: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ -#line 118 "sql.y" -{ yylhsminor.yy953.first = yymsp[-2].minor.yy0; yylhsminor.yy953.second = yymsp[0].minor.yy0; } -#line 4650 "sql.c" - yymsp[-2].minor.yy953 = yylhsminor.yy953; +{ yylhsminor.yy347.first = yymsp[-2].minor.yy0; yylhsminor.yy347.second = yymsp[0].minor.yy0; } + yymsp[-2].minor.yy347 = yylhsminor.yy347; break; case 41: /* priv_level ::= db_name NK_DOT NK_STAR */ -#line 119 "sql.y" -{ yylhsminor.yy953.first = yymsp[-2].minor.yy497; yylhsminor.yy953.second = yymsp[0].minor.yy0; } -#line 4656 "sql.c" - yymsp[-2].minor.yy953 = yylhsminor.yy953; +{ yylhsminor.yy347.first = yymsp[-2].minor.yy371; yylhsminor.yy347.second = yymsp[0].minor.yy0; } + yymsp[-2].minor.yy347 = yylhsminor.yy347; break; case 42: /* priv_level ::= db_name NK_DOT table_name */ -#line 120 "sql.y" -{ yylhsminor.yy953.first = yymsp[-2].minor.yy497; yylhsminor.yy953.second = yymsp[0].minor.yy497; } -#line 4662 "sql.c" - yymsp[-2].minor.yy953 = yylhsminor.yy953; +{ yylhsminor.yy347.first = yymsp[-2].minor.yy371; yylhsminor.yy347.second = yymsp[0].minor.yy371; } + yymsp[-2].minor.yy347 = yylhsminor.yy347; break; case 43: /* priv_level ::= topic_name */ -#line 121 "sql.y" -{ yylhsminor.yy953.first = yymsp[0].minor.yy497; yylhsminor.yy953.second = nil_token; } -#line 4668 "sql.c" - yymsp[0].minor.yy953 = yylhsminor.yy953; +{ yylhsminor.yy347.first = yymsp[0].minor.yy371; yylhsminor.yy347.second = nil_token; } + yymsp[0].minor.yy347 = yylhsminor.yy347; break; case 44: /* with_opt ::= */ - case 141: /* start_opt ::= */ yytestcase(yyruleno==141); - case 145: /* end_opt ::= */ yytestcase(yyruleno==145); - case 269: /* like_pattern_opt ::= */ yytestcase(yyruleno==269); - case 346: /* subtable_opt ::= */ yytestcase(yyruleno==346); - case 464: /* case_when_else_opt ::= */ yytestcase(yyruleno==464); - case 494: /* from_clause_opt ::= */ yytestcase(yyruleno==494); - case 523: /* where_clause_opt ::= */ yytestcase(yyruleno==523); - case 532: /* twindow_clause_opt ::= */ yytestcase(yyruleno==532); - case 538: /* sliding_opt ::= */ yytestcase(yyruleno==538); - case 540: /* fill_opt ::= */ yytestcase(yyruleno==540); - case 554: /* having_clause_opt ::= */ yytestcase(yyruleno==554); - case 556: /* range_opt ::= */ yytestcase(yyruleno==556); - case 558: /* every_opt ::= */ yytestcase(yyruleno==558); - case 571: /* slimit_clause_opt ::= */ yytestcase(yyruleno==571); - case 575: /* limit_clause_opt ::= */ yytestcase(yyruleno==575); -#line 123 "sql.y" -{ yymsp[1].minor.yy164 = NULL; } -#line 4689 "sql.c" + case 144: /* start_opt ::= */ yytestcase(yyruleno==144); + case 148: /* end_opt ::= */ yytestcase(yyruleno==148); + case 276: /* like_pattern_opt ::= */ yytestcase(yyruleno==276); + case 354: /* subtable_opt ::= */ yytestcase(yyruleno==354); + case 472: /* case_when_else_opt ::= */ yytestcase(yyruleno==472); + case 502: /* from_clause_opt ::= */ yytestcase(yyruleno==502); + case 531: /* where_clause_opt ::= */ yytestcase(yyruleno==531); + case 540: /* twindow_clause_opt ::= */ yytestcase(yyruleno==540); + case 546: /* sliding_opt ::= */ yytestcase(yyruleno==546); + case 548: /* fill_opt ::= */ yytestcase(yyruleno==548); + case 562: /* having_clause_opt ::= */ yytestcase(yyruleno==562); + case 564: /* range_opt ::= */ yytestcase(yyruleno==564); + case 567: /* every_opt ::= */ yytestcase(yyruleno==567); + case 580: /* slimit_clause_opt ::= */ yytestcase(yyruleno==580); + case 584: /* limit_clause_opt ::= */ yytestcase(yyruleno==584); +{ yymsp[1].minor.yy452 = NULL; } break; case 45: /* with_opt ::= WITH search_condition */ - case 495: /* from_clause_opt ::= FROM table_reference_list */ yytestcase(yyruleno==495); - case 524: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==524); - case 555: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==555); -#line 124 "sql.y" -{ yymsp[-1].minor.yy164 = yymsp[0].minor.yy164; } -#line 4697 "sql.c" + case 503: /* from_clause_opt ::= FROM table_reference_list */ yytestcase(yyruleno==503); + case 532: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==532); + case 563: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==563); +{ yymsp[-1].minor.yy452 = yymsp[0].minor.yy452; } break; case 46: /* cmd ::= CREATE DNODE dnode_endpoint */ -#line 127 "sql.y" -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy497, NULL); } -#line 4702 "sql.c" +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy371, NULL); } break; case 47: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ -#line 128 "sql.y" -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy497, &yymsp[0].minor.yy0); } -#line 4707 "sql.c" +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy0); } break; case 48: /* cmd ::= DROP DNODE NK_INTEGER force_opt */ -#line 129 "sql.y" -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy441); } -#line 4712 "sql.c" +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy667, false); } break; case 49: /* cmd ::= DROP DNODE dnode_endpoint force_opt */ -#line 130 "sql.y" -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy497, yymsp[0].minor.yy441); } -#line 4717 "sql.c" +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy371, yymsp[0].minor.yy667, false); } break; - case 50: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ -#line 131 "sql.y" + case 50: /* cmd ::= DROP DNODE NK_INTEGER unsafe_opt */ +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, false, yymsp[0].minor.yy667); } + break; + case 51: /* cmd ::= DROP DNODE dnode_endpoint unsafe_opt */ +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy371, false, yymsp[0].minor.yy667); } + break; + case 52: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } -#line 4722 "sql.c" break; - case 51: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ -#line 132 "sql.y" + case 53: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } -#line 4727 "sql.c" break; - case 52: /* cmd ::= ALTER ALL DNODES NK_STRING */ -#line 133 "sql.y" + case 54: /* cmd ::= ALTER ALL DNODES NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[0].minor.yy0, NULL); } -#line 4732 "sql.c" break; - case 53: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ -#line 134 "sql.y" + case 55: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } -#line 4737 "sql.c" break; - case 54: /* cmd ::= RESTORE DNODE NK_INTEGER */ -#line 135 "sql.y" + case 56: /* cmd ::= RESTORE DNODE NK_INTEGER */ { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_DNODE_STMT, &yymsp[0].minor.yy0); } -#line 4742 "sql.c" - break; - case 55: /* dnode_endpoint ::= NK_STRING */ - case 56: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==56); - case 57: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==57); - case 293: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==293); - case 294: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==294); - case 295: /* sma_func_name ::= LAST */ yytestcase(yyruleno==295); - case 296: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==296); - case 390: /* db_name ::= NK_ID */ yytestcase(yyruleno==390); - case 391: /* table_name ::= NK_ID */ yytestcase(yyruleno==391); - case 392: /* column_name ::= NK_ID */ yytestcase(yyruleno==392); - case 393: /* function_name ::= NK_ID */ yytestcase(yyruleno==393); - case 394: /* table_alias ::= NK_ID */ yytestcase(yyruleno==394); - case 395: /* column_alias ::= NK_ID */ yytestcase(yyruleno==395); - case 396: /* user_name ::= NK_ID */ yytestcase(yyruleno==396); - case 397: /* topic_name ::= NK_ID */ yytestcase(yyruleno==397); - case 398: /* stream_name ::= NK_ID */ yytestcase(yyruleno==398); - case 399: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==399); - case 400: /* index_name ::= NK_ID */ yytestcase(yyruleno==400); - case 440: /* noarg_func ::= NOW */ yytestcase(yyruleno==440); - case 441: /* noarg_func ::= TODAY */ yytestcase(yyruleno==441); - case 442: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==442); - case 443: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==443); - case 444: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==444); - case 445: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==445); - case 446: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==446); - case 447: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==447); - case 448: /* noarg_func ::= USER */ yytestcase(yyruleno==448); - case 449: /* star_func ::= COUNT */ yytestcase(yyruleno==449); - case 450: /* star_func ::= FIRST */ yytestcase(yyruleno==450); - case 451: /* star_func ::= LAST */ yytestcase(yyruleno==451); - case 452: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==452); -#line 139 "sql.y" -{ yylhsminor.yy497 = yymsp[0].minor.yy0; } -#line 4777 "sql.c" - yymsp[0].minor.yy497 = yylhsminor.yy497; - break; - case 58: /* force_opt ::= */ - case 81: /* not_exists_opt ::= */ yytestcase(yyruleno==81); - case 83: /* exists_opt ::= */ yytestcase(yyruleno==83); - case 313: /* analyze_opt ::= */ yytestcase(yyruleno==313); - case 320: /* agg_func_opt ::= */ yytestcase(yyruleno==320); - case 326: /* or_replace_opt ::= */ yytestcase(yyruleno==326); - case 348: /* ignore_opt ::= */ yytestcase(yyruleno==348); - case 513: /* set_quantifier_opt ::= */ yytestcase(yyruleno==513); -#line 145 "sql.y" -{ yymsp[1].minor.yy441 = false; } -#line 4790 "sql.c" - break; - case 59: /* force_opt ::= FORCE */ - case 314: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==314); - case 321: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==321); - case 514: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==514); -#line 146 "sql.y" -{ yymsp[0].minor.yy441 = true; } -#line 4798 "sql.c" - break; - case 60: /* cmd ::= ALTER LOCAL NK_STRING */ -#line 149 "sql.y" + break; + case 57: /* dnode_endpoint ::= NK_STRING */ + case 58: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==58); + case 59: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==59); + case 300: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==300); + case 301: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==301); + case 302: /* sma_func_name ::= LAST */ yytestcase(yyruleno==302); + case 303: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==303); + case 398: /* db_name ::= NK_ID */ yytestcase(yyruleno==398); + case 399: /* table_name ::= NK_ID */ yytestcase(yyruleno==399); + case 400: /* column_name ::= NK_ID */ yytestcase(yyruleno==400); + case 401: /* function_name ::= NK_ID */ yytestcase(yyruleno==401); + case 402: /* table_alias ::= NK_ID */ yytestcase(yyruleno==402); + case 403: /* column_alias ::= NK_ID */ yytestcase(yyruleno==403); + case 404: /* user_name ::= NK_ID */ yytestcase(yyruleno==404); + case 405: /* topic_name ::= NK_ID */ yytestcase(yyruleno==405); + case 406: /* stream_name ::= NK_ID */ yytestcase(yyruleno==406); + case 407: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==407); + case 408: /* index_name ::= NK_ID */ yytestcase(yyruleno==408); + case 448: /* noarg_func ::= NOW */ yytestcase(yyruleno==448); + case 449: /* noarg_func ::= TODAY */ yytestcase(yyruleno==449); + case 450: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==450); + case 451: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==451); + case 452: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==452); + case 453: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==453); + case 454: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==454); + case 455: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==455); + case 456: /* noarg_func ::= USER */ yytestcase(yyruleno==456); + case 457: /* star_func ::= COUNT */ yytestcase(yyruleno==457); + case 458: /* star_func ::= FIRST */ yytestcase(yyruleno==458); + case 459: /* star_func ::= LAST */ yytestcase(yyruleno==459); + case 460: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==460); +{ yylhsminor.yy371 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy371 = yylhsminor.yy371; + break; + case 60: /* force_opt ::= */ + case 84: /* not_exists_opt ::= */ yytestcase(yyruleno==84); + case 86: /* exists_opt ::= */ yytestcase(yyruleno==86); + case 321: /* analyze_opt ::= */ yytestcase(yyruleno==321); + case 328: /* agg_func_opt ::= */ yytestcase(yyruleno==328); + case 334: /* or_replace_opt ::= */ yytestcase(yyruleno==334); + case 356: /* ignore_opt ::= */ yytestcase(yyruleno==356); + case 521: /* set_quantifier_opt ::= */ yytestcase(yyruleno==521); +{ yymsp[1].minor.yy667 = false; } + break; + case 61: /* force_opt ::= FORCE */ + case 62: /* unsafe_opt ::= UNSAFE */ yytestcase(yyruleno==62); + case 322: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==322); + case 329: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==329); + case 522: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==522); +{ yymsp[0].minor.yy667 = true; } + break; + case 63: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } -#line 4803 "sql.c" break; - case 61: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */ -#line 150 "sql.y" + case 64: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } -#line 4808 "sql.c" break; - case 62: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ -#line 153 "sql.y" + case 65: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_QNODE_STMT, &yymsp[0].minor.yy0); } -#line 4813 "sql.c" break; - case 63: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ -#line 154 "sql.y" + case 66: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_QNODE_STMT, &yymsp[0].minor.yy0); } -#line 4818 "sql.c" break; - case 64: /* cmd ::= RESTORE QNODE ON DNODE NK_INTEGER */ -#line 155 "sql.y" + case 67: /* cmd ::= RESTORE QNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_QNODE_STMT, &yymsp[0].minor.yy0); } -#line 4823 "sql.c" break; - case 65: /* cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ -#line 158 "sql.y" + case 68: /* cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_BNODE_STMT, &yymsp[0].minor.yy0); } -#line 4828 "sql.c" break; - case 66: /* cmd ::= DROP BNODE ON DNODE NK_INTEGER */ -#line 159 "sql.y" + case 69: /* cmd ::= DROP BNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_BNODE_STMT, &yymsp[0].minor.yy0); } -#line 4833 "sql.c" break; - case 67: /* cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ -#line 162 "sql.y" + case 70: /* cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_SNODE_STMT, &yymsp[0].minor.yy0); } -#line 4838 "sql.c" break; - case 68: /* cmd ::= DROP SNODE ON DNODE NK_INTEGER */ -#line 163 "sql.y" + case 71: /* cmd ::= DROP SNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_SNODE_STMT, &yymsp[0].minor.yy0); } -#line 4843 "sql.c" break; - case 69: /* cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ -#line 166 "sql.y" + case 72: /* cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_MNODE_STMT, &yymsp[0].minor.yy0); } -#line 4848 "sql.c" break; - case 70: /* cmd ::= DROP MNODE ON DNODE NK_INTEGER */ -#line 167 "sql.y" + case 73: /* cmd ::= DROP MNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } -#line 4853 "sql.c" break; - case 71: /* cmd ::= RESTORE MNODE ON DNODE NK_INTEGER */ -#line 168 "sql.y" + case 74: /* cmd ::= RESTORE MNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_MNODE_STMT, &yymsp[0].minor.yy0); } -#line 4858 "sql.c" break; - case 72: /* cmd ::= RESTORE VNODE ON DNODE NK_INTEGER */ -#line 171 "sql.y" + case 75: /* cmd ::= RESTORE VNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_VNODE_STMT, &yymsp[0].minor.yy0); } -#line 4863 "sql.c" - break; - case 73: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -#line 174 "sql.y" -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy441, &yymsp[-1].minor.yy497, yymsp[0].minor.yy164); } -#line 4868 "sql.c" - break; - case 74: /* cmd ::= DROP DATABASE exists_opt db_name */ -#line 175 "sql.y" -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy441, &yymsp[0].minor.yy497); } -#line 4873 "sql.c" - break; - case 75: /* cmd ::= USE db_name */ -#line 176 "sql.y" -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy497); } -#line 4878 "sql.c" - break; - case 76: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -#line 177 "sql.y" -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy497, yymsp[0].minor.yy164); } -#line 4883 "sql.c" - break; - case 77: /* cmd ::= FLUSH DATABASE db_name */ -#line 178 "sql.y" -{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy497); } -#line 4888 "sql.c" - break; - case 78: /* cmd ::= TRIM DATABASE db_name speed_opt */ -#line 179 "sql.y" -{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy497, yymsp[0].minor.yy560); } -#line 4893 "sql.c" - break; - case 79: /* cmd ::= COMPACT DATABASE db_name start_opt end_opt */ -#line 180 "sql.y" -{ pCxt->pRootNode = createCompactStmt(pCxt, &yymsp[-2].minor.yy497, yymsp[-1].minor.yy164, yymsp[0].minor.yy164); } -#line 4898 "sql.c" - break; - case 80: /* not_exists_opt ::= IF NOT EXISTS */ -#line 184 "sql.y" -{ yymsp[-2].minor.yy441 = true; } -#line 4903 "sql.c" - break; - case 82: /* exists_opt ::= IF EXISTS */ - case 327: /* or_replace_opt ::= OR REPLACE */ yytestcase(yyruleno==327); - case 349: /* ignore_opt ::= IGNORE UNTREATED */ yytestcase(yyruleno==349); -#line 189 "sql.y" -{ yymsp[-1].minor.yy441 = true; } -#line 4910 "sql.c" - break; - case 84: /* db_options ::= */ -#line 192 "sql.y" -{ yymsp[1].minor.yy164 = createDefaultDatabaseOptions(pCxt); } -#line 4915 "sql.c" - break; - case 85: /* db_options ::= db_options BUFFER NK_INTEGER */ -#line 193 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } -#line 4920 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 86: /* db_options ::= db_options CACHEMODEL NK_STRING */ -#line 194 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } -#line 4926 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 87: /* db_options ::= db_options CACHESIZE NK_INTEGER */ -#line 195 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } -#line 4932 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 88: /* db_options ::= db_options COMP NK_INTEGER */ -#line 196 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_COMP, &yymsp[0].minor.yy0); } -#line 4938 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 89: /* db_options ::= db_options DURATION NK_INTEGER */ - case 90: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==90); -#line 197 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } -#line 4945 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 91: /* db_options ::= db_options MAXROWS NK_INTEGER */ -#line 199 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } -#line 4951 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 92: /* db_options ::= db_options MINROWS NK_INTEGER */ -#line 200 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } -#line 4957 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 93: /* db_options ::= db_options KEEP integer_list */ - case 94: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==94); -#line 201 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_KEEP, yymsp[0].minor.yy72); } -#line 4964 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 95: /* db_options ::= db_options PAGES NK_INTEGER */ -#line 203 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } -#line 4970 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 96: /* db_options ::= db_options PAGESIZE NK_INTEGER */ -#line 204 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } -#line 4976 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 97: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ -#line 205 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } -#line 4982 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 98: /* db_options ::= db_options PRECISION NK_STRING */ -#line 206 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } -#line 4988 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 99: /* db_options ::= db_options REPLICA NK_INTEGER */ -#line 207 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } -#line 4994 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 100: /* db_options ::= db_options VGROUPS NK_INTEGER */ -#line 209 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } -#line 5000 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 101: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -#line 210 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } -#line 5006 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 102: /* db_options ::= db_options RETENTIONS retention_list */ -#line 211 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_RETENTIONS, yymsp[0].minor.yy72); } -#line 5012 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 103: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ -#line 212 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } -#line 5018 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 104: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ -#line 213 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_WAL, &yymsp[0].minor.yy0); } -#line 5024 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 105: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ -#line 214 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } -#line 5030 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 106: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ -#line 215 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } -#line 5036 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 107: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ -#line 216 "sql.y" + break; + case 76: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy667, &yymsp[-1].minor.yy371, yymsp[0].minor.yy452); } + break; + case 77: /* cmd ::= DROP DATABASE exists_opt db_name */ +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy667, &yymsp[0].minor.yy371); } + break; + case 78: /* cmd ::= USE db_name */ +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy371); } + break; + case 79: /* cmd ::= ALTER DATABASE db_name alter_db_options */ +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy371, yymsp[0].minor.yy452); } + break; + case 80: /* cmd ::= FLUSH DATABASE db_name */ +{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy371); } + break; + case 81: /* cmd ::= TRIM DATABASE db_name speed_opt */ +{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy371, yymsp[0].minor.yy416); } + break; + case 82: /* cmd ::= COMPACT DATABASE db_name start_opt end_opt */ +{ pCxt->pRootNode = createCompactStmt(pCxt, &yymsp[-2].minor.yy371, yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } + break; + case 83: /* not_exists_opt ::= IF NOT EXISTS */ +{ yymsp[-2].minor.yy667 = true; } + break; + case 85: /* exists_opt ::= IF EXISTS */ + case 335: /* or_replace_opt ::= OR REPLACE */ yytestcase(yyruleno==335); + case 357: /* ignore_opt ::= IGNORE UNTREATED */ yytestcase(yyruleno==357); +{ yymsp[-1].minor.yy667 = true; } + break; + case 87: /* db_options ::= */ +{ yymsp[1].minor.yy452 = createDefaultDatabaseOptions(pCxt); } + break; + case 88: /* db_options ::= db_options BUFFER NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 89: /* db_options ::= db_options CACHEMODEL NK_STRING */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 90: /* db_options ::= db_options CACHESIZE NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 91: /* db_options ::= db_options COMP NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 92: /* db_options ::= db_options DURATION NK_INTEGER */ + case 93: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==93); +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 94: /* db_options ::= db_options MAXROWS NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 95: /* db_options ::= db_options MINROWS NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 96: /* db_options ::= db_options KEEP integer_list */ + case 97: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==97); +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_KEEP, yymsp[0].minor.yy812); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 98: /* db_options ::= db_options PAGES NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 99: /* db_options ::= db_options PAGESIZE NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 100: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 101: /* db_options ::= db_options PRECISION NK_STRING */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 102: /* db_options ::= db_options REPLICA NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 103: /* db_options ::= db_options VGROUPS NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 104: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 105: /* db_options ::= db_options RETENTIONS retention_list */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_RETENTIONS, yymsp[0].minor.yy812); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 106: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 107: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 108: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 109: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 110: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-3].minor.yy164, DB_OPTION_WAL_RETENTION_PERIOD, &t); + yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-3].minor.yy452, DB_OPTION_WAL_RETENTION_PERIOD, &t); } -#line 5046 "sql.c" - yymsp[-3].minor.yy164 = yylhsminor.yy164; + yymsp[-3].minor.yy452 = yylhsminor.yy452; break; - case 108: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ -#line 221 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } -#line 5052 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; + case 111: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 109: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ -#line 222 "sql.y" + case 112: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-3].minor.yy164, DB_OPTION_WAL_RETENTION_SIZE, &t); + yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-3].minor.yy452, DB_OPTION_WAL_RETENTION_SIZE, &t); } -#line 5062 "sql.c" - yymsp[-3].minor.yy164 = yylhsminor.yy164; - break; - case 110: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ -#line 227 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } -#line 5068 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 111: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ -#line 228 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } -#line 5074 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 112: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ -#line 229 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } -#line 5080 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 113: /* db_options ::= db_options TABLE_PREFIX signed */ -#line 230 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_TABLE_PREFIX, yymsp[0].minor.yy164); } -#line 5086 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 114: /* db_options ::= db_options TABLE_SUFFIX signed */ -#line 231 "sql.y" -{ yylhsminor.yy164 = setDatabaseOption(pCxt, yymsp[-2].minor.yy164, DB_OPTION_TABLE_SUFFIX, yymsp[0].minor.yy164); } -#line 5092 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 115: /* alter_db_options ::= alter_db_option */ -#line 233 "sql.y" -{ yylhsminor.yy164 = createAlterDatabaseOptions(pCxt); yylhsminor.yy164 = setAlterDatabaseOption(pCxt, yylhsminor.yy164, &yymsp[0].minor.yy761); } -#line 5098 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 116: /* alter_db_options ::= alter_db_options alter_db_option */ -#line 234 "sql.y" -{ yylhsminor.yy164 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy164, &yymsp[0].minor.yy761); } -#line 5104 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; - break; - case 117: /* alter_db_option ::= BUFFER NK_INTEGER */ -#line 238 "sql.y" -{ yymsp[-1].minor.yy761.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy761.val = yymsp[0].minor.yy0; } -#line 5110 "sql.c" - break; - case 118: /* alter_db_option ::= CACHEMODEL NK_STRING */ -#line 239 "sql.y" -{ yymsp[-1].minor.yy761.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy761.val = yymsp[0].minor.yy0; } -#line 5115 "sql.c" - break; - case 119: /* alter_db_option ::= CACHESIZE NK_INTEGER */ -#line 240 "sql.y" -{ yymsp[-1].minor.yy761.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy761.val = yymsp[0].minor.yy0; } -#line 5120 "sql.c" - break; - case 120: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ -#line 241 "sql.y" -{ yymsp[-1].minor.yy761.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy761.val = yymsp[0].minor.yy0; } -#line 5125 "sql.c" - break; - case 121: /* alter_db_option ::= KEEP integer_list */ - case 122: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==122); -#line 242 "sql.y" -{ yymsp[-1].minor.yy761.type = DB_OPTION_KEEP; yymsp[-1].minor.yy761.pList = yymsp[0].minor.yy72; } -#line 5131 "sql.c" - break; - case 123: /* alter_db_option ::= PAGES NK_INTEGER */ -#line 244 "sql.y" -{ yymsp[-1].minor.yy761.type = DB_OPTION_PAGES; yymsp[-1].minor.yy761.val = yymsp[0].minor.yy0; } -#line 5136 "sql.c" - break; - case 124: /* alter_db_option ::= REPLICA NK_INTEGER */ -#line 245 "sql.y" -{ yymsp[-1].minor.yy761.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy761.val = yymsp[0].minor.yy0; } -#line 5141 "sql.c" - break; - case 125: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ -#line 247 "sql.y" -{ yymsp[-1].minor.yy761.type = DB_OPTION_WAL; yymsp[-1].minor.yy761.val = yymsp[0].minor.yy0; } -#line 5146 "sql.c" - break; - case 126: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ -#line 248 "sql.y" -{ yymsp[-1].minor.yy761.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy761.val = yymsp[0].minor.yy0; } -#line 5151 "sql.c" - break; - case 127: /* alter_db_option ::= MINROWS NK_INTEGER */ -#line 249 "sql.y" -{ yymsp[-1].minor.yy761.type = DB_OPTION_MINROWS; yymsp[-1].minor.yy761.val = yymsp[0].minor.yy0; } -#line 5156 "sql.c" - break; - case 128: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ -#line 250 "sql.y" -{ yymsp[-1].minor.yy761.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-1].minor.yy761.val = yymsp[0].minor.yy0; } -#line 5161 "sql.c" - break; - case 129: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ -#line 251 "sql.y" + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 113: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 114: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 115: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 116: /* db_options ::= db_options TABLE_PREFIX signed */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_TABLE_PREFIX, yymsp[0].minor.yy452); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 117: /* db_options ::= db_options TABLE_SUFFIX signed */ +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_TABLE_SUFFIX, yymsp[0].minor.yy452); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 118: /* alter_db_options ::= alter_db_option */ +{ yylhsminor.yy452 = createAlterDatabaseOptions(pCxt); yylhsminor.yy452 = setAlterDatabaseOption(pCxt, yylhsminor.yy452, &yymsp[0].minor.yy365); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 119: /* alter_db_options ::= alter_db_options alter_db_option */ +{ yylhsminor.yy452 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy452, &yymsp[0].minor.yy365); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 120: /* alter_db_option ::= BUFFER NK_INTEGER */ +{ yymsp[-1].minor.yy365.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } + break; + case 121: /* alter_db_option ::= CACHEMODEL NK_STRING */ +{ yymsp[-1].minor.yy365.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } + break; + case 122: /* alter_db_option ::= CACHESIZE NK_INTEGER */ +{ yymsp[-1].minor.yy365.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } + break; + case 123: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ +{ yymsp[-1].minor.yy365.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } + break; + case 124: /* alter_db_option ::= KEEP integer_list */ + case 125: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==125); +{ yymsp[-1].minor.yy365.type = DB_OPTION_KEEP; yymsp[-1].minor.yy365.pList = yymsp[0].minor.yy812; } + break; + case 126: /* alter_db_option ::= PAGES NK_INTEGER */ +{ yymsp[-1].minor.yy365.type = DB_OPTION_PAGES; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } + break; + case 127: /* alter_db_option ::= REPLICA NK_INTEGER */ +{ yymsp[-1].minor.yy365.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } + break; + case 128: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ +{ yymsp[-1].minor.yy365.type = DB_OPTION_WAL; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } + break; + case 129: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ +{ yymsp[-1].minor.yy365.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } + break; + case 130: /* alter_db_option ::= MINROWS NK_INTEGER */ +{ yymsp[-1].minor.yy365.type = DB_OPTION_MINROWS; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } + break; + case 131: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ +{ yymsp[-1].minor.yy365.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } + break; + case 132: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yymsp[-2].minor.yy761.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-2].minor.yy761.val = t; + yymsp[-2].minor.yy365.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-2].minor.yy365.val = t; } -#line 5170 "sql.c" break; - case 130: /* alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ -#line 256 "sql.y" -{ yymsp[-1].minor.yy761.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-1].minor.yy761.val = yymsp[0].minor.yy0; } -#line 5175 "sql.c" + case 133: /* alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ +{ yymsp[-1].minor.yy365.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } break; - case 131: /* alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ -#line 257 "sql.y" + case 134: /* alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yymsp[-2].minor.yy761.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-2].minor.yy761.val = t; + yymsp[-2].minor.yy365.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-2].minor.yy365.val = t; } -#line 5184 "sql.c" - break; - case 132: /* integer_list ::= NK_INTEGER */ -#line 265 "sql.y" -{ yylhsminor.yy72 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } -#line 5189 "sql.c" - yymsp[0].minor.yy72 = yylhsminor.yy72; - break; - case 133: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ - case 359: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==359); -#line 266 "sql.y" -{ yylhsminor.yy72 = addNodeToList(pCxt, yymsp[-2].minor.yy72, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } -#line 5196 "sql.c" - yymsp[-2].minor.yy72 = yylhsminor.yy72; - break; - case 134: /* variable_list ::= NK_VARIABLE */ -#line 270 "sql.y" -{ yylhsminor.yy72 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } -#line 5202 "sql.c" - yymsp[0].minor.yy72 = yylhsminor.yy72; - break; - case 135: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ -#line 271 "sql.y" -{ yylhsminor.yy72 = addNodeToList(pCxt, yymsp[-2].minor.yy72, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } -#line 5208 "sql.c" - yymsp[-2].minor.yy72 = yylhsminor.yy72; - break; - case 136: /* retention_list ::= retention */ - case 166: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==166); - case 169: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==169); - case 176: /* column_def_list ::= column_def */ yytestcase(yyruleno==176); - case 219: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==219); - case 224: /* col_name_list ::= col_name */ yytestcase(yyruleno==224); - case 275: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==275); - case 289: /* func_list ::= func */ yytestcase(yyruleno==289); - case 388: /* literal_list ::= signed_literal */ yytestcase(yyruleno==388); - case 455: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==455); - case 461: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==461); - case 516: /* select_list ::= select_item */ yytestcase(yyruleno==516); - case 527: /* partition_list ::= partition_item */ yytestcase(yyruleno==527); - case 582: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==582); -#line 275 "sql.y" -{ yylhsminor.yy72 = createNodeList(pCxt, yymsp[0].minor.yy164); } -#line 5227 "sql.c" - yymsp[0].minor.yy72 = yylhsminor.yy72; - break; - case 137: /* retention_list ::= retention_list NK_COMMA retention */ - case 170: /* multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ yytestcase(yyruleno==170); - case 177: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==177); - case 220: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==220); - case 225: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==225); - case 276: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==276); - case 290: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==290); - case 389: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==389); - case 456: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==456); - case 517: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==517); - case 528: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==528); - case 583: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==583); -#line 276 "sql.y" -{ yylhsminor.yy72 = addNodeToList(pCxt, yymsp[-2].minor.yy72, yymsp[0].minor.yy164); } -#line 5244 "sql.c" - yymsp[-2].minor.yy72 = yylhsminor.yy72; - break; - case 138: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -#line 278 "sql.y" -{ yylhsminor.yy164 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } -#line 5250 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 139: /* speed_opt ::= */ - case 322: /* bufsize_opt ::= */ yytestcase(yyruleno==322); -#line 282 "sql.y" -{ yymsp[1].minor.yy560 = 0; } -#line 5257 "sql.c" - break; - case 140: /* speed_opt ::= MAX_SPEED NK_INTEGER */ - case 323: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==323); -#line 283 "sql.y" -{ yymsp[-1].minor.yy560 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } -#line 5263 "sql.c" - break; - case 142: /* start_opt ::= START WITH NK_INTEGER */ - case 146: /* end_opt ::= END WITH NK_INTEGER */ yytestcase(yyruleno==146); -#line 286 "sql.y" -{ yymsp[-2].minor.yy164 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } -#line 5269 "sql.c" - break; - case 143: /* start_opt ::= START WITH NK_STRING */ - case 147: /* end_opt ::= END WITH NK_STRING */ yytestcase(yyruleno==147); -#line 287 "sql.y" -{ yymsp[-2].minor.yy164 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } -#line 5275 "sql.c" - break; - case 144: /* start_opt ::= START WITH TIMESTAMP NK_STRING */ - case 148: /* end_opt ::= END WITH TIMESTAMP NK_STRING */ yytestcase(yyruleno==148); -#line 288 "sql.y" -{ yymsp[-3].minor.yy164 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } -#line 5281 "sql.c" - break; - case 149: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - case 151: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==151); -#line 297 "sql.y" -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy441, yymsp[-5].minor.yy164, yymsp[-3].minor.yy72, yymsp[-1].minor.yy72, yymsp[0].minor.yy164); } -#line 5287 "sql.c" - break; - case 150: /* cmd ::= CREATE TABLE multi_create_clause */ -#line 298 "sql.y" -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy72); } -#line 5292 "sql.c" - break; - case 152: /* cmd ::= DROP TABLE multi_drop_clause */ -#line 301 "sql.y" -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy72); } -#line 5297 "sql.c" - break; - case 153: /* cmd ::= DROP STABLE exists_opt full_table_name */ -#line 302 "sql.y" -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy441, yymsp[0].minor.yy164); } -#line 5302 "sql.c" - break; - case 154: /* cmd ::= ALTER TABLE alter_table_clause */ - case 361: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==361); - case 362: /* cmd ::= insert_query */ yytestcase(yyruleno==362); -#line 304 "sql.y" -{ pCxt->pRootNode = yymsp[0].minor.yy164; } -#line 5309 "sql.c" - break; - case 155: /* cmd ::= ALTER STABLE alter_table_clause */ -#line 305 "sql.y" -{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy164); } -#line 5314 "sql.c" - break; - case 156: /* alter_table_clause ::= full_table_name alter_table_options */ -#line 307 "sql.y" -{ yylhsminor.yy164 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy164, yymsp[0].minor.yy164); } -#line 5319 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; - break; - case 157: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -#line 309 "sql.y" -{ yylhsminor.yy164 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy164, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy497, yymsp[0].minor.yy700); } -#line 5325 "sql.c" - yymsp[-4].minor.yy164 = yylhsminor.yy164; - break; - case 158: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -#line 310 "sql.y" -{ yylhsminor.yy164 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy164, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy497); } -#line 5331 "sql.c" - yymsp[-3].minor.yy164 = yylhsminor.yy164; - break; - case 159: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -#line 312 "sql.y" -{ yylhsminor.yy164 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy164, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy497, yymsp[0].minor.yy700); } -#line 5337 "sql.c" - yymsp[-4].minor.yy164 = yylhsminor.yy164; - break; - case 160: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -#line 314 "sql.y" -{ yylhsminor.yy164 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy164, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy497, &yymsp[0].minor.yy497); } -#line 5343 "sql.c" - yymsp[-4].minor.yy164 = yylhsminor.yy164; - break; - case 161: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -#line 316 "sql.y" -{ yylhsminor.yy164 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy164, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy497, yymsp[0].minor.yy700); } -#line 5349 "sql.c" - yymsp[-4].minor.yy164 = yylhsminor.yy164; - break; - case 162: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -#line 317 "sql.y" -{ yylhsminor.yy164 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy164, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy497); } -#line 5355 "sql.c" - yymsp[-3].minor.yy164 = yylhsminor.yy164; - break; - case 163: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -#line 319 "sql.y" -{ yylhsminor.yy164 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy164, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy497, yymsp[0].minor.yy700); } -#line 5361 "sql.c" - yymsp[-4].minor.yy164 = yylhsminor.yy164; - break; - case 164: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -#line 321 "sql.y" -{ yylhsminor.yy164 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy164, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy497, &yymsp[0].minor.yy497); } -#line 5367 "sql.c" - yymsp[-4].minor.yy164 = yylhsminor.yy164; - break; - case 165: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ -#line 323 "sql.y" -{ yylhsminor.yy164 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy164, &yymsp[-2].minor.yy497, yymsp[0].minor.yy164); } -#line 5373 "sql.c" - yymsp[-5].minor.yy164 = yylhsminor.yy164; - break; - case 167: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ - case 462: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==462); -#line 328 "sql.y" -{ yylhsminor.yy72 = addNodeToList(pCxt, yymsp[-1].minor.yy72, yymsp[0].minor.yy164); } -#line 5380 "sql.c" - yymsp[-1].minor.yy72 = yylhsminor.yy72; - break; - case 168: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ -#line 332 "sql.y" -{ yylhsminor.yy164 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy441, yymsp[-8].minor.yy164, yymsp[-6].minor.yy164, yymsp[-5].minor.yy72, yymsp[-2].minor.yy72, yymsp[0].minor.yy164); } -#line 5386 "sql.c" - yymsp[-9].minor.yy164 = yylhsminor.yy164; - break; - case 171: /* drop_table_clause ::= exists_opt full_table_name */ -#line 339 "sql.y" -{ yylhsminor.yy164 = createDropTableClause(pCxt, yymsp[-1].minor.yy441, yymsp[0].minor.yy164); } -#line 5392 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; - break; - case 172: /* specific_cols_opt ::= */ - case 202: /* tags_def_opt ::= */ yytestcase(yyruleno==202); - case 274: /* tag_list_opt ::= */ yytestcase(yyruleno==274); - case 332: /* col_list_opt ::= */ yytestcase(yyruleno==332); - case 334: /* tag_def_or_ref_opt ::= */ yytestcase(yyruleno==334); - case 525: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==525); - case 550: /* group_by_clause_opt ::= */ yytestcase(yyruleno==550); - case 569: /* order_by_clause_opt ::= */ yytestcase(yyruleno==569); -#line 343 "sql.y" -{ yymsp[1].minor.yy72 = NULL; } -#line 5405 "sql.c" - break; - case 173: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ - case 333: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==333); -#line 344 "sql.y" -{ yymsp[-2].minor.yy72 = yymsp[-1].minor.yy72; } -#line 5411 "sql.c" - break; - case 174: /* full_table_name ::= table_name */ -#line 346 "sql.y" -{ yylhsminor.yy164 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy497, NULL); } -#line 5416 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 175: /* full_table_name ::= db_name NK_DOT table_name */ -#line 347 "sql.y" -{ yylhsminor.yy164 = createRealTableNode(pCxt, &yymsp[-2].minor.yy497, &yymsp[0].minor.yy497, NULL); } -#line 5422 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 178: /* column_def ::= column_name type_name */ -#line 354 "sql.y" -{ yylhsminor.yy164 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy497, yymsp[0].minor.yy700, NULL); } -#line 5428 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; - break; - case 179: /* type_name ::= BOOL */ -#line 359 "sql.y" -{ yymsp[0].minor.yy700 = createDataType(TSDB_DATA_TYPE_BOOL); } -#line 5434 "sql.c" - break; - case 180: /* type_name ::= TINYINT */ -#line 360 "sql.y" -{ yymsp[0].minor.yy700 = createDataType(TSDB_DATA_TYPE_TINYINT); } -#line 5439 "sql.c" - break; - case 181: /* type_name ::= SMALLINT */ -#line 361 "sql.y" -{ yymsp[0].minor.yy700 = createDataType(TSDB_DATA_TYPE_SMALLINT); } -#line 5444 "sql.c" - break; - case 182: /* type_name ::= INT */ - case 183: /* type_name ::= INTEGER */ yytestcase(yyruleno==183); -#line 362 "sql.y" -{ yymsp[0].minor.yy700 = createDataType(TSDB_DATA_TYPE_INT); } -#line 5450 "sql.c" - break; - case 184: /* type_name ::= BIGINT */ -#line 364 "sql.y" -{ yymsp[0].minor.yy700 = createDataType(TSDB_DATA_TYPE_BIGINT); } -#line 5455 "sql.c" - break; - case 185: /* type_name ::= FLOAT */ -#line 365 "sql.y" -{ yymsp[0].minor.yy700 = createDataType(TSDB_DATA_TYPE_FLOAT); } -#line 5460 "sql.c" - break; - case 186: /* type_name ::= DOUBLE */ -#line 366 "sql.y" -{ yymsp[0].minor.yy700 = createDataType(TSDB_DATA_TYPE_DOUBLE); } -#line 5465 "sql.c" - break; - case 187: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -#line 367 "sql.y" -{ yymsp[-3].minor.yy700 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } -#line 5470 "sql.c" - break; - case 188: /* type_name ::= TIMESTAMP */ -#line 368 "sql.y" -{ yymsp[0].minor.yy700 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } -#line 5475 "sql.c" - break; - case 189: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -#line 369 "sql.y" -{ yymsp[-3].minor.yy700 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } -#line 5480 "sql.c" - break; - case 190: /* type_name ::= TINYINT UNSIGNED */ -#line 370 "sql.y" -{ yymsp[-1].minor.yy700 = createDataType(TSDB_DATA_TYPE_UTINYINT); } -#line 5485 "sql.c" - break; - case 191: /* type_name ::= SMALLINT UNSIGNED */ -#line 371 "sql.y" -{ yymsp[-1].minor.yy700 = createDataType(TSDB_DATA_TYPE_USMALLINT); } -#line 5490 "sql.c" - break; - case 192: /* type_name ::= INT UNSIGNED */ -#line 372 "sql.y" -{ yymsp[-1].minor.yy700 = createDataType(TSDB_DATA_TYPE_UINT); } -#line 5495 "sql.c" - break; - case 193: /* type_name ::= BIGINT UNSIGNED */ -#line 373 "sql.y" -{ yymsp[-1].minor.yy700 = createDataType(TSDB_DATA_TYPE_UBIGINT); } -#line 5500 "sql.c" - break; - case 194: /* type_name ::= JSON */ -#line 374 "sql.y" -{ yymsp[0].minor.yy700 = createDataType(TSDB_DATA_TYPE_JSON); } -#line 5505 "sql.c" - break; - case 195: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -#line 375 "sql.y" -{ yymsp[-3].minor.yy700 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } -#line 5510 "sql.c" - break; - case 196: /* type_name ::= MEDIUMBLOB */ -#line 376 "sql.y" -{ yymsp[0].minor.yy700 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } -#line 5515 "sql.c" - break; - case 197: /* type_name ::= BLOB */ -#line 377 "sql.y" -{ yymsp[0].minor.yy700 = createDataType(TSDB_DATA_TYPE_BLOB); } -#line 5520 "sql.c" - break; - case 198: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -#line 378 "sql.y" -{ yymsp[-3].minor.yy700 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } -#line 5525 "sql.c" - break; - case 199: /* type_name ::= DECIMAL */ -#line 379 "sql.y" -{ yymsp[0].minor.yy700 = createDataType(TSDB_DATA_TYPE_DECIMAL); } -#line 5530 "sql.c" - break; - case 200: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -#line 380 "sql.y" -{ yymsp[-3].minor.yy700 = createDataType(TSDB_DATA_TYPE_DECIMAL); } -#line 5535 "sql.c" - break; - case 201: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -#line 381 "sql.y" -{ yymsp[-5].minor.yy700 = createDataType(TSDB_DATA_TYPE_DECIMAL); } -#line 5540 "sql.c" - break; - case 203: /* tags_def_opt ::= tags_def */ - case 335: /* tag_def_or_ref_opt ::= tags_def */ yytestcase(yyruleno==335); - case 454: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==454); -#line 386 "sql.y" -{ yylhsminor.yy72 = yymsp[0].minor.yy72; } -#line 5547 "sql.c" - yymsp[0].minor.yy72 = yylhsminor.yy72; - break; - case 204: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ - case 336: /* tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ yytestcase(yyruleno==336); -#line 390 "sql.y" -{ yymsp[-3].minor.yy72 = yymsp[-1].minor.yy72; } -#line 5554 "sql.c" - break; - case 205: /* table_options ::= */ -#line 392 "sql.y" -{ yymsp[1].minor.yy164 = createDefaultTableOptions(pCxt); } -#line 5559 "sql.c" - break; - case 206: /* table_options ::= table_options COMMENT NK_STRING */ -#line 393 "sql.y" -{ yylhsminor.yy164 = setTableOption(pCxt, yymsp[-2].minor.yy164, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } -#line 5564 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 207: /* table_options ::= table_options MAX_DELAY duration_list */ -#line 394 "sql.y" -{ yylhsminor.yy164 = setTableOption(pCxt, yymsp[-2].minor.yy164, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy72); } -#line 5570 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 208: /* table_options ::= table_options WATERMARK duration_list */ -#line 395 "sql.y" -{ yylhsminor.yy164 = setTableOption(pCxt, yymsp[-2].minor.yy164, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy72); } -#line 5576 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 209: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ -#line 396 "sql.y" -{ yylhsminor.yy164 = setTableOption(pCxt, yymsp[-4].minor.yy164, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy72); } -#line 5582 "sql.c" - yymsp[-4].minor.yy164 = yylhsminor.yy164; - break; - case 210: /* table_options ::= table_options TTL NK_INTEGER */ -#line 397 "sql.y" -{ yylhsminor.yy164 = setTableOption(pCxt, yymsp[-2].minor.yy164, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } -#line 5588 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 211: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -#line 398 "sql.y" -{ yylhsminor.yy164 = setTableOption(pCxt, yymsp[-4].minor.yy164, TABLE_OPTION_SMA, yymsp[-1].minor.yy72); } -#line 5594 "sql.c" - yymsp[-4].minor.yy164 = yylhsminor.yy164; - break; - case 212: /* table_options ::= table_options DELETE_MARK duration_list */ -#line 399 "sql.y" -{ yylhsminor.yy164 = setTableOption(pCxt, yymsp[-2].minor.yy164, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy72); } -#line 5600 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 213: /* alter_table_options ::= alter_table_option */ -#line 401 "sql.y" -{ yylhsminor.yy164 = createAlterTableOptions(pCxt); yylhsminor.yy164 = setTableOption(pCxt, yylhsminor.yy164, yymsp[0].minor.yy761.type, &yymsp[0].minor.yy761.val); } -#line 5606 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 214: /* alter_table_options ::= alter_table_options alter_table_option */ -#line 402 "sql.y" -{ yylhsminor.yy164 = setTableOption(pCxt, yymsp[-1].minor.yy164, yymsp[0].minor.yy761.type, &yymsp[0].minor.yy761.val); } -#line 5612 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; - break; - case 215: /* alter_table_option ::= COMMENT NK_STRING */ -#line 406 "sql.y" -{ yymsp[-1].minor.yy761.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy761.val = yymsp[0].minor.yy0; } -#line 5618 "sql.c" - break; - case 216: /* alter_table_option ::= TTL NK_INTEGER */ -#line 407 "sql.y" -{ yymsp[-1].minor.yy761.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy761.val = yymsp[0].minor.yy0; } -#line 5623 "sql.c" - break; - case 217: /* duration_list ::= duration_literal */ - case 418: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==418); -#line 411 "sql.y" -{ yylhsminor.yy72 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy164)); } -#line 5629 "sql.c" - yymsp[0].minor.yy72 = yylhsminor.yy72; - break; - case 218: /* duration_list ::= duration_list NK_COMMA duration_literal */ - case 419: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==419); -#line 412 "sql.y" -{ yylhsminor.yy72 = addNodeToList(pCxt, yymsp[-2].minor.yy72, releaseRawExprNode(pCxt, yymsp[0].minor.yy164)); } -#line 5636 "sql.c" - yymsp[-2].minor.yy72 = yylhsminor.yy72; - break; - case 221: /* rollup_func_name ::= function_name */ -#line 419 "sql.y" -{ yylhsminor.yy164 = createFunctionNode(pCxt, &yymsp[0].minor.yy497, NULL); } -#line 5642 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 222: /* rollup_func_name ::= FIRST */ - case 223: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==223); - case 278: /* tag_item ::= QTAGS */ yytestcase(yyruleno==278); -#line 420 "sql.y" -{ yylhsminor.yy164 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } -#line 5650 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 226: /* col_name ::= column_name */ - case 279: /* tag_item ::= column_name */ yytestcase(yyruleno==279); -#line 428 "sql.y" -{ yylhsminor.yy164 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy497); } -#line 5657 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 227: /* cmd ::= SHOW DNODES */ -#line 431 "sql.y" + break; + case 135: /* integer_list ::= NK_INTEGER */ +{ yylhsminor.yy812 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy812 = yylhsminor.yy812; + break; + case 136: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ + case 367: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==367); +{ yylhsminor.yy812 = addNodeToList(pCxt, yymsp[-2].minor.yy812, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy812 = yylhsminor.yy812; + break; + case 137: /* variable_list ::= NK_VARIABLE */ +{ yylhsminor.yy812 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy812 = yylhsminor.yy812; + break; + case 138: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ +{ yylhsminor.yy812 = addNodeToList(pCxt, yymsp[-2].minor.yy812, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy812 = yylhsminor.yy812; + break; + case 139: /* retention_list ::= retention */ + case 169: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==169); + case 172: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==172); + case 179: /* column_def_list ::= column_def */ yytestcase(yyruleno==179); + case 223: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==223); + case 228: /* col_name_list ::= col_name */ yytestcase(yyruleno==228); + case 282: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==282); + case 296: /* func_list ::= func */ yytestcase(yyruleno==296); + case 396: /* literal_list ::= signed_literal */ yytestcase(yyruleno==396); + case 463: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==463); + case 469: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==469); + case 524: /* select_list ::= select_item */ yytestcase(yyruleno==524); + case 535: /* partition_list ::= partition_item */ yytestcase(yyruleno==535); + case 591: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==591); +{ yylhsminor.yy812 = createNodeList(pCxt, yymsp[0].minor.yy452); } + yymsp[0].minor.yy812 = yylhsminor.yy812; + break; + case 140: /* retention_list ::= retention_list NK_COMMA retention */ + case 173: /* multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ yytestcase(yyruleno==173); + case 180: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==180); + case 224: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==224); + case 229: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==229); + case 283: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==283); + case 297: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==297); + case 397: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==397); + case 464: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==464); + case 525: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==525); + case 536: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==536); + case 592: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==592); +{ yylhsminor.yy812 = addNodeToList(pCxt, yymsp[-2].minor.yy812, yymsp[0].minor.yy452); } + yymsp[-2].minor.yy812 = yylhsminor.yy812; + break; + case 141: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ +{ yylhsminor.yy452 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 142: /* speed_opt ::= */ + case 330: /* bufsize_opt ::= */ yytestcase(yyruleno==330); +{ yymsp[1].minor.yy416 = 0; } + break; + case 143: /* speed_opt ::= MAX_SPEED NK_INTEGER */ + case 331: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==331); +{ yymsp[-1].minor.yy416 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } + break; + case 145: /* start_opt ::= START WITH NK_INTEGER */ + case 149: /* end_opt ::= END WITH NK_INTEGER */ yytestcase(yyruleno==149); +{ yymsp[-2].minor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + break; + case 146: /* start_opt ::= START WITH NK_STRING */ + case 150: /* end_opt ::= END WITH NK_STRING */ yytestcase(yyruleno==150); +{ yymsp[-2].minor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 147: /* start_opt ::= START WITH TIMESTAMP NK_STRING */ + case 151: /* end_opt ::= END WITH TIMESTAMP NK_STRING */ yytestcase(yyruleno==151); +{ yymsp[-3].minor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 152: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + case 154: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==154); +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy667, yymsp[-5].minor.yy452, yymsp[-3].minor.yy812, yymsp[-1].minor.yy812, yymsp[0].minor.yy452); } + break; + case 153: /* cmd ::= CREATE TABLE multi_create_clause */ +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy812); } + break; + case 155: /* cmd ::= DROP TABLE multi_drop_clause */ +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy812); } + break; + case 156: /* cmd ::= DROP STABLE exists_opt full_table_name */ +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy667, yymsp[0].minor.yy452); } + break; + case 157: /* cmd ::= ALTER TABLE alter_table_clause */ + case 369: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==369); + case 370: /* cmd ::= insert_query */ yytestcase(yyruleno==370); +{ pCxt->pRootNode = yymsp[0].minor.yy452; } + break; + case 158: /* cmd ::= ALTER STABLE alter_table_clause */ +{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy452); } + break; + case 159: /* alter_table_clause ::= full_table_name alter_table_options */ +{ yylhsminor.yy452 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 160: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ +{ yylhsminor.yy452 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy452, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy371, yymsp[0].minor.yy310); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; + break; + case 161: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ +{ yylhsminor.yy452 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy452, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy371); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 162: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ +{ yylhsminor.yy452 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy452, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy371, yymsp[0].minor.yy310); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; + break; + case 163: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ +{ yylhsminor.yy452 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy452, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy371, &yymsp[0].minor.yy371); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; + break; + case 164: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ +{ yylhsminor.yy452 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy452, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy371, yymsp[0].minor.yy310); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; + break; + case 165: /* alter_table_clause ::= full_table_name DROP TAG column_name */ +{ yylhsminor.yy452 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy452, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy371); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 166: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ +{ yylhsminor.yy452 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy452, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy371, yymsp[0].minor.yy310); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; + break; + case 167: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ +{ yylhsminor.yy452 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy452, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy371, &yymsp[0].minor.yy371); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; + break; + case 168: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ +{ yylhsminor.yy452 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy452, &yymsp[-2].minor.yy371, yymsp[0].minor.yy452); } + yymsp[-5].minor.yy452 = yylhsminor.yy452; + break; + case 170: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ + case 470: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==470); +{ yylhsminor.yy812 = addNodeToList(pCxt, yymsp[-1].minor.yy812, yymsp[0].minor.yy452); } + yymsp[-1].minor.yy812 = yylhsminor.yy812; + break; + case 171: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ +{ yylhsminor.yy452 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy667, yymsp[-8].minor.yy452, yymsp[-6].minor.yy452, yymsp[-5].minor.yy812, yymsp[-2].minor.yy812, yymsp[0].minor.yy452); } + yymsp[-9].minor.yy452 = yylhsminor.yy452; + break; + case 174: /* drop_table_clause ::= exists_opt full_table_name */ +{ yylhsminor.yy452 = createDropTableClause(pCxt, yymsp[-1].minor.yy667, yymsp[0].minor.yy452); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 175: /* specific_cols_opt ::= */ + case 206: /* tags_def_opt ::= */ yytestcase(yyruleno==206); + case 281: /* tag_list_opt ::= */ yytestcase(yyruleno==281); + case 340: /* col_list_opt ::= */ yytestcase(yyruleno==340); + case 342: /* tag_def_or_ref_opt ::= */ yytestcase(yyruleno==342); + case 533: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==533); + case 558: /* group_by_clause_opt ::= */ yytestcase(yyruleno==558); + case 578: /* order_by_clause_opt ::= */ yytestcase(yyruleno==578); +{ yymsp[1].minor.yy812 = NULL; } + break; + case 176: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ + case 341: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==341); +{ yymsp[-2].minor.yy812 = yymsp[-1].minor.yy812; } + break; + case 177: /* full_table_name ::= table_name */ +{ yylhsminor.yy452 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy371, NULL); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 178: /* full_table_name ::= db_name NK_DOT table_name */ +{ yylhsminor.yy452 = createRealTableNode(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy371, NULL); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 181: /* column_def ::= column_name type_name */ +{ yylhsminor.yy452 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy371, yymsp[0].minor.yy310, NULL); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 182: /* type_name ::= BOOL */ +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_BOOL); } + break; + case 183: /* type_name ::= TINYINT */ +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_TINYINT); } + break; + case 184: /* type_name ::= SMALLINT */ +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_SMALLINT); } + break; + case 185: /* type_name ::= INT */ + case 186: /* type_name ::= INTEGER */ yytestcase(yyruleno==186); +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_INT); } + break; + case 187: /* type_name ::= BIGINT */ +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_BIGINT); } + break; + case 188: /* type_name ::= FLOAT */ +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_FLOAT); } + break; + case 189: /* type_name ::= DOUBLE */ +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_DOUBLE); } + break; + case 190: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy310 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } + break; + case 191: /* type_name ::= TIMESTAMP */ +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } + break; + case 192: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy310 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } + break; + case 193: /* type_name ::= TINYINT UNSIGNED */ +{ yymsp[-1].minor.yy310 = createDataType(TSDB_DATA_TYPE_UTINYINT); } + break; + case 194: /* type_name ::= SMALLINT UNSIGNED */ +{ yymsp[-1].minor.yy310 = createDataType(TSDB_DATA_TYPE_USMALLINT); } + break; + case 195: /* type_name ::= INT UNSIGNED */ +{ yymsp[-1].minor.yy310 = createDataType(TSDB_DATA_TYPE_UINT); } + break; + case 196: /* type_name ::= BIGINT UNSIGNED */ +{ yymsp[-1].minor.yy310 = createDataType(TSDB_DATA_TYPE_UBIGINT); } + break; + case 197: /* type_name ::= JSON */ +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_JSON); } + break; + case 198: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy310 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } + break; + case 199: /* type_name ::= MEDIUMBLOB */ +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } + break; + case 200: /* type_name ::= BLOB */ +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_BLOB); } + break; + case 201: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy310 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } + break; + case 202: /* type_name ::= GEOMETRY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy310 = createVarLenDataType(TSDB_DATA_TYPE_GEOMETRY, &yymsp[-1].minor.yy0); } + break; + case 203: /* type_name ::= DECIMAL */ +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + break; + case 204: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy310 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + break; + case 205: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ +{ yymsp[-5].minor.yy310 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + break; + case 207: /* tags_def_opt ::= tags_def */ + case 343: /* tag_def_or_ref_opt ::= tags_def */ yytestcase(yyruleno==343); + case 462: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==462); +{ yylhsminor.yy812 = yymsp[0].minor.yy812; } + yymsp[0].minor.yy812 = yylhsminor.yy812; + break; + case 208: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ + case 344: /* tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ yytestcase(yyruleno==344); +{ yymsp[-3].minor.yy812 = yymsp[-1].minor.yy812; } + break; + case 209: /* table_options ::= */ +{ yymsp[1].minor.yy452 = createDefaultTableOptions(pCxt); } + break; + case 210: /* table_options ::= table_options COMMENT NK_STRING */ +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-2].minor.yy452, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 211: /* table_options ::= table_options MAX_DELAY duration_list */ +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-2].minor.yy452, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy812); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 212: /* table_options ::= table_options WATERMARK duration_list */ +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-2].minor.yy452, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy812); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 213: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-4].minor.yy452, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy812); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; + break; + case 214: /* table_options ::= table_options TTL NK_INTEGER */ +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-2].minor.yy452, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 215: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-4].minor.yy452, TABLE_OPTION_SMA, yymsp[-1].minor.yy812); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; + break; + case 216: /* table_options ::= table_options DELETE_MARK duration_list */ +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-2].minor.yy452, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy812); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 217: /* alter_table_options ::= alter_table_option */ +{ yylhsminor.yy452 = createAlterTableOptions(pCxt); yylhsminor.yy452 = setTableOption(pCxt, yylhsminor.yy452, yymsp[0].minor.yy365.type, &yymsp[0].minor.yy365.val); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 218: /* alter_table_options ::= alter_table_options alter_table_option */ +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-1].minor.yy452, yymsp[0].minor.yy365.type, &yymsp[0].minor.yy365.val); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 219: /* alter_table_option ::= COMMENT NK_STRING */ +{ yymsp[-1].minor.yy365.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } + break; + case 220: /* alter_table_option ::= TTL NK_INTEGER */ +{ yymsp[-1].minor.yy365.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } + break; + case 221: /* duration_list ::= duration_literal */ + case 426: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==426); +{ yylhsminor.yy812 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy452)); } + yymsp[0].minor.yy812 = yylhsminor.yy812; + break; + case 222: /* duration_list ::= duration_list NK_COMMA duration_literal */ + case 427: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==427); +{ yylhsminor.yy812 = addNodeToList(pCxt, yymsp[-2].minor.yy812, releaseRawExprNode(pCxt, yymsp[0].minor.yy452)); } + yymsp[-2].minor.yy812 = yylhsminor.yy812; + break; + case 225: /* rollup_func_name ::= function_name */ +{ yylhsminor.yy452 = createFunctionNode(pCxt, &yymsp[0].minor.yy371, NULL); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 226: /* rollup_func_name ::= FIRST */ + case 227: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==227); + case 285: /* tag_item ::= QTAGS */ yytestcase(yyruleno==285); +{ yylhsminor.yy452 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 230: /* col_name ::= column_name */ + case 286: /* tag_item ::= column_name */ yytestcase(yyruleno==286); +{ yylhsminor.yy452 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy371); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 231: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } -#line 5663 "sql.c" break; - case 228: /* cmd ::= SHOW USERS */ -#line 432 "sql.y" + case 232: /* cmd ::= SHOW USERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT); } -#line 5668 "sql.c" break; - case 229: /* cmd ::= SHOW USER PRIVILEGES */ -#line 433 "sql.y" + case 233: /* cmd ::= SHOW USER PRIVILEGES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT); } -#line 5673 "sql.c" break; - case 230: /* cmd ::= SHOW DATABASES */ -#line 434 "sql.y" + case 234: /* cmd ::= SHOW DATABASES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); } -#line 5678 "sql.c" break; - case 231: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -#line 435 "sql.y" -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy164, yymsp[0].minor.yy164, OP_TYPE_LIKE); } -#line 5683 "sql.c" + case 235: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy452, yymsp[0].minor.yy452, OP_TYPE_LIKE); } break; - case 232: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -#line 436 "sql.y" -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy164, yymsp[0].minor.yy164, OP_TYPE_LIKE); } -#line 5688 "sql.c" + case 236: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy452, yymsp[0].minor.yy452, OP_TYPE_LIKE); } break; - case 233: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -#line 437 "sql.y" -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy164, NULL, OP_TYPE_LIKE); } -#line 5693 "sql.c" + case 237: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy452, NULL, OP_TYPE_LIKE); } break; - case 234: /* cmd ::= SHOW MNODES */ -#line 438 "sql.y" + case 238: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); } -#line 5698 "sql.c" break; - case 235: /* cmd ::= SHOW QNODES */ -#line 440 "sql.y" + case 239: /* cmd ::= SHOW QNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT); } -#line 5703 "sql.c" break; - case 236: /* cmd ::= SHOW FUNCTIONS */ -#line 441 "sql.y" + case 240: /* cmd ::= SHOW FUNCTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } -#line 5708 "sql.c" break; - case 237: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -#line 442 "sql.y" -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy164, yymsp[-1].minor.yy164, OP_TYPE_EQUAL); } -#line 5713 "sql.c" + case 241: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy452, yymsp[-1].minor.yy452, OP_TYPE_EQUAL); } + break; + case 242: /* cmd ::= SHOW INDEXES FROM db_name NK_DOT table_name */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy371), createIdentifierValueNode(pCxt, &yymsp[0].minor.yy371), OP_TYPE_EQUAL); } break; - case 238: /* cmd ::= SHOW STREAMS */ -#line 443 "sql.y" + case 243: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } -#line 5718 "sql.c" break; - case 239: /* cmd ::= SHOW ACCOUNTS */ -#line 444 "sql.y" + case 244: /* cmd ::= SHOW ACCOUNTS */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } -#line 5723 "sql.c" break; - case 240: /* cmd ::= SHOW APPS */ -#line 445 "sql.y" + case 245: /* cmd ::= SHOW APPS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT); } -#line 5728 "sql.c" break; - case 241: /* cmd ::= SHOW CONNECTIONS */ -#line 446 "sql.y" + case 246: /* cmd ::= SHOW CONNECTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT); } -#line 5733 "sql.c" break; - case 242: /* cmd ::= SHOW LICENCES */ - case 243: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==243); -#line 447 "sql.y" + case 247: /* cmd ::= SHOW LICENCES */ + case 248: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==248); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); } -#line 5739 "sql.c" break; - case 244: /* cmd ::= SHOW CREATE DATABASE db_name */ -#line 449 "sql.y" -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy497); } -#line 5744 "sql.c" + case 249: /* cmd ::= SHOW CREATE DATABASE db_name */ +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy371); } break; - case 245: /* cmd ::= SHOW CREATE TABLE full_table_name */ -#line 450 "sql.y" -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy164); } -#line 5749 "sql.c" + case 250: /* cmd ::= SHOW CREATE TABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy452); } break; - case 246: /* cmd ::= SHOW CREATE STABLE full_table_name */ -#line 451 "sql.y" -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy164); } -#line 5754 "sql.c" + case 251: /* cmd ::= SHOW CREATE STABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy452); } break; - case 247: /* cmd ::= SHOW QUERIES */ -#line 452 "sql.y" + case 252: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } -#line 5759 "sql.c" break; - case 248: /* cmd ::= SHOW SCORES */ -#line 453 "sql.y" + case 253: /* cmd ::= SHOW SCORES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); } -#line 5764 "sql.c" break; - case 249: /* cmd ::= SHOW TOPICS */ -#line 454 "sql.y" + case 254: /* cmd ::= SHOW TOPICS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); } -#line 5769 "sql.c" break; - case 250: /* cmd ::= SHOW VARIABLES */ - case 251: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==251); -#line 455 "sql.y" + case 255: /* cmd ::= SHOW VARIABLES */ + case 256: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==256); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLES_STMT); } -#line 5775 "sql.c" break; - case 252: /* cmd ::= SHOW LOCAL VARIABLES */ -#line 457 "sql.y" + case 257: /* cmd ::= SHOW LOCAL VARIABLES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } -#line 5780 "sql.c" break; - case 253: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ -#line 458 "sql.y" -{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy164); } -#line 5785 "sql.c" + case 258: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy452); } break; - case 254: /* cmd ::= SHOW BNODES */ -#line 459 "sql.y" + case 259: /* cmd ::= SHOW BNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } -#line 5790 "sql.c" break; - case 255: /* cmd ::= SHOW SNODES */ -#line 460 "sql.y" + case 260: /* cmd ::= SHOW SNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } -#line 5795 "sql.c" break; - case 256: /* cmd ::= SHOW CLUSTER */ -#line 461 "sql.y" + case 261: /* cmd ::= SHOW CLUSTER */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } -#line 5800 "sql.c" break; - case 257: /* cmd ::= SHOW TRANSACTIONS */ -#line 462 "sql.y" + case 262: /* cmd ::= SHOW TRANSACTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } -#line 5805 "sql.c" break; - case 258: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ -#line 463 "sql.y" -{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy164); } -#line 5810 "sql.c" + case 263: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ +{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy452); } break; - case 259: /* cmd ::= SHOW CONSUMERS */ -#line 464 "sql.y" + case 264: /* cmd ::= SHOW CONSUMERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } -#line 5815 "sql.c" break; - case 260: /* cmd ::= SHOW SUBSCRIPTIONS */ -#line 465 "sql.y" + case 265: /* cmd ::= SHOW SUBSCRIPTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } -#line 5820 "sql.c" break; - case 261: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ -#line 466 "sql.y" -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy164, yymsp[-1].minor.yy164, OP_TYPE_EQUAL); } -#line 5825 "sql.c" + case 266: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy452, yymsp[-1].minor.yy452, OP_TYPE_EQUAL); } + break; + case 267: /* cmd ::= SHOW TAGS FROM db_name NK_DOT table_name */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy371), createIdentifierValueNode(pCxt, &yymsp[0].minor.yy371), OP_TYPE_EQUAL); } + break; + case 268: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy452, yymsp[0].minor.yy452, yymsp[-3].minor.yy812); } break; - case 262: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ -#line 467 "sql.y" -{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy164, yymsp[0].minor.yy164, yymsp[-3].minor.yy72); } -#line 5830 "sql.c" + case 269: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM db_name NK_DOT table_name */ +{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, createIdentifierValueNode(pCxt, &yymsp[0].minor.yy371), createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy371), yymsp[-4].minor.yy812); } break; - case 263: /* cmd ::= SHOW VNODES NK_INTEGER */ -#line 468 "sql.y" + case 270: /* cmd ::= SHOW VNODES NK_INTEGER */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0), NULL); } -#line 5835 "sql.c" break; - case 264: /* cmd ::= SHOW VNODES NK_STRING */ -#line 469 "sql.y" + case 271: /* cmd ::= SHOW VNODES NK_STRING */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &yymsp[0].minor.yy0)); } -#line 5840 "sql.c" break; - case 265: /* cmd ::= SHOW db_name_cond_opt ALIVE */ -#line 471 "sql.y" -{ pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy164, QUERY_NODE_SHOW_DB_ALIVE_STMT); } -#line 5845 "sql.c" + case 272: /* cmd ::= SHOW db_name_cond_opt ALIVE */ +{ pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy452, QUERY_NODE_SHOW_DB_ALIVE_STMT); } break; - case 266: /* cmd ::= SHOW CLUSTER ALIVE */ -#line 472 "sql.y" + case 273: /* cmd ::= SHOW CLUSTER ALIVE */ { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } -#line 5850 "sql.c" - break; - case 267: /* db_name_cond_opt ::= */ - case 272: /* from_db_opt ::= */ yytestcase(yyruleno==272); -#line 474 "sql.y" -{ yymsp[1].minor.yy164 = createDefaultDatabaseCondValue(pCxt); } -#line 5856 "sql.c" - break; - case 268: /* db_name_cond_opt ::= db_name NK_DOT */ -#line 475 "sql.y" -{ yylhsminor.yy164 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy497); } -#line 5861 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; - break; - case 270: /* like_pattern_opt ::= LIKE NK_STRING */ -#line 478 "sql.y" -{ yymsp[-1].minor.yy164 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } -#line 5867 "sql.c" - break; - case 271: /* table_name_cond ::= table_name */ -#line 480 "sql.y" -{ yylhsminor.yy164 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy497); } -#line 5872 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 273: /* from_db_opt ::= FROM db_name */ -#line 483 "sql.y" -{ yymsp[-1].minor.yy164 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy497); } -#line 5878 "sql.c" - break; - case 277: /* tag_item ::= TBNAME */ -#line 491 "sql.y" -{ yylhsminor.yy164 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } -#line 5883 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 280: /* tag_item ::= column_name column_alias */ -#line 494 "sql.y" -{ yylhsminor.yy164 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy497), &yymsp[0].minor.yy497); } -#line 5889 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; - break; - case 281: /* tag_item ::= column_name AS column_alias */ -#line 495 "sql.y" -{ yylhsminor.yy164 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy497), &yymsp[0].minor.yy497); } -#line 5895 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 282: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ -#line 499 "sql.y" -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy441, yymsp[-3].minor.yy164, yymsp[-1].minor.yy164, NULL, yymsp[0].minor.yy164); } -#line 5901 "sql.c" - break; - case 283: /* cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ -#line 501 "sql.y" -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy441, yymsp[-5].minor.yy164, yymsp[-3].minor.yy164, yymsp[-1].minor.yy72, NULL); } -#line 5906 "sql.c" - break; - case 284: /* cmd ::= DROP INDEX exists_opt full_index_name */ -#line 502 "sql.y" -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy441, yymsp[0].minor.yy164); } -#line 5911 "sql.c" - break; - case 285: /* full_index_name ::= index_name */ -#line 504 "sql.y" -{ yylhsminor.yy164 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy497); } -#line 5916 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 286: /* full_index_name ::= db_name NK_DOT index_name */ -#line 505 "sql.y" -{ yylhsminor.yy164 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy497, &yymsp[0].minor.yy497); } -#line 5922 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 287: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ -#line 508 "sql.y" -{ yymsp[-9].minor.yy164 = createIndexOption(pCxt, yymsp[-7].minor.yy72, releaseRawExprNode(pCxt, yymsp[-3].minor.yy164), NULL, yymsp[-1].minor.yy164, yymsp[0].minor.yy164); } -#line 5928 "sql.c" - break; - case 288: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ -#line 511 "sql.y" -{ yymsp[-11].minor.yy164 = createIndexOption(pCxt, yymsp[-9].minor.yy72, releaseRawExprNode(pCxt, yymsp[-5].minor.yy164), releaseRawExprNode(pCxt, yymsp[-3].minor.yy164), yymsp[-1].minor.yy164, yymsp[0].minor.yy164); } -#line 5933 "sql.c" - break; - case 291: /* func ::= sma_func_name NK_LP expression_list NK_RP */ -#line 518 "sql.y" -{ yylhsminor.yy164 = createFunctionNode(pCxt, &yymsp[-3].minor.yy497, yymsp[-1].minor.yy72); } -#line 5938 "sql.c" - yymsp[-3].minor.yy164 = yylhsminor.yy164; - break; - case 292: /* sma_func_name ::= function_name */ - case 505: /* alias_opt ::= table_alias */ yytestcase(yyruleno==505); -#line 522 "sql.y" -{ yylhsminor.yy497 = yymsp[0].minor.yy497; } -#line 5945 "sql.c" - yymsp[0].minor.yy497 = yylhsminor.yy497; - break; - case 297: /* sma_stream_opt ::= */ - case 337: /* stream_options ::= */ yytestcase(yyruleno==337); -#line 528 "sql.y" -{ yymsp[1].minor.yy164 = createStreamOptions(pCxt); } -#line 5952 "sql.c" - break; - case 298: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ -#line 529 "sql.y" -{ ((SStreamOptions*)yymsp[-2].minor.yy164)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy164); yylhsminor.yy164 = yymsp[-2].minor.yy164; } -#line 5957 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 299: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ -#line 530 "sql.y" -{ ((SStreamOptions*)yymsp[-2].minor.yy164)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy164); yylhsminor.yy164 = yymsp[-2].minor.yy164; } -#line 5963 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 300: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ -#line 531 "sql.y" -{ ((SStreamOptions*)yymsp[-2].minor.yy164)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy164); yylhsminor.yy164 = yymsp[-2].minor.yy164; } -#line 5969 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 301: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ -#line 534 "sql.y" -{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy441, &yymsp[-2].minor.yy497, yymsp[0].minor.yy164); } -#line 5975 "sql.c" - break; - case 302: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ -#line 535 "sql.y" -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy441, &yymsp[-3].minor.yy497, &yymsp[0].minor.yy497, false); } -#line 5980 "sql.c" - break; - case 303: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ -#line 537 "sql.y" -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy441, &yymsp[-5].minor.yy497, &yymsp[0].minor.yy497, true); } -#line 5985 "sql.c" - break; - case 304: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ -#line 539 "sql.y" -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy441, &yymsp[-3].minor.yy497, yymsp[0].minor.yy164, false); } -#line 5990 "sql.c" - break; - case 305: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ -#line 541 "sql.y" -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy441, &yymsp[-5].minor.yy497, yymsp[0].minor.yy164, true); } -#line 5995 "sql.c" - break; - case 306: /* cmd ::= DROP TOPIC exists_opt topic_name */ -#line 542 "sql.y" -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy441, &yymsp[0].minor.yy497); } -#line 6000 "sql.c" - break; - case 307: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ -#line 543 "sql.y" -{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy441, &yymsp[-2].minor.yy497, &yymsp[0].minor.yy497); } -#line 6005 "sql.c" - break; - case 308: /* cmd ::= DESC full_table_name */ - case 309: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==309); -#line 546 "sql.y" -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy164); } -#line 6011 "sql.c" - break; - case 310: /* cmd ::= RESET QUERY CACHE */ -#line 550 "sql.y" + break; + case 274: /* db_name_cond_opt ::= */ + case 279: /* from_db_opt ::= */ yytestcase(yyruleno==279); +{ yymsp[1].minor.yy452 = createDefaultDatabaseCondValue(pCxt); } + break; + case 275: /* db_name_cond_opt ::= db_name NK_DOT */ +{ yylhsminor.yy452 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy371); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 277: /* like_pattern_opt ::= LIKE NK_STRING */ +{ yymsp[-1].minor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + break; + case 278: /* table_name_cond ::= table_name */ +{ yylhsminor.yy452 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy371); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 280: /* from_db_opt ::= FROM db_name */ +{ yymsp[-1].minor.yy452 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy371); } + break; + case 284: /* tag_item ::= TBNAME */ +{ yylhsminor.yy452 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 287: /* tag_item ::= column_name column_alias */ +{ yylhsminor.yy452 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy371), &yymsp[0].minor.yy371); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 288: /* tag_item ::= column_name AS column_alias */ +{ yylhsminor.yy452 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy371), &yymsp[0].minor.yy371); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 289: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy667, yymsp[-3].minor.yy452, yymsp[-1].minor.yy452, NULL, yymsp[0].minor.yy452); } + break; + case 290: /* cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy667, yymsp[-5].minor.yy452, yymsp[-3].minor.yy452, yymsp[-1].minor.yy812, NULL); } + break; + case 291: /* cmd ::= DROP INDEX exists_opt full_index_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy667, yymsp[0].minor.yy452); } + break; + case 292: /* full_index_name ::= index_name */ +{ yylhsminor.yy452 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy371); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 293: /* full_index_name ::= db_name NK_DOT index_name */ +{ yylhsminor.yy452 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy371); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 294: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-9].minor.yy452 = createIndexOption(pCxt, yymsp[-7].minor.yy812, releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), NULL, yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } + break; + case 295: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-11].minor.yy452 = createIndexOption(pCxt, yymsp[-9].minor.yy812, releaseRawExprNode(pCxt, yymsp[-5].minor.yy452), releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } + break; + case 298: /* func ::= sma_func_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy452 = createFunctionNode(pCxt, &yymsp[-3].minor.yy371, yymsp[-1].minor.yy812); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 299: /* sma_func_name ::= function_name */ + case 513: /* alias_opt ::= table_alias */ yytestcase(yyruleno==513); +{ yylhsminor.yy371 = yymsp[0].minor.yy371; } + yymsp[0].minor.yy371 = yylhsminor.yy371; + break; + case 304: /* sma_stream_opt ::= */ + case 345: /* stream_options ::= */ yytestcase(yyruleno==345); +{ yymsp[1].minor.yy452 = createStreamOptions(pCxt); } + break; + case 305: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy452)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy452); yylhsminor.yy452 = yymsp[-2].minor.yy452; } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 306: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy452)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy452); yylhsminor.yy452 = yymsp[-2].minor.yy452; } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 307: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy452)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy452); yylhsminor.yy452 = yymsp[-2].minor.yy452; } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 308: /* with_meta ::= AS */ +{ yymsp[0].minor.yy416 = 0; } + break; + case 309: /* with_meta ::= WITH META AS */ +{ yymsp[-2].minor.yy416 = 1; } + break; + case 310: /* with_meta ::= ONLY META AS */ +{ yymsp[-2].minor.yy416 = 2; } + break; + case 311: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ +{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy667, &yymsp[-2].minor.yy371, yymsp[0].minor.yy452); } + break; + case 312: /* cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy667, &yymsp[-3].minor.yy371, &yymsp[0].minor.yy371, yymsp[-2].minor.yy416); } + break; + case 313: /* cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta STABLE full_table_name where_clause_opt */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-5].minor.yy667, &yymsp[-4].minor.yy371, yymsp[-1].minor.yy452, yymsp[-3].minor.yy416, yymsp[0].minor.yy452); } + break; + case 314: /* cmd ::= DROP TOPIC exists_opt topic_name */ +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy667, &yymsp[0].minor.yy371); } + break; + case 315: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ +{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy667, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy371); } + break; + case 316: /* cmd ::= DESC full_table_name */ + case 317: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==317); +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy452); } + break; + case 318: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } -#line 6016 "sql.c" - break; - case 311: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - case 312: /* cmd ::= EXPLAIN analyze_opt explain_options insert_query */ yytestcase(yyruleno==312); -#line 553 "sql.y" -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy441, yymsp[-1].minor.yy164, yymsp[0].minor.yy164); } -#line 6022 "sql.c" - break; - case 315: /* explain_options ::= */ -#line 561 "sql.y" -{ yymsp[1].minor.yy164 = createDefaultExplainOptions(pCxt); } -#line 6027 "sql.c" - break; - case 316: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -#line 562 "sql.y" -{ yylhsminor.yy164 = setExplainVerbose(pCxt, yymsp[-2].minor.yy164, &yymsp[0].minor.yy0); } -#line 6032 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 317: /* explain_options ::= explain_options RATIO NK_FLOAT */ -#line 563 "sql.y" -{ yylhsminor.yy164 = setExplainRatio(pCxt, yymsp[-2].minor.yy164, &yymsp[0].minor.yy0); } -#line 6038 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 318: /* cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ -#line 568 "sql.y" -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-7].minor.yy441, yymsp[-9].minor.yy441, &yymsp[-6].minor.yy497, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy700, yymsp[-1].minor.yy560, &yymsp[0].minor.yy497, yymsp[-10].minor.yy441); } -#line 6044 "sql.c" - break; - case 319: /* cmd ::= DROP FUNCTION exists_opt function_name */ -#line 569 "sql.y" -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy441, &yymsp[0].minor.yy497); } -#line 6049 "sql.c" - break; - case 324: /* language_opt ::= */ -#line 583 "sql.y" -{ yymsp[1].minor.yy497 = nil_token; } -#line 6054 "sql.c" - break; - case 325: /* language_opt ::= LANGUAGE NK_STRING */ -#line 584 "sql.y" -{ yymsp[-1].minor.yy497 = yymsp[0].minor.yy0; } -#line 6059 "sql.c" - break; - case 328: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ -#line 594 "sql.y" -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy441, &yymsp[-8].minor.yy497, yymsp[-5].minor.yy164, yymsp[-7].minor.yy164, yymsp[-3].minor.yy72, yymsp[-2].minor.yy164, yymsp[0].minor.yy164, yymsp[-4].minor.yy72); } -#line 6064 "sql.c" - break; - case 329: /* cmd ::= DROP STREAM exists_opt stream_name */ -#line 595 "sql.y" -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy441, &yymsp[0].minor.yy497); } -#line 6069 "sql.c" - break; - case 330: /* cmd ::= PAUSE STREAM exists_opt stream_name */ -#line 596 "sql.y" -{ pCxt->pRootNode = createPauseStreamStmt(pCxt, yymsp[-1].minor.yy441, &yymsp[0].minor.yy497); } -#line 6074 "sql.c" - break; - case 331: /* cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */ -#line 597 "sql.y" -{ pCxt->pRootNode = createResumeStreamStmt(pCxt, yymsp[-2].minor.yy441, yymsp[-1].minor.yy441, &yymsp[0].minor.yy497); } -#line 6079 "sql.c" - break; - case 338: /* stream_options ::= stream_options TRIGGER AT_ONCE */ - case 339: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ yytestcase(yyruleno==339); -#line 611 "sql.y" -{ yylhsminor.yy164 = setStreamOptions(pCxt, yymsp[-2].minor.yy164, SOPT_TRIGGER_TYPE_SET, &yymsp[0].minor.yy0, NULL); } -#line 6085 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 340: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ -#line 613 "sql.y" -{ yylhsminor.yy164 = setStreamOptions(pCxt, yymsp[-3].minor.yy164, SOPT_TRIGGER_TYPE_SET, &yymsp[-1].minor.yy0, releaseRawExprNode(pCxt, yymsp[0].minor.yy164)); } -#line 6091 "sql.c" - yymsp[-3].minor.yy164 = yylhsminor.yy164; - break; - case 341: /* stream_options ::= stream_options WATERMARK duration_literal */ -#line 614 "sql.y" -{ yylhsminor.yy164 = setStreamOptions(pCxt, yymsp[-2].minor.yy164, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy164)); } -#line 6097 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 342: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ -#line 615 "sql.y" -{ yylhsminor.yy164 = setStreamOptions(pCxt, yymsp[-3].minor.yy164, SOPT_IGNORE_EXPIRED_SET, &yymsp[0].minor.yy0, NULL); } -#line 6103 "sql.c" - yymsp[-3].minor.yy164 = yylhsminor.yy164; - break; - case 343: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ -#line 616 "sql.y" -{ yylhsminor.yy164 = setStreamOptions(pCxt, yymsp[-2].minor.yy164, SOPT_FILL_HISTORY_SET, &yymsp[0].minor.yy0, NULL); } -#line 6109 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 344: /* stream_options ::= stream_options DELETE_MARK duration_literal */ -#line 617 "sql.y" -{ yylhsminor.yy164 = setStreamOptions(pCxt, yymsp[-2].minor.yy164, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy164)); } -#line 6115 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 345: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ -#line 618 "sql.y" -{ yylhsminor.yy164 = setStreamOptions(pCxt, yymsp[-3].minor.yy164, SOPT_IGNORE_UPDATE_SET, &yymsp[0].minor.yy0, NULL); } -#line 6121 "sql.c" - yymsp[-3].minor.yy164 = yylhsminor.yy164; - break; - case 347: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - case 539: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==539); - case 559: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==559); -#line 621 "sql.y" -{ yymsp[-3].minor.yy164 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy164); } -#line 6129 "sql.c" - break; - case 350: /* cmd ::= KILL CONNECTION NK_INTEGER */ -#line 629 "sql.y" + break; + case 319: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + case 320: /* cmd ::= EXPLAIN analyze_opt explain_options insert_query */ yytestcase(yyruleno==320); +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy667, yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } + break; + case 323: /* explain_options ::= */ +{ yymsp[1].minor.yy452 = createDefaultExplainOptions(pCxt); } + break; + case 324: /* explain_options ::= explain_options VERBOSE NK_BOOL */ +{ yylhsminor.yy452 = setExplainVerbose(pCxt, yymsp[-2].minor.yy452, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 325: /* explain_options ::= explain_options RATIO NK_FLOAT */ +{ yylhsminor.yy452 = setExplainRatio(pCxt, yymsp[-2].minor.yy452, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 326: /* cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-7].minor.yy667, yymsp[-9].minor.yy667, &yymsp[-6].minor.yy371, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy310, yymsp[-1].minor.yy416, &yymsp[0].minor.yy371, yymsp[-10].minor.yy667); } + break; + case 327: /* cmd ::= DROP FUNCTION exists_opt function_name */ +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy667, &yymsp[0].minor.yy371); } + break; + case 332: /* language_opt ::= */ +{ yymsp[1].minor.yy371 = nil_token; } + break; + case 333: /* language_opt ::= LANGUAGE NK_STRING */ +{ yymsp[-1].minor.yy371 = yymsp[0].minor.yy0; } + break; + case 336: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy667, &yymsp[-8].minor.yy371, yymsp[-5].minor.yy452, yymsp[-7].minor.yy452, yymsp[-3].minor.yy812, yymsp[-2].minor.yy452, yymsp[0].minor.yy452, yymsp[-4].minor.yy812); } + break; + case 337: /* cmd ::= DROP STREAM exists_opt stream_name */ +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy667, &yymsp[0].minor.yy371); } + break; + case 338: /* cmd ::= PAUSE STREAM exists_opt stream_name */ +{ pCxt->pRootNode = createPauseStreamStmt(pCxt, yymsp[-1].minor.yy667, &yymsp[0].minor.yy371); } + break; + case 339: /* cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */ +{ pCxt->pRootNode = createResumeStreamStmt(pCxt, yymsp[-2].minor.yy667, yymsp[-1].minor.yy667, &yymsp[0].minor.yy371); } + break; + case 346: /* stream_options ::= stream_options TRIGGER AT_ONCE */ + case 347: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ yytestcase(yyruleno==347); +{ yylhsminor.yy452 = setStreamOptions(pCxt, yymsp[-2].minor.yy452, SOPT_TRIGGER_TYPE_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 348: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ +{ yylhsminor.yy452 = setStreamOptions(pCxt, yymsp[-3].minor.yy452, SOPT_TRIGGER_TYPE_SET, &yymsp[-1].minor.yy0, releaseRawExprNode(pCxt, yymsp[0].minor.yy452)); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 349: /* stream_options ::= stream_options WATERMARK duration_literal */ +{ yylhsminor.yy452 = setStreamOptions(pCxt, yymsp[-2].minor.yy452, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy452)); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 350: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ +{ yylhsminor.yy452 = setStreamOptions(pCxt, yymsp[-3].minor.yy452, SOPT_IGNORE_EXPIRED_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 351: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ +{ yylhsminor.yy452 = setStreamOptions(pCxt, yymsp[-2].minor.yy452, SOPT_FILL_HISTORY_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 352: /* stream_options ::= stream_options DELETE_MARK duration_literal */ +{ yylhsminor.yy452 = setStreamOptions(pCxt, yymsp[-2].minor.yy452, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy452)); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 353: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ +{ yylhsminor.yy452 = setStreamOptions(pCxt, yymsp[-3].minor.yy452, SOPT_IGNORE_UPDATE_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 355: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + case 547: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==547); + case 568: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==568); +{ yymsp[-3].minor.yy452 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy452); } + break; + case 358: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } -#line 6134 "sql.c" break; - case 351: /* cmd ::= KILL QUERY NK_STRING */ -#line 630 "sql.y" + case 359: /* cmd ::= KILL QUERY NK_STRING */ { pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); } -#line 6139 "sql.c" break; - case 352: /* cmd ::= KILL TRANSACTION NK_INTEGER */ -#line 631 "sql.y" + case 360: /* cmd ::= KILL TRANSACTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); } -#line 6144 "sql.c" break; - case 353: /* cmd ::= BALANCE VGROUP */ -#line 634 "sql.y" + case 361: /* cmd ::= BALANCE VGROUP */ { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } -#line 6149 "sql.c" break; - case 354: /* cmd ::= BALANCE VGROUP LEADER */ -#line 635 "sql.y" + case 362: /* cmd ::= BALANCE VGROUP LEADER */ { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt); } -#line 6154 "sql.c" break; - case 355: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ -#line 636 "sql.y" + case 363: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } -#line 6159 "sql.c" break; - case 356: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ -#line 637 "sql.y" -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy72); } -#line 6164 "sql.c" + case 364: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy812); } break; - case 357: /* cmd ::= SPLIT VGROUP NK_INTEGER */ -#line 638 "sql.y" + case 365: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } -#line 6169 "sql.c" - break; - case 358: /* dnode_list ::= DNODE NK_INTEGER */ -#line 642 "sql.y" -{ yymsp[-1].minor.yy72 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } -#line 6174 "sql.c" - break; - case 360: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ -#line 649 "sql.y" -{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy164, yymsp[0].minor.yy164); } -#line 6179 "sql.c" - break; - case 363: /* insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ -#line 658 "sql.y" -{ yymsp[-6].minor.yy164 = createInsertStmt(pCxt, yymsp[-4].minor.yy164, yymsp[-2].minor.yy72, yymsp[0].minor.yy164); } -#line 6184 "sql.c" - break; - case 364: /* insert_query ::= INSERT INTO full_table_name query_or_subquery */ -#line 659 "sql.y" -{ yymsp[-3].minor.yy164 = createInsertStmt(pCxt, yymsp[-1].minor.yy164, NULL, yymsp[0].minor.yy164); } -#line 6189 "sql.c" - break; - case 365: /* literal ::= NK_INTEGER */ -#line 662 "sql.y" -{ yylhsminor.yy164 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } -#line 6194 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 366: /* literal ::= NK_FLOAT */ -#line 663 "sql.y" -{ yylhsminor.yy164 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } -#line 6200 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 367: /* literal ::= NK_STRING */ -#line 664 "sql.y" -{ yylhsminor.yy164 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } -#line 6206 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 368: /* literal ::= NK_BOOL */ -#line 665 "sql.y" -{ yylhsminor.yy164 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } -#line 6212 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 369: /* literal ::= TIMESTAMP NK_STRING */ -#line 666 "sql.y" -{ yylhsminor.yy164 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } -#line 6218 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; - break; - case 370: /* literal ::= duration_literal */ - case 380: /* signed_literal ::= signed */ yytestcase(yyruleno==380); - case 401: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==401); - case 402: /* expression ::= literal */ yytestcase(yyruleno==402); - case 403: /* expression ::= pseudo_column */ yytestcase(yyruleno==403); - case 404: /* expression ::= column_reference */ yytestcase(yyruleno==404); - case 405: /* expression ::= function_expression */ yytestcase(yyruleno==405); - case 406: /* expression ::= case_when_expression */ yytestcase(yyruleno==406); - case 437: /* function_expression ::= literal_func */ yytestcase(yyruleno==437); - case 486: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==486); - case 490: /* boolean_primary ::= predicate */ yytestcase(yyruleno==490); - case 492: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==492); - case 493: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==493); - case 496: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==496); - case 498: /* table_reference ::= table_primary */ yytestcase(yyruleno==498); - case 499: /* table_reference ::= joined_table */ yytestcase(yyruleno==499); - case 503: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==503); - case 561: /* query_simple ::= query_specification */ yytestcase(yyruleno==561); - case 562: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==562); - case 565: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==565); - case 567: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==567); -#line 667 "sql.y" -{ yylhsminor.yy164 = yymsp[0].minor.yy164; } -#line 6244 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 371: /* literal ::= NULL */ -#line 668 "sql.y" -{ yylhsminor.yy164 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } -#line 6250 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 372: /* literal ::= NK_QUESTION */ -#line 669 "sql.y" -{ yylhsminor.yy164 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } -#line 6256 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 373: /* duration_literal ::= NK_VARIABLE */ -#line 671 "sql.y" -{ yylhsminor.yy164 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } -#line 6262 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 374: /* signed ::= NK_INTEGER */ -#line 673 "sql.y" -{ yylhsminor.yy164 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } -#line 6268 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 375: /* signed ::= NK_PLUS NK_INTEGER */ -#line 674 "sql.y" -{ yymsp[-1].minor.yy164 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } -#line 6274 "sql.c" - break; - case 376: /* signed ::= NK_MINUS NK_INTEGER */ -#line 675 "sql.y" + break; + case 366: /* dnode_list ::= DNODE NK_INTEGER */ +{ yymsp[-1].minor.yy812 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + break; + case 368: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ +{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } + break; + case 371: /* insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ +{ yymsp[-6].minor.yy452 = createInsertStmt(pCxt, yymsp[-4].minor.yy452, yymsp[-2].minor.yy812, yymsp[0].minor.yy452); } + break; + case 372: /* insert_query ::= INSERT INTO full_table_name query_or_subquery */ +{ yymsp[-3].minor.yy452 = createInsertStmt(pCxt, yymsp[-1].minor.yy452, NULL, yymsp[0].minor.yy452); } + break; + case 373: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 374: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 375: /* literal ::= NK_STRING */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 376: /* literal ::= NK_BOOL */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 377: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 378: /* literal ::= duration_literal */ + case 388: /* signed_literal ::= signed */ yytestcase(yyruleno==388); + case 409: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==409); + case 410: /* expression ::= literal */ yytestcase(yyruleno==410); + case 411: /* expression ::= pseudo_column */ yytestcase(yyruleno==411); + case 412: /* expression ::= column_reference */ yytestcase(yyruleno==412); + case 413: /* expression ::= function_expression */ yytestcase(yyruleno==413); + case 414: /* expression ::= case_when_expression */ yytestcase(yyruleno==414); + case 445: /* function_expression ::= literal_func */ yytestcase(yyruleno==445); + case 494: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==494); + case 498: /* boolean_primary ::= predicate */ yytestcase(yyruleno==498); + case 500: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==500); + case 501: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==501); + case 504: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==504); + case 506: /* table_reference ::= table_primary */ yytestcase(yyruleno==506); + case 507: /* table_reference ::= joined_table */ yytestcase(yyruleno==507); + case 511: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==511); + case 570: /* query_simple ::= query_specification */ yytestcase(yyruleno==570); + case 571: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==571); + case 574: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==574); + case 576: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==576); +{ yylhsminor.yy452 = yymsp[0].minor.yy452; } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 379: /* literal ::= NULL */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 380: /* literal ::= NK_QUESTION */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 381: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 382: /* signed ::= NK_INTEGER */ +{ yylhsminor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 383: /* signed ::= NK_PLUS NK_INTEGER */ +{ yymsp[-1].minor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + break; + case 384: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy164 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } -#line 6283 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; - break; - case 377: /* signed ::= NK_FLOAT */ -#line 680 "sql.y" -{ yylhsminor.yy164 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } -#line 6289 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 378: /* signed ::= NK_PLUS NK_FLOAT */ -#line 681 "sql.y" -{ yymsp[-1].minor.yy164 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } -#line 6295 "sql.c" - break; - case 379: /* signed ::= NK_MINUS NK_FLOAT */ -#line 682 "sql.y" + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 385: /* signed ::= NK_FLOAT */ +{ yylhsminor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 386: /* signed ::= NK_PLUS NK_FLOAT */ +{ yymsp[-1].minor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + break; + case 387: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy164 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } -#line 6304 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; - break; - case 381: /* signed_literal ::= NK_STRING */ -#line 689 "sql.y" -{ yylhsminor.yy164 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } -#line 6310 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 382: /* signed_literal ::= NK_BOOL */ -#line 690 "sql.y" -{ yylhsminor.yy164 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } -#line 6316 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 383: /* signed_literal ::= TIMESTAMP NK_STRING */ -#line 691 "sql.y" -{ yymsp[-1].minor.yy164 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } -#line 6322 "sql.c" - break; - case 384: /* signed_literal ::= duration_literal */ - case 386: /* signed_literal ::= literal_func */ yytestcase(yyruleno==386); - case 457: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==457); - case 519: /* select_item ::= common_expression */ yytestcase(yyruleno==519); - case 529: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==529); - case 566: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==566); - case 568: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==568); - case 581: /* search_condition ::= common_expression */ yytestcase(yyruleno==581); -#line 692 "sql.y" -{ yylhsminor.yy164 = releaseRawExprNode(pCxt, yymsp[0].minor.yy164); } -#line 6334 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 385: /* signed_literal ::= NULL */ -#line 693 "sql.y" -{ yylhsminor.yy164 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } -#line 6340 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 387: /* signed_literal ::= NK_QUESTION */ -#line 695 "sql.y" -{ yylhsminor.yy164 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } -#line 6346 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 407: /* expression ::= NK_LP expression NK_RP */ - case 491: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==491); - case 580: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==580); -#line 756 "sql.y" -{ yylhsminor.yy164 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy164)); } -#line 6354 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 408: /* expression ::= NK_PLUS expr_or_subquery */ -#line 757 "sql.y" + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 389: /* signed_literal ::= NK_STRING */ +{ yylhsminor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 390: /* signed_literal ::= NK_BOOL */ +{ yylhsminor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 391: /* signed_literal ::= TIMESTAMP NK_STRING */ +{ yymsp[-1].minor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 392: /* signed_literal ::= duration_literal */ + case 394: /* signed_literal ::= literal_func */ yytestcase(yyruleno==394); + case 465: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==465); + case 527: /* select_item ::= common_expression */ yytestcase(yyruleno==527); + case 537: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==537); + case 575: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==575); + case 577: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==577); + case 590: /* search_condition ::= common_expression */ yytestcase(yyruleno==590); +{ yylhsminor.yy452 = releaseRawExprNode(pCxt, yymsp[0].minor.yy452); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 393: /* signed_literal ::= NULL */ +{ yylhsminor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 395: /* signed_literal ::= NK_QUESTION */ +{ yylhsminor.yy452 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 415: /* expression ::= NK_LP expression NK_RP */ + case 499: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==499); + case 589: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==589); +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy452)); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 416: /* expression ::= NK_PLUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy164)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy452)); } -#line 6363 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; - case 409: /* expression ::= NK_MINUS expr_or_subquery */ -#line 761 "sql.y" + case 417: /* expression ::= NK_MINUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy164), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy452), NULL)); } -#line 6372 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; - case 410: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ -#line 765 "sql.y" + case 418: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy164); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), releaseRawExprNode(pCxt, yymsp[0].minor.yy164))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } -#line 6382 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 411: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ -#line 770 "sql.y" + case 419: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy164); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), releaseRawExprNode(pCxt, yymsp[0].minor.yy164))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } -#line 6392 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 412: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ -#line 775 "sql.y" + case 420: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy164); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), releaseRawExprNode(pCxt, yymsp[0].minor.yy164))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } -#line 6402 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 413: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ -#line 780 "sql.y" + case 421: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy164); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), releaseRawExprNode(pCxt, yymsp[0].minor.yy164))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } -#line 6412 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 414: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ -#line 785 "sql.y" + case 422: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy164); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), releaseRawExprNode(pCxt, yymsp[0].minor.yy164))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } -#line 6422 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 415: /* expression ::= column_reference NK_ARROW NK_STRING */ -#line 790 "sql.y" + case 423: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } -#line 6431 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 416: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ -#line 794 "sql.y" + case 424: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy164); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), releaseRawExprNode(pCxt, yymsp[0].minor.yy164))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } -#line 6441 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 417: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ -#line 799 "sql.y" + case 425: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy164); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), releaseRawExprNode(pCxt, yymsp[0].minor.yy164))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } -#line 6451 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 420: /* column_reference ::= column_name */ -#line 810 "sql.y" -{ yylhsminor.yy164 = createRawExprNode(pCxt, &yymsp[0].minor.yy497, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy497)); } -#line 6457 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 421: /* column_reference ::= table_name NK_DOT column_name */ -#line 811 "sql.y" -{ yylhsminor.yy164 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy497, &yymsp[0].minor.yy497, createColumnNode(pCxt, &yymsp[-2].minor.yy497, &yymsp[0].minor.yy497)); } -#line 6463 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 422: /* pseudo_column ::= ROWTS */ - case 423: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==423); - case 425: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==425); - case 426: /* pseudo_column ::= QEND */ yytestcase(yyruleno==426); - case 427: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==427); - case 428: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==428); - case 429: /* pseudo_column ::= WEND */ yytestcase(yyruleno==429); - case 430: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==430); - case 431: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==431); - case 432: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==432); - case 433: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==433); - case 439: /* literal_func ::= NOW */ yytestcase(yyruleno==439); -#line 813 "sql.y" -{ yylhsminor.yy164 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } -#line 6480 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 424: /* pseudo_column ::= table_name NK_DOT TBNAME */ -#line 815 "sql.y" -{ yylhsminor.yy164 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy497, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy497)))); } -#line 6486 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 434: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 435: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==435); -#line 826 "sql.y" -{ yylhsminor.yy164 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy497, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy497, yymsp[-1].minor.yy72)); } -#line 6493 "sql.c" - yymsp[-3].minor.yy164 = yylhsminor.yy164; - break; - case 436: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ -#line 829 "sql.y" -{ yylhsminor.yy164 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy164), yymsp[-1].minor.yy700)); } -#line 6499 "sql.c" - yymsp[-5].minor.yy164 = yylhsminor.yy164; - break; - case 438: /* literal_func ::= noarg_func NK_LP NK_RP */ -#line 832 "sql.y" -{ yylhsminor.yy164 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy497, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy497, NULL)); } -#line 6505 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 453: /* star_func_para_list ::= NK_STAR */ -#line 856 "sql.y" -{ yylhsminor.yy72 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } -#line 6511 "sql.c" - yymsp[0].minor.yy72 = yylhsminor.yy72; - break; - case 458: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 522: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==522); -#line 865 "sql.y" -{ yylhsminor.yy164 = createColumnNode(pCxt, &yymsp[-2].minor.yy497, &yymsp[0].minor.yy0); } -#line 6518 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 459: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ -#line 868 "sql.y" -{ yylhsminor.yy164 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy72, yymsp[-1].minor.yy164)); } -#line 6524 "sql.c" - yymsp[-3].minor.yy164 = yylhsminor.yy164; - break; - case 460: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ -#line 870 "sql.y" -{ yylhsminor.yy164 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy164), yymsp[-2].minor.yy72, yymsp[-1].minor.yy164)); } -#line 6530 "sql.c" - yymsp[-4].minor.yy164 = yylhsminor.yy164; - break; - case 463: /* when_then_expr ::= WHEN common_expression THEN common_expression */ -#line 877 "sql.y" -{ yymsp[-3].minor.yy164 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), releaseRawExprNode(pCxt, yymsp[0].minor.yy164)); } -#line 6536 "sql.c" - break; - case 465: /* case_when_else_opt ::= ELSE common_expression */ -#line 880 "sql.y" -{ yymsp[-1].minor.yy164 = releaseRawExprNode(pCxt, yymsp[0].minor.yy164); } -#line 6541 "sql.c" - break; - case 466: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ - case 471: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==471); -#line 883 "sql.y" + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 428: /* column_reference ::= column_name */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy371, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy371)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 429: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy371, createColumnNode(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy371)); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 430: /* pseudo_column ::= ROWTS */ + case 431: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==431); + case 433: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==433); + case 434: /* pseudo_column ::= QEND */ yytestcase(yyruleno==434); + case 435: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==435); + case 436: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==436); + case 437: /* pseudo_column ::= WEND */ yytestcase(yyruleno==437); + case 438: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==438); + case 439: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==439); + case 440: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==440); + case 441: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==441); + case 447: /* literal_func ::= NOW */ yytestcase(yyruleno==447); +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 432: /* pseudo_column ::= table_name NK_DOT TBNAME */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy371)))); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 442: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 443: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==443); +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy371, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy371, yymsp[-1].minor.yy812)); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 444: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), yymsp[-1].minor.yy310)); } + yymsp[-5].minor.yy452 = yylhsminor.yy452; + break; + case 446: /* literal_func ::= noarg_func NK_LP NK_RP */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy371, NULL)); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 461: /* star_func_para_list ::= NK_STAR */ +{ yylhsminor.yy812 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy812 = yylhsminor.yy812; + break; + case 466: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 530: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==530); +{ yylhsminor.yy452 = createColumnNode(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 467: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy812, yymsp[-1].minor.yy452)); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 468: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), yymsp[-2].minor.yy812, yymsp[-1].minor.yy452)); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; + break; + case 471: /* when_then_expr ::= WHEN common_expression THEN common_expression */ +{ yymsp[-3].minor.yy452 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452)); } + break; + case 473: /* case_when_else_opt ::= ELSE common_expression */ +{ yymsp[-1].minor.yy452 = releaseRawExprNode(pCxt, yymsp[0].minor.yy452); } + break; + case 474: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ + case 479: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==479); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy164); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy796, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), releaseRawExprNode(pCxt, yymsp[0].minor.yy164))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy354, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } -#line 6551 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 467: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ -#line 890 "sql.y" + case 475: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy164); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy164), releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), releaseRawExprNode(pCxt, yymsp[0].minor.yy164))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy452), releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } -#line 6561 "sql.c" - yymsp[-4].minor.yy164 = yylhsminor.yy164; + yymsp[-4].minor.yy452 = yylhsminor.yy452; break; - case 468: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ -#line 896 "sql.y" + case 476: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy164); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy164), releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), releaseRawExprNode(pCxt, yymsp[0].minor.yy164))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy452), releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } -#line 6571 "sql.c" - yymsp[-5].minor.yy164 = yylhsminor.yy164; + yymsp[-5].minor.yy452 = yylhsminor.yy452; break; - case 469: /* predicate ::= expr_or_subquery IS NULL */ -#line 901 "sql.y" + case 477: /* predicate ::= expr_or_subquery IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), NULL)); } -#line 6580 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 470: /* predicate ::= expr_or_subquery IS NOT NULL */ -#line 905 "sql.y" + case 478: /* predicate ::= expr_or_subquery IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy164), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), NULL)); } -#line 6589 "sql.c" - yymsp[-3].minor.yy164 = yylhsminor.yy164; - break; - case 472: /* compare_op ::= NK_LT */ -#line 917 "sql.y" -{ yymsp[0].minor.yy796 = OP_TYPE_LOWER_THAN; } -#line 6595 "sql.c" - break; - case 473: /* compare_op ::= NK_GT */ -#line 918 "sql.y" -{ yymsp[0].minor.yy796 = OP_TYPE_GREATER_THAN; } -#line 6600 "sql.c" - break; - case 474: /* compare_op ::= NK_LE */ -#line 919 "sql.y" -{ yymsp[0].minor.yy796 = OP_TYPE_LOWER_EQUAL; } -#line 6605 "sql.c" - break; - case 475: /* compare_op ::= NK_GE */ -#line 920 "sql.y" -{ yymsp[0].minor.yy796 = OP_TYPE_GREATER_EQUAL; } -#line 6610 "sql.c" - break; - case 476: /* compare_op ::= NK_NE */ -#line 921 "sql.y" -{ yymsp[0].minor.yy796 = OP_TYPE_NOT_EQUAL; } -#line 6615 "sql.c" - break; - case 477: /* compare_op ::= NK_EQ */ -#line 922 "sql.y" -{ yymsp[0].minor.yy796 = OP_TYPE_EQUAL; } -#line 6620 "sql.c" - break; - case 478: /* compare_op ::= LIKE */ -#line 923 "sql.y" -{ yymsp[0].minor.yy796 = OP_TYPE_LIKE; } -#line 6625 "sql.c" - break; - case 479: /* compare_op ::= NOT LIKE */ -#line 924 "sql.y" -{ yymsp[-1].minor.yy796 = OP_TYPE_NOT_LIKE; } -#line 6630 "sql.c" - break; - case 480: /* compare_op ::= MATCH */ -#line 925 "sql.y" -{ yymsp[0].minor.yy796 = OP_TYPE_MATCH; } -#line 6635 "sql.c" - break; - case 481: /* compare_op ::= NMATCH */ -#line 926 "sql.y" -{ yymsp[0].minor.yy796 = OP_TYPE_NMATCH; } -#line 6640 "sql.c" - break; - case 482: /* compare_op ::= CONTAINS */ -#line 927 "sql.y" -{ yymsp[0].minor.yy796 = OP_TYPE_JSON_CONTAINS; } -#line 6645 "sql.c" - break; - case 483: /* in_op ::= IN */ -#line 931 "sql.y" -{ yymsp[0].minor.yy796 = OP_TYPE_IN; } -#line 6650 "sql.c" - break; - case 484: /* in_op ::= NOT IN */ -#line 932 "sql.y" -{ yymsp[-1].minor.yy796 = OP_TYPE_NOT_IN; } -#line 6655 "sql.c" - break; - case 485: /* in_predicate_value ::= NK_LP literal_list NK_RP */ -#line 934 "sql.y" -{ yylhsminor.yy164 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy72)); } -#line 6660 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 487: /* boolean_value_expression ::= NOT boolean_primary */ -#line 938 "sql.y" + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 480: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy354 = OP_TYPE_LOWER_THAN; } + break; + case 481: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy354 = OP_TYPE_GREATER_THAN; } + break; + case 482: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy354 = OP_TYPE_LOWER_EQUAL; } + break; + case 483: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy354 = OP_TYPE_GREATER_EQUAL; } + break; + case 484: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy354 = OP_TYPE_NOT_EQUAL; } + break; + case 485: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy354 = OP_TYPE_EQUAL; } + break; + case 486: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy354 = OP_TYPE_LIKE; } + break; + case 487: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy354 = OP_TYPE_NOT_LIKE; } + break; + case 488: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy354 = OP_TYPE_MATCH; } + break; + case 489: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy354 = OP_TYPE_NMATCH; } + break; + case 490: /* compare_op ::= CONTAINS */ +{ yymsp[0].minor.yy354 = OP_TYPE_JSON_CONTAINS; } + break; + case 491: /* in_op ::= IN */ +{ yymsp[0].minor.yy354 = OP_TYPE_IN; } + break; + case 492: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy354 = OP_TYPE_NOT_IN; } + break; + case 493: /* in_predicate_value ::= NK_LP literal_list NK_RP */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy812)); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 495: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy164), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy452), NULL)); } -#line 6669 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; - case 488: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ -#line 943 "sql.y" + case 496: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy164); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), releaseRawExprNode(pCxt, yymsp[0].minor.yy164))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } -#line 6679 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 489: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ -#line 949 "sql.y" + case 497: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy164); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy164); - yylhsminor.yy164 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), releaseRawExprNode(pCxt, yymsp[0].minor.yy164))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } -#line 6689 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 497: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -#line 967 "sql.y" -{ yylhsminor.yy164 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy164, yymsp[0].minor.yy164, NULL); } -#line 6695 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 500: /* table_primary ::= table_name alias_opt */ -#line 973 "sql.y" -{ yylhsminor.yy164 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy497, &yymsp[0].minor.yy497); } -#line 6701 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; - break; - case 501: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -#line 974 "sql.y" -{ yylhsminor.yy164 = createRealTableNode(pCxt, &yymsp[-3].minor.yy497, &yymsp[-1].minor.yy497, &yymsp[0].minor.yy497); } -#line 6707 "sql.c" - yymsp[-3].minor.yy164 = yylhsminor.yy164; - break; - case 502: /* table_primary ::= subquery alias_opt */ -#line 975 "sql.y" -{ yylhsminor.yy164 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy164), &yymsp[0].minor.yy497); } -#line 6713 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; - break; - case 504: /* alias_opt ::= */ -#line 980 "sql.y" -{ yymsp[1].minor.yy497 = nil_token; } -#line 6719 "sql.c" - break; - case 506: /* alias_opt ::= AS table_alias */ -#line 982 "sql.y" -{ yymsp[-1].minor.yy497 = yymsp[0].minor.yy497; } -#line 6724 "sql.c" - break; - case 507: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 508: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==508); -#line 984 "sql.y" -{ yymsp[-2].minor.yy164 = yymsp[-1].minor.yy164; } -#line 6730 "sql.c" - break; - case 509: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -#line 989 "sql.y" -{ yylhsminor.yy164 = createJoinTableNode(pCxt, yymsp[-4].minor.yy196, yymsp[-5].minor.yy164, yymsp[-2].minor.yy164, yymsp[0].minor.yy164); } -#line 6735 "sql.c" - yymsp[-5].minor.yy164 = yylhsminor.yy164; - break; - case 510: /* join_type ::= */ -#line 993 "sql.y" -{ yymsp[1].minor.yy196 = JOIN_TYPE_INNER; } -#line 6741 "sql.c" - break; - case 511: /* join_type ::= INNER */ -#line 994 "sql.y" -{ yymsp[0].minor.yy196 = JOIN_TYPE_INNER; } -#line 6746 "sql.c" - break; - case 512: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ -#line 1000 "sql.y" + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 505: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy452 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy452, yymsp[0].minor.yy452, NULL); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 508: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy452 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy371, &yymsp[0].minor.yy371); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 509: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy452 = createRealTableNode(pCxt, &yymsp[-3].minor.yy371, &yymsp[-1].minor.yy371, &yymsp[0].minor.yy371); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 510: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy452 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy452), &yymsp[0].minor.yy371); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 512: /* alias_opt ::= */ +{ yymsp[1].minor.yy371 = nil_token; } + break; + case 514: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy371 = yymsp[0].minor.yy371; } + break; + case 515: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 516: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==516); +{ yymsp[-2].minor.yy452 = yymsp[-1].minor.yy452; } + break; + case 517: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy452 = createJoinTableNode(pCxt, yymsp[-4].minor.yy140, yymsp[-5].minor.yy452, yymsp[-2].minor.yy452, yymsp[0].minor.yy452); } + yymsp[-5].minor.yy452 = yylhsminor.yy452; + break; + case 518: /* join_type ::= */ +{ yymsp[1].minor.yy140 = JOIN_TYPE_INNER; } + break; + case 519: /* join_type ::= INNER */ +{ yymsp[0].minor.yy140 = JOIN_TYPE_INNER; } + break; + case 520: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-11].minor.yy164 = createSelectStmt(pCxt, yymsp[-10].minor.yy441, yymsp[-9].minor.yy72, yymsp[-8].minor.yy164); - yymsp[-11].minor.yy164 = addWhereClause(pCxt, yymsp[-11].minor.yy164, yymsp[-7].minor.yy164); - yymsp[-11].minor.yy164 = addPartitionByClause(pCxt, yymsp[-11].minor.yy164, yymsp[-6].minor.yy72); - yymsp[-11].minor.yy164 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy164, yymsp[-2].minor.yy164); - yymsp[-11].minor.yy164 = addGroupByClause(pCxt, yymsp[-11].minor.yy164, yymsp[-1].minor.yy72); - yymsp[-11].minor.yy164 = addHavingClause(pCxt, yymsp[-11].minor.yy164, yymsp[0].minor.yy164); - yymsp[-11].minor.yy164 = addRangeClause(pCxt, yymsp[-11].minor.yy164, yymsp[-5].minor.yy164); - yymsp[-11].minor.yy164 = addEveryClause(pCxt, yymsp[-11].minor.yy164, yymsp[-4].minor.yy164); - yymsp[-11].minor.yy164 = addFillClause(pCxt, yymsp[-11].minor.yy164, yymsp[-3].minor.yy164); + yymsp[-11].minor.yy452 = createSelectStmt(pCxt, yymsp[-10].minor.yy667, yymsp[-9].minor.yy812, yymsp[-8].minor.yy452); + yymsp[-11].minor.yy452 = addWhereClause(pCxt, yymsp[-11].minor.yy452, yymsp[-7].minor.yy452); + yymsp[-11].minor.yy452 = addPartitionByClause(pCxt, yymsp[-11].minor.yy452, yymsp[-6].minor.yy812); + yymsp[-11].minor.yy452 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy452, yymsp[-2].minor.yy452); + yymsp[-11].minor.yy452 = addGroupByClause(pCxt, yymsp[-11].minor.yy452, yymsp[-1].minor.yy812); + yymsp[-11].minor.yy452 = addHavingClause(pCxt, yymsp[-11].minor.yy452, yymsp[0].minor.yy452); + yymsp[-11].minor.yy452 = addRangeClause(pCxt, yymsp[-11].minor.yy452, yymsp[-5].minor.yy452); + yymsp[-11].minor.yy452 = addEveryClause(pCxt, yymsp[-11].minor.yy452, yymsp[-4].minor.yy452); + yymsp[-11].minor.yy452 = addFillClause(pCxt, yymsp[-11].minor.yy452, yymsp[-3].minor.yy452); } -#line 6761 "sql.c" - break; - case 515: /* set_quantifier_opt ::= ALL */ -#line 1016 "sql.y" -{ yymsp[0].minor.yy441 = false; } -#line 6766 "sql.c" - break; - case 518: /* select_item ::= NK_STAR */ -#line 1023 "sql.y" -{ yylhsminor.yy164 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } -#line 6771 "sql.c" - yymsp[0].minor.yy164 = yylhsminor.yy164; - break; - case 520: /* select_item ::= common_expression column_alias */ - case 530: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==530); -#line 1025 "sql.y" -{ yylhsminor.yy164 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy164), &yymsp[0].minor.yy497); } -#line 6778 "sql.c" - yymsp[-1].minor.yy164 = yylhsminor.yy164; - break; - case 521: /* select_item ::= common_expression AS column_alias */ - case 531: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==531); -#line 1026 "sql.y" -{ yylhsminor.yy164 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), &yymsp[0].minor.yy497); } -#line 6785 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 526: /* partition_by_clause_opt ::= PARTITION BY partition_list */ - case 551: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==551); - case 570: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==570); -#line 1035 "sql.y" -{ yymsp[-2].minor.yy72 = yymsp[0].minor.yy72; } -#line 6793 "sql.c" - break; - case 533: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -#line 1048 "sql.y" -{ yymsp[-5].minor.yy164 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy164), releaseRawExprNode(pCxt, yymsp[-1].minor.yy164)); } -#line 6798 "sql.c" - break; - case 534: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ -#line 1049 "sql.y" -{ yymsp[-3].minor.yy164 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy164)); } -#line 6803 "sql.c" - break; - case 535: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -#line 1051 "sql.y" -{ yymsp[-5].minor.yy164 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy164), NULL, yymsp[-1].minor.yy164, yymsp[0].minor.yy164); } -#line 6808 "sql.c" - break; - case 536: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -#line 1054 "sql.y" -{ yymsp[-7].minor.yy164 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy164), releaseRawExprNode(pCxt, yymsp[-3].minor.yy164), yymsp[-1].minor.yy164, yymsp[0].minor.yy164); } -#line 6813 "sql.c" - break; - case 537: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ -#line 1056 "sql.y" -{ yymsp[-6].minor.yy164 = createEventWindowNode(pCxt, yymsp[-3].minor.yy164, yymsp[0].minor.yy164); } -#line 6818 "sql.c" - break; - case 541: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -#line 1062 "sql.y" -{ yymsp[-3].minor.yy164 = createFillNode(pCxt, yymsp[-1].minor.yy446, NULL); } -#line 6823 "sql.c" - break; - case 542: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ -#line 1063 "sql.y" -{ yymsp[-5].minor.yy164 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy72)); } -#line 6828 "sql.c" - break; - case 543: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ -#line 1064 "sql.y" -{ yymsp[-5].minor.yy164 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy72)); } -#line 6833 "sql.c" - break; - case 544: /* fill_mode ::= NONE */ -#line 1068 "sql.y" -{ yymsp[0].minor.yy446 = FILL_MODE_NONE; } -#line 6838 "sql.c" - break; - case 545: /* fill_mode ::= PREV */ -#line 1069 "sql.y" -{ yymsp[0].minor.yy446 = FILL_MODE_PREV; } -#line 6843 "sql.c" - break; - case 546: /* fill_mode ::= NULL */ -#line 1070 "sql.y" -{ yymsp[0].minor.yy446 = FILL_MODE_NULL; } -#line 6848 "sql.c" - break; - case 547: /* fill_mode ::= NULL_F */ -#line 1071 "sql.y" -{ yymsp[0].minor.yy446 = FILL_MODE_NULL_F; } -#line 6853 "sql.c" - break; - case 548: /* fill_mode ::= LINEAR */ -#line 1072 "sql.y" -{ yymsp[0].minor.yy446 = FILL_MODE_LINEAR; } -#line 6858 "sql.c" - break; - case 549: /* fill_mode ::= NEXT */ -#line 1073 "sql.y" -{ yymsp[0].minor.yy446 = FILL_MODE_NEXT; } -#line 6863 "sql.c" - break; - case 552: /* group_by_list ::= expr_or_subquery */ -#line 1082 "sql.y" -{ yylhsminor.yy72 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy164))); } -#line 6868 "sql.c" - yymsp[0].minor.yy72 = yylhsminor.yy72; - break; - case 553: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ -#line 1083 "sql.y" -{ yylhsminor.yy72 = addNodeToList(pCxt, yymsp[-2].minor.yy72, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy164))); } -#line 6874 "sql.c" - yymsp[-2].minor.yy72 = yylhsminor.yy72; - break; - case 557: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ -#line 1090 "sql.y" -{ yymsp[-5].minor.yy164 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy164), releaseRawExprNode(pCxt, yymsp[-1].minor.yy164)); } -#line 6880 "sql.c" - break; - case 560: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ -#line 1097 "sql.y" + break; + case 523: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy667 = false; } + break; + case 526: /* select_item ::= NK_STAR */ +{ yylhsminor.yy452 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 528: /* select_item ::= common_expression column_alias */ + case 538: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==538); +{ yylhsminor.yy452 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy452), &yymsp[0].minor.yy371); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 529: /* select_item ::= common_expression AS column_alias */ + case 539: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==539); +{ yylhsminor.yy452 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), &yymsp[0].minor.yy371); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 534: /* partition_by_clause_opt ::= PARTITION BY partition_list */ + case 559: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==559); + case 579: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==579); +{ yymsp[-2].minor.yy812 = yymsp[0].minor.yy812; } + break; + case 541: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy452 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), releaseRawExprNode(pCxt, yymsp[-1].minor.yy452)); } + break; + case 542: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ +{ yymsp[-3].minor.yy452 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy452)); } + break; + case 543: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy452 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), NULL, yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } + break; + case 544: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy452 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy452), releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } + break; + case 545: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ +{ yymsp[-6].minor.yy452 = createEventWindowNode(pCxt, yymsp[-3].minor.yy452, yymsp[0].minor.yy452); } + break; + case 549: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy452 = createFillNode(pCxt, yymsp[-1].minor.yy844, NULL); } + break; + case 550: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ +{ yymsp[-5].minor.yy452 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy812)); } + break; + case 551: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ +{ yymsp[-5].minor.yy452 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy812)); } + break; + case 552: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy844 = FILL_MODE_NONE; } + break; + case 553: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy844 = FILL_MODE_PREV; } + break; + case 554: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy844 = FILL_MODE_NULL; } + break; + case 555: /* fill_mode ::= NULL_F */ +{ yymsp[0].minor.yy844 = FILL_MODE_NULL_F; } + break; + case 556: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy844 = FILL_MODE_LINEAR; } + break; + case 557: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy844 = FILL_MODE_NEXT; } + break; + case 560: /* group_by_list ::= expr_or_subquery */ +{ yylhsminor.yy812 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } + yymsp[0].minor.yy812 = yylhsminor.yy812; + break; + case 561: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ +{ yylhsminor.yy812 = addNodeToList(pCxt, yymsp[-2].minor.yy812, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } + yymsp[-2].minor.yy812 = yylhsminor.yy812; + break; + case 565: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ +{ yymsp[-5].minor.yy452 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), releaseRawExprNode(pCxt, yymsp[-1].minor.yy452)); } + break; + case 566: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ +{ yymsp[-3].minor.yy452 = createInterpTimePoint(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy452)); } + break; + case 569: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy164 = addOrderByClause(pCxt, yymsp[-3].minor.yy164, yymsp[-2].minor.yy72); - yylhsminor.yy164 = addSlimitClause(pCxt, yylhsminor.yy164, yymsp[-1].minor.yy164); - yylhsminor.yy164 = addLimitClause(pCxt, yylhsminor.yy164, yymsp[0].minor.yy164); + yylhsminor.yy452 = addOrderByClause(pCxt, yymsp[-3].minor.yy452, yymsp[-2].minor.yy812); + yylhsminor.yy452 = addSlimitClause(pCxt, yylhsminor.yy452, yymsp[-1].minor.yy452); + yylhsminor.yy452 = addLimitClause(pCxt, yylhsminor.yy452, yymsp[0].minor.yy452); } -#line 6889 "sql.c" - yymsp[-3].minor.yy164 = yylhsminor.yy164; - break; - case 563: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ -#line 1107 "sql.y" -{ yylhsminor.yy164 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy164, yymsp[0].minor.yy164); } -#line 6895 "sql.c" - yymsp[-3].minor.yy164 = yylhsminor.yy164; - break; - case 564: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ -#line 1109 "sql.y" -{ yylhsminor.yy164 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy164, yymsp[0].minor.yy164); } -#line 6901 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 572: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 576: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==576); -#line 1123 "sql.y" -{ yymsp[-1].minor.yy164 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } -#line 6908 "sql.c" - break; - case 573: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 577: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==577); -#line 1124 "sql.y" -{ yymsp[-3].minor.yy164 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } -#line 6914 "sql.c" - break; - case 574: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 578: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==578); -#line 1125 "sql.y" -{ yymsp[-3].minor.yy164 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } -#line 6920 "sql.c" - break; - case 579: /* subquery ::= NK_LP query_expression NK_RP */ -#line 1133 "sql.y" -{ yylhsminor.yy164 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy164); } -#line 6925 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 584: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ -#line 1147 "sql.y" -{ yylhsminor.yy164 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy164), yymsp[-1].minor.yy550, yymsp[0].minor.yy517); } -#line 6931 "sql.c" - yymsp[-2].minor.yy164 = yylhsminor.yy164; - break; - case 585: /* ordering_specification_opt ::= */ -#line 1151 "sql.y" -{ yymsp[1].minor.yy550 = ORDER_ASC; } -#line 6937 "sql.c" - break; - case 586: /* ordering_specification_opt ::= ASC */ -#line 1152 "sql.y" -{ yymsp[0].minor.yy550 = ORDER_ASC; } -#line 6942 "sql.c" - break; - case 587: /* ordering_specification_opt ::= DESC */ -#line 1153 "sql.y" -{ yymsp[0].minor.yy550 = ORDER_DESC; } -#line 6947 "sql.c" - break; - case 588: /* null_ordering_opt ::= */ -#line 1157 "sql.y" -{ yymsp[1].minor.yy517 = NULL_ORDER_DEFAULT; } -#line 6952 "sql.c" - break; - case 589: /* null_ordering_opt ::= NULLS FIRST */ -#line 1158 "sql.y" -{ yymsp[-1].minor.yy517 = NULL_ORDER_FIRST; } -#line 6957 "sql.c" - break; - case 590: /* null_ordering_opt ::= NULLS LAST */ -#line 1159 "sql.y" -{ yymsp[-1].minor.yy517 = NULL_ORDER_LAST; } -#line 6962 "sql.c" + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 572: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ +{ yylhsminor.yy452 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy452, yymsp[0].minor.yy452); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 573: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ +{ yylhsminor.yy452 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy452, yymsp[0].minor.yy452); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 581: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 585: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==585); +{ yymsp[-1].minor.yy452 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + break; + case 582: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 586: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==586); +{ yymsp[-3].minor.yy452 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + break; + case 583: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 587: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==587); +{ yymsp[-3].minor.yy452 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + break; + case 588: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy452); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 593: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy452 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), yymsp[-1].minor.yy690, yymsp[0].minor.yy399); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 594: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy690 = ORDER_ASC; } + break; + case 595: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy690 = ORDER_ASC; } + break; + case 596: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy690 = ORDER_DESC; } + break; + case 597: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy399 = NULL_ORDER_DEFAULT; } + break; + case 598: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy399 = NULL_ORDER_FIRST; } + break; + case 599: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy399 = NULL_ORDER_LAST; } break; default: break; @@ -7020,7 +6233,6 @@ static void yy_syntax_error( ParseCTX_FETCH #define TOKEN yyminor /************ Begin %syntax_error code ****************************************/ -#line 29 "sql.y" if (TSDB_CODE_SUCCESS == pCxt->errCode) { if(TOKEN.z) { @@ -7031,7 +6243,6 @@ static void yy_syntax_error( } else if (TSDB_CODE_PAR_DB_NOT_SPECIFIED == pCxt->errCode && TK_NK_FLOAT == TOKEN.type) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); } -#line 7035 "sql.c" /************ End %syntax_error code ******************************************/ ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ ParseCTX_STORE diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 6a08193a39a1e1da7f321ae111445e5d714ac1ac..856fdb48045a95e0b36a857e523b9bf86023d6d0 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -542,6 +542,18 @@ TEST_F(ParserInitialCTest, createSmaIndex) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_INDEX_STMT); SMCreateSmaReq req = {0}; + ASSERT_TRUE(pQuery->pPrevRoot); + ASSERT_EQ(QUERY_NODE_SELECT_STMT, nodeType(pQuery->pPrevRoot)); + + SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pQuery->pRoot; + SCmdMsgInfo* pCmdMsg = (SCmdMsgInfo*)taosMemoryMalloc(sizeof(SCmdMsgInfo)); + if (NULL == pCmdMsg) FAIL(); + pCmdMsg->msgType = TDMT_MND_CREATE_SMA; + pCmdMsg->msgLen = tSerializeSMCreateSmaReq(NULL, 0, pStmt->pReq); + pCmdMsg->pMsg = taosMemoryMalloc(pCmdMsg->msgLen); + if (!pCmdMsg->pMsg) FAIL(); + tSerializeSMCreateSmaReq(pCmdMsg->pMsg, pCmdMsg->msgLen, pStmt->pReq); + ((SQuery*)pQuery)->pCmdMsg = pCmdMsg; ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); ASSERT_EQ(std::string(req.name), std::string(expect.name)); @@ -765,7 +777,7 @@ TEST_F(ParserInitialCTest, createStable) { "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, " "a8 BINARY(20), a9 SMALLINT, a10 SMALLINT UNSIGNED, a11 TINYINT, " "a12 TINYINT UNSIGNED, a13 BOOL, a14 NCHAR(30), a15 VARCHAR(50)) " - "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) MAX_DELAY 100s,10m WATERMARK 10a,1m " + "COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) MAX_DELAY 100s,10m WATERMARK 10a,1m " "DELETE_MARK 1000s,200m"); clearCreateStbReq(); } @@ -885,12 +897,12 @@ TEST_F(ParserInitialCTest, createStream) { setCreateStreamReq( "s1", "test", - "create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 fill_history 1 ignore " + "create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 fill_history 0 ignore " "update 1 into st3 as select count(*) from t1 interval(10s)", "st3", 1); setStreamOptions(STREAM_CREATE_STABLE_TRUE, STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, - 10 * MILLISECOND_PER_SECOND, 0, 1, 1); - run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 FILL_HISTORY 1 IGNORE " + 10 * MILLISECOND_PER_SECOND, 0, 0, 1); + run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 FILL_HISTORY 0 IGNORE " "UPDATE 1 INTO st3 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)"); clearCreateStreamReq(); @@ -1005,7 +1017,7 @@ TEST_F(ParserInitialCTest, createTable) { ASSERT_EQ(pReq->flags, pExpect->flags); ASSERT_EQ(std::string(pReq->name), std::string(pExpect->name)); ASSERT_EQ(pReq->uid, pExpect->uid); - ASSERT_EQ(pReq->ctime, pExpect->ctime); + ASSERT_EQ(pReq->btime, pExpect->btime); ASSERT_EQ(pReq->ttl, pExpect->ttl); ASSERT_EQ(pReq->commentLen, pExpect->commentLen); ASSERT_EQ(std::string(pReq->comment), std::string(pExpect->comment)); @@ -1038,7 +1050,7 @@ TEST_F(ParserInitialCTest, createTable) { "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, a8 BINARY(20), " "a9 SMALLINT, a10 SMALLINT UNSIGNED, a11 TINYINT, a12 TINYINT UNSIGNED, a13 BOOL, " "a14 NCHAR(30), a15 VARCHAR(50)) " - "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN)"); + "COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN)"); run("CREATE TABLE IF NOT EXISTS t1 USING st1 TAGS(1, 'wxy', NOW)"); @@ -1145,6 +1157,15 @@ TEST_F(ParserInitialCTest, createTopic) { setCreateTopicReq("tp1", 1, "create topic if not exists tp1 with meta as stable st1", nullptr, "test", "st1", 1); run("CREATE TOPIC IF NOT EXISTS tp1 WITH META AS STABLE st1"); clearCreateTopicReq(); + + setCreateTopicReq("tp1", 1, "create topic if not exists tp1 as stable st1 where tag1 > 0", nullptr, "test", "st1"); + run("CREATE TOPIC IF NOT EXISTS tp1 AS STABLE st1 WHERE tag1 > 0"); + clearCreateTopicReq(); + + setCreateTopicReq("tp1", 1, "create topic if not exists tp1 with meta as stable st1 where tag1 > 0", nullptr, "test", "st1", 1); + run("CREATE TOPIC IF NOT EXISTS tp1 WITH META AS STABLE st1 WHERE tag1 > 0"); + clearCreateTopicReq(); + } /* diff --git a/source/libs/parser/test/parInitialDTest.cpp b/source/libs/parser/test/parInitialDTest.cpp index 203156ec883427f943c847a24ea2ac734c645f2a..937f76176eea335272dd2940267e85a1c79b7ec8 100644 --- a/source/libs/parser/test/parInitialDTest.cpp +++ b/source/libs/parser/test/parInitialDTest.cpp @@ -94,15 +94,17 @@ TEST_F(ParserInitialDTest, dropDnode) { auto clearDropDnodeReq = [&]() { memset(&expect, 0, sizeof(SDropDnodeReq)); }; - auto setDropDnodeReqById = [&](int32_t dnodeId, bool force = false) { + auto setDropDnodeReqById = [&](int32_t dnodeId, bool force = false, bool unsafe = false) { expect.dnodeId = dnodeId; expect.force = force; + expect.unsafe = unsafe; }; - auto setDropDnodeReqByEndpoint = [&](const char* pFqdn, int32_t port = tsServerPort, bool force = false) { + auto setDropDnodeReqByEndpoint = [&](const char* pFqdn, int32_t port = tsServerPort, bool force = false, bool unsafe = false) { strcpy(expect.fqdn, pFqdn); expect.port = port; expect.force = force; + expect.unsafe = unsafe; }; setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { @@ -114,6 +116,7 @@ TEST_F(ParserInitialDTest, dropDnode) { ASSERT_EQ(std::string(req.fqdn), std::string(expect.fqdn)); ASSERT_EQ(req.port, expect.port); ASSERT_EQ(req.force, expect.force); + ASSERT_EQ(req.unsafe, expect.unsafe); }); setDropDnodeReqById(1); @@ -124,6 +127,10 @@ TEST_F(ParserInitialDTest, dropDnode) { run("DROP DNODE 2 FORCE"); clearDropDnodeReq(); + setDropDnodeReqById(2, false, true); + run("DROP DNODE 2 UNSAFE"); + clearDropDnodeReq(); + setDropDnodeReqByEndpoint("host1", 7030); run("DROP DNODE 'host1:7030'"); clearDropDnodeReq(); @@ -132,6 +139,10 @@ TEST_F(ParserInitialDTest, dropDnode) { run("DROP DNODE 'host2:8030' FORCE"); clearDropDnodeReq(); + setDropDnodeReqByEndpoint("host2", 8030, false, true); + run("DROP DNODE 'host2:8030' UNSAFE"); + clearDropDnodeReq(); + setDropDnodeReqByEndpoint("host1"); run("DROP DNODE host1"); clearDropDnodeReq(); @@ -139,6 +150,10 @@ TEST_F(ParserInitialDTest, dropDnode) { setDropDnodeReqByEndpoint("host2", tsServerPort, true); run("DROP DNODE host2 FORCE"); clearDropDnodeReq(); + + setDropDnodeReqByEndpoint("host2", tsServerPort, false, true); + run("DROP DNODE host2 UNSAFE"); + clearDropDnodeReq(); } // todo DROP function @@ -276,4 +291,13 @@ TEST_F(ParserInitialDTest, dropUser) { run("DROP USER wxy"); } +TEST_F(ParserInitialDTest, IntervalOnSysTable) { + login("root"); + run("SELECT count('reboot_time') FROM information_schema.ins_dnodes interval(14m) sliding(9m)", + TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED, PARSER_STAGE_TRANSLATE); + + run("SELECT count('create_time') FROM information_schema.ins_qnodes interval(14m) sliding(9m)", + TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED, PARSER_STAGE_TRANSLATE); +} + } // namespace ParserTest diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 2d8ce55b72152a474f142634875f2c3e8201729b..68ded3afddd627c37091e30af551a1a826c27bc3 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -117,6 +117,15 @@ TEST_F(ParserSelectTest, timelineFunc) { run("SELECT LAST(*), FIRST(*) FROM t1 INTERVAL(10s)"); run("SELECT diff(c1) FROM t1"); + + run("select diff(ts) from (select _wstart as ts, count(*) from st1 partition by tbname interval(1d))", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); + + run("select diff(ts) from (select _wstart as ts, count(*) from st1 partition by tbname interval(1d) order by ts)"); + + run("select t1.* from st1s1 t1, (select _wstart as ts, count(*) from st1s2 partition by tbname interval(1d)) WHERE t1.ts = t2.ts", TSDB_CODE_PAR_NOT_SUPPORT_JOIN); + + run("select t1.* from st1s1 t1, (select _wstart as ts, count(*) from st1s2 partition by tbname interval(1d) order by ts) t2 WHERE t1.ts = t2.ts"); + } TEST_F(ParserSelectTest, selectFunc) { @@ -325,6 +334,10 @@ TEST_F(ParserSelectTest, subquery) { run("SELECT SUM(a) FROM (SELECT MAX(c1) a, _wstart FROM st1s1 PARTITION BY TBNAME INTERVAL(1m) ORDER BY _WSTART) " "INTERVAL(1n)"); + run("SELECT diff(a) FROM (SELECT _wstart, tag1, tag2, MAX(c1) a FROM st1 PARTITION BY tag1 INTERVAL(1m)) PARTITION BY tag1"); + + run("SELECT diff(a) FROM (SELECT _wstart, tag1, tag2, MAX(c1) a FROM st1 PARTITION BY tag1 INTERVAL(1m)) PARTITION BY tag2", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); + run("SELECT _C0 FROM (SELECT _ROWTS, ts FROM st1s1)"); run("SELECT ts FROM (SELECT t1.ts FROM st1s1 t1)"); @@ -472,6 +485,8 @@ TEST_F(ParserSelectTest, joinSemanticCheck) { run("SELECT * FROM (SELECT tag1, SUM(c1) s FROM st1 GROUP BY tag1) t1, st1 t2 where t1.tag1 = t2.tag1", TSDB_CODE_PAR_NOT_SUPPORT_JOIN); + + run("SELECT count(*) FROM t1 a join t1 b on a.ts=b.ts where a.ts=b.ts"); } } // namespace ParserTest diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index 82abc5d1a973dbafa820c41e0614e1b2c90a92c3..092fe1741187dcf3706a64a7be64a3032835b44a 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -43,6 +43,9 @@ int32_t splitLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan); int32_t scaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQueryLogicPlan** pLogicPlan); int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList); +bool isPartTableAgg(SAggLogicNode* pAgg); +bool isPartTableWinodw(SWindowLogicNode* pWindow); + #ifdef __cplusplus } #endif diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 66b478004f9dfbc3040047b7ea5d36a37bb95b95..9345834ec0f3989946b539a5b098148e5635f970 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -37,19 +37,24 @@ typedef struct SRewriteExprCxt { int32_t errCode; SNodeList* pExprs; bool* pOutputs; + bool isPartitionBy; } SRewriteExprCxt; -static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol) { +static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol, bool isPartitionBy) { switch (pFunc->funcType) { case FUNCTION_TYPE_TBNAME: pCol->colType = COLUMN_TYPE_TBNAME; break; case FUNCTION_TYPE_WSTART: - pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + if (!isPartitionBy) { + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } pCol->colType = COLUMN_TYPE_WINDOW_START; break; case FUNCTION_TYPE_WEND: - pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + if (!isPartitionBy) { + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } pCol->colType = COLUMN_TYPE_WINDOW_END; break; case FUNCTION_TYPE_WDURATION: @@ -100,9 +105,10 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); pCol->node.resType = pToBeRewrittenExpr->resType; strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); + strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias); strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { - setColumnInfo((SFunctionNode*)pExpr, pCol); + setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy); } nodesDestroyNode(*pNode); *pNode = (SNode*)pCol; @@ -141,7 +147,8 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClause clause) { nodesWalkExpr(pExpr, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL}; + bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL, .isPartitionBy = isPartitionBy}; cxt.errCode = nodesListMakeAppend(&cxt.pExprs, pExpr); if (TSDB_CODE_SUCCESS == cxt.errCode) { nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); @@ -169,7 +176,8 @@ static int32_t cloneRewriteExprs(SNodeList* pExprs, bool* pOutputs, SNodeList** static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause, SNodeList** pRewriteExprs) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = isPartitionBy}; if (NULL != pRewriteExprs) { cxt.pOutputs = taosMemoryCalloc(LIST_LENGTH(pExprs), sizeof(bool)); if (NULL == cxt.pOutputs) { @@ -186,14 +194,14 @@ static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ES static int32_t rewriteExpr(SNodeList* pExprs, SNode** pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = false}; nodesRewriteExpr(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = false}; nodesRewriteExprs(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } @@ -428,7 +436,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pJoin->joinType = pJoinTable->joinType; pJoin->isSingleTableJoin = pJoinTable->table.singleTable; - pJoin->inputTsOrder = ORDER_ASC; + pJoin->node.inputTsOrder = ORDER_ASC; pJoin->node.groupAction = GROUP_ACTION_CLEAR; pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; pJoin->node.resultDataOrder = DATA_ORDER_LEVEL_GLOBAL; @@ -543,11 +551,16 @@ static SNode* createGroupingSetNode(SNode* pExpr) { return (SNode*)pGroupingSet; } -static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { +static EGroupAction getDistinctGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; } +static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { + return ((pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) && !pSelect->isDistinct) ? GROUP_ACTION_KEEP + : GROUP_ACTION_NONE; +} + static EDataOrderLevel getRequireDataOrder(bool needTimeline, SSelectStmt* pSelect) { return needTimeline ? (NULL != pSelect->pPartitionByList ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL) : DATA_ORDER_LEVEL_NONE; @@ -728,8 +741,8 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm pWindow->igExpired = pCxt->pPlanCxt->igExpired; pWindow->igCheckUpdate = pCxt->pPlanCxt->igCheckUpdate; } - pWindow->inputTsOrder = ORDER_ASC; - pWindow->outputTsOrder = ORDER_ASC; + pWindow->node.inputTsOrder = ORDER_ASC; + pWindow->node.outputTsOrder = ORDER_ASC; int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_WINDOW, fmIsWindowClauseFunc, &pWindow->pFuncs); if (TSDB_CODE_SUCCESS == code) { @@ -834,7 +847,6 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva : (pSelect->hasTimeLineFunc ? getRequireDataOrder(true, pSelect) : DATA_ORDER_LEVEL_IN_BLOCK); pWindow->node.resultDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : getRequireDataOrder(true, pSelect); - pWindow->pTspk = nodesCloneNode(pInterval->pCol); if (NULL == pWindow->pTspk) { nodesDestroyNode((SNode*)pWindow); @@ -959,7 +971,7 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pFill->node.groupAction = getGroupAction(pCxt, pSelect); pFill->node.requireDataOrder = getRequireDataOrder(true, pSelect); pFill->node.resultDataOrder = pFill->node.requireDataOrder; - pFill->inputTsOrder = ORDER_ASC; + pFill->node.inputTsOrder = 0; int32_t code = partFillExprs(pSelect, &pFill->pFillExprs, &pFill->pNotFillExprs); if (TSDB_CODE_SUCCESS == code) { @@ -1020,7 +1032,6 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pSort->node.resultDataOrder = isPrimaryKeySort(pSelect->pOrderByList) ? (pSort->groupSort ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL) : DATA_ORDER_LEVEL_NONE; - int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets); if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) { code = nodesListMakeStrictAppend(&pSort->node.pTargets, @@ -1032,6 +1043,20 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect if (NULL == pSort->pSortKeys) { code = TSDB_CODE_OUT_OF_MEMORY; } + SNode* pNode = NULL; + SOrderByExprNode* firstSortKey = (SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0); + if (isPrimaryKeySort(pSelect->pOrderByList)) pSort->node.outputTsOrder = firstSortKey->order; + if (firstSortKey->pExpr->type == QUERY_NODE_COLUMN) { + SColumnNode* pCol = (SColumnNode*)firstSortKey->pExpr; + int16_t projIdx = 1; + FOREACH(pNode, pSelect->pProjectionList) { + SExprNode* pExpr = (SExprNode*)pNode; + if (0 == strcmp(pCol->node.aliasName, pExpr->aliasName)) { + pCol->projIdx = projIdx; break; + } + projIdx++; + } + } } if (TSDB_CODE_SUCCESS == code) { @@ -1166,7 +1191,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return TSDB_CODE_OUT_OF_MEMORY; } - pAgg->node.groupAction = GROUP_ACTION_CLEAR; + pAgg->node.groupAction = GROUP_ACTION_CLEAR;//getDistinctGroupAction(pCxt, pSelect); pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 72e5081ab9c12abb7d71ff433c3abf17ce07c4fa..16440be5115c877a2cceb53df00571aae8498ab2 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -116,25 +116,33 @@ static EDealRes optRebuildTbanme(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static void optSetParentOrder(SLogicNode* pNode, EOrder order) { +static void optSetParentOrder(SLogicNode* pNode, EOrder order, SLogicNode* pNodeForcePropagate) { if (NULL == pNode) { return; } + pNode->inputTsOrder = order; switch (nodeType(pNode)) { - case QUERY_NODE_LOGIC_PLAN_WINDOW: - ((SWindowLogicNode*)pNode)->inputTsOrder = order; - // window has a sorting function, and the operator behind it uses its output order - return; + // for those nodes that will change the order, stop propagating + // case QUERY_NODE_LOGIC_PLAN_WINDOW: case QUERY_NODE_LOGIC_PLAN_JOIN: - ((SJoinLogicNode*)pNode)->inputTsOrder = order; - break; - case QUERY_NODE_LOGIC_PLAN_FILL: - ((SFillLogicNode*)pNode)->inputTsOrder = order; + case QUERY_NODE_LOGIC_PLAN_AGG: + case QUERY_NODE_LOGIC_PLAN_SORT: + if (pNode == pNodeForcePropagate) { + pNode->outputTsOrder = order; + break; + } else + return; + case QUERY_NODE_LOGIC_PLAN_WINDOW: + // Window output ts order default to be asc, and changed when doing sort by primary key optimization. + // We stop propagate the original order to parents. + // Use window output ts order instead. + order = pNode->outputTsOrder; break; default: + pNode->outputTsOrder = order; break; } - optSetParentOrder(pNode->pParent, order); + optSetParentOrder(pNode->pParent, order, pNodeForcePropagate); } EDealRes scanPathOptHaveNormalColImpl(SNode* pNode, void* pContext) { @@ -159,25 +167,25 @@ static bool scanPathOptMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) { return false; } - if (NULL == pNode->pParent || (QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) && - QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) && - QUERY_NODE_LOGIC_PLAN_PARTITION != nodeType(pNode->pParent))) { - return false; - } - if ((QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent) && - WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType) || - (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode->pParent) && pNode->pParent->pParent && - QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent->pParent) && + return true; +} + +static bool scanPathOptShouldGetFuncs(SLogicNode* pNode) { + if ((QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode) && + WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode)->winType) || + (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode) && pNode->pParent && + QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent) && WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType)) { return true; } - if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode->pParent)) { - return !scanPathOptHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); + if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode)) { + return !scanPathOptHaveNormalCol(((SAggLogicNode*)pNode)->pGroupKeys); } return false; } static SNodeList* scanPathOptGetAllFuncs(SLogicNode* pNode) { + if (!scanPathOptShouldGetFuncs(pNode)) return NULL; switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_WINDOW: return ((SWindowLogicNode*)pNode)->pFuncs; @@ -339,12 +347,12 @@ static void scanPathOptSetScanOrder(EScanOrder scanOrder, SScanLogicNode* pScan) case SCAN_ORDER_ASC: pScan->scanSeq[0] = 1; pScan->scanSeq[1] = 0; - optSetParentOrder(pScan->node.pParent, ORDER_ASC); + optSetParentOrder(pScan->node.pParent, ORDER_ASC, NULL); break; case SCAN_ORDER_DESC: pScan->scanSeq[0] = 0; pScan->scanSeq[1] = 1; - optSetParentOrder(pScan->node.pParent, ORDER_DESC); + optSetParentOrder(pScan->node.pParent, ORDER_DESC, NULL); break; case SCAN_ORDER_BOTH: pScan->scanSeq[0] = 1; @@ -355,6 +363,18 @@ static void scanPathOptSetScanOrder(EScanOrder scanOrder, SScanLogicNode* pScan) } } +static void scanPathOptSetGroupOrderScan(SScanLogicNode* pScan) { + if (pScan->tableType != TSDB_SUPER_TABLE) return; + + if (pScan->node.pParent && nodeType(pScan->node.pParent) == QUERY_NODE_LOGIC_PLAN_AGG) { + SAggLogicNode* pAgg = (SAggLogicNode*)pScan->node.pParent; + bool withSlimit = pAgg->node.pSlimit != NULL || (pAgg->node.pParent && pAgg->node.pParent->pSlimit); + if (withSlimit && isPartTableAgg(pAgg)) { + pScan->groupOrderScan = pAgg->node.forceCreateNonBlockingOptr = true; + } + } +} + static int32_t scanPathOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SOsdInfo info = {.scanOrder = SCAN_ORDER_ASC}; int32_t code = scanPathOptMatch(pCxt, pLogicSubplan->pNode, &info); @@ -363,6 +383,7 @@ static int32_t scanPathOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub if (!pCxt->pPlanCxt->streamQuery) { scanPathOptSetScanOrder(info.scanOrder, info.pScan); } + scanPathOptSetGroupOrderScan(info.pScan); } if (TSDB_CODE_SUCCESS == code && (NULL != info.pDsoFuncs || NULL != info.pSdrFuncs)) { info.pScan->dataRequired = scanPathOptGetDataRequired(info.pSdrFuncs); @@ -740,6 +761,86 @@ static int32_t pushDownCondOptJoinExtractMergeCond(SOptimizeContext* pCxt, SJoin return code; } +static bool pushDownCondOptIsTableColumn(SNode* pNode, SNodeList* pTableCols) { + if (QUERY_NODE_COLUMN != nodeType(pNode)) { + return false; + } + SColumnNode* pCol = (SColumnNode*)pNode; + return pushDownCondOptBelongThisTable(pNode, pTableCols); +} + +static bool pushDownCondOptIsColEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { + if (QUERY_NODE_OPERATOR != nodeType(pCond)) { + return false; + } + SOperatorNode* pOper = (SOperatorNode*)pCond; + if (OP_TYPE_EQUAL != pOper->opType) { + return false; + } + if (QUERY_NODE_COLUMN != nodeType(pOper->pLeft) || QUERY_NODE_COLUMN != nodeType(pOper->pRight)) { + return false; + } + SColumnNode* pLeft = (SColumnNode*)(pOper->pLeft); + SColumnNode* pRight = (SColumnNode*)(pOper->pRight); + // TODO: add cast to operator and remove this restriction of optimization + if (pLeft->node.resType.type != pRight->node.resType.type || + pLeft->node.resType.bytes != pRight->node.resType.bytes) { + return false; + } + SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; + SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; + if (pushDownCondOptIsTableColumn(pOper->pLeft, pLeftCols)) { + return pushDownCondOptIsTableColumn(pOper->pRight, pRightCols); + } else if (pushDownCondOptIsTableColumn(pOper->pLeft, pRightCols)) { + return pushDownCondOptIsTableColumn(pOper->pRight, pLeftCols); + } + return false; +} + +static int32_t pushDownCondOptJoinExtractColEqualOnLogicCond(SJoinLogicNode* pJoin) { + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); + + int32_t code = TSDB_CODE_SUCCESS; + SNodeList* pEqualOnConds = NULL; + SNode* pCond = NULL; + FOREACH(pCond, pLogicCond->pParameterList) { + if (pushDownCondOptIsColEqualOnCond(pJoin, pCond)) { + code = nodesListMakeAppend(&pEqualOnConds, nodesCloneNode(pCond)); + } + } + + SNode* pTempTagEqCond = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = nodesMergeConds(&pTempTagEqCond, &pEqualOnConds); + } + + if (TSDB_CODE_SUCCESS == code) { + pJoin->pColEqualOnConditions = pTempTagEqCond; + return TSDB_CODE_SUCCESS; + } else { + nodesDestroyList(pEqualOnConds); + return TSDB_CODE_PLAN_INTERNAL_ERROR; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t pushDownCondOptJoinExtractColEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { + if (NULL == pJoin->pOnConditions) { + pJoin->pColEqualOnConditions = NULL; + return TSDB_CODE_SUCCESS; + } + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && + LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { + return pushDownCondOptJoinExtractColEqualOnLogicCond(pJoin); + } + + if (pushDownCondOptIsColEqualOnCond(pJoin, pJoin->pOnConditions)) { + pJoin->pColEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); + } + + return TSDB_CODE_SUCCESS; +} + static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) { return TSDB_CODE_SUCCESS; @@ -774,6 +875,10 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p code = pushDownCondOptJoinExtractMergeCond(pCxt, pJoin); } + if (TSDB_CODE_SUCCESS == code) { + code = pushDownCondOptJoinExtractColEqualOnCond(pCxt, pJoin); + } + if (TSDB_CODE_SUCCESS == code) { OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE); pCxt->optimized = true; @@ -1076,7 +1181,8 @@ static bool sortPriKeyOptMayBeOptimized(SLogicNode* pNode) { return false; } SSortLogicNode* pSort = (SSortLogicNode*)pNode; - if (!sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) { + if (pSort->skipPKSortOpt || !sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys) || + 1 != LIST_LENGTH(pSort->node.pChildren)) { return false; } SNode* pChild; @@ -1089,8 +1195,8 @@ static bool sortPriKeyOptMayBeOptimized(SLogicNode* pNode) { return true; } -static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool groupSort, bool* pNotOptimize, - SNodeList** pSequencingNodes) { +static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool groupSort, EOrder sortOrder, + bool* pNotOptimize, SNodeList** pSequencingNodes) { if (NULL != pNode->pLimit || NULL != pNode->pSlimit) { *pNotOptimize = false; return TSDB_CODE_SUCCESS; @@ -1107,15 +1213,21 @@ static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool group } case QUERY_NODE_LOGIC_PLAN_JOIN: { int32_t code = sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), groupSort, - pNotOptimize, pSequencingNodes); + sortOrder, pNotOptimize, pSequencingNodes); if (TSDB_CODE_SUCCESS == code) { code = sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), groupSort, - pNotOptimize, pSequencingNodes); + sortOrder, pNotOptimize, pSequencingNodes); } return code; } - case QUERY_NODE_LOGIC_PLAN_WINDOW: - return nodesListMakeAppend(pSequencingNodes, (SNode*)pNode); + case QUERY_NODE_LOGIC_PLAN_WINDOW: { + SWindowLogicNode* pWindowLogicNode = (SWindowLogicNode*)pNode; + // For interval window, we always apply sortPriKey optimization. + // For session/event/state window, the output ts order will always be ASC. + // If sort order is also asc, we apply optimization, otherwise we keep sort node to get correct output order. + if (pWindowLogicNode->winType == WINDOW_TYPE_INTERVAL || sortOrder == ORDER_ASC) + return nodesListMakeAppend(pSequencingNodes, (SNode*)pNode); + } case QUERY_NODE_LOGIC_PLAN_AGG: case QUERY_NODE_LOGIC_PLAN_PARTITION: *pNotOptimize = true; @@ -1129,23 +1241,25 @@ static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool group return TSDB_CODE_SUCCESS; } - return sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), groupSort, + return sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), groupSort, sortOrder, pNotOptimize, pSequencingNodes); } -static int32_t sortPriKeyOptGetSequencingNodes(SLogicNode* pNode, bool groupSort, SNodeList** pSequencingNodes) { +static EOrder sortPriKeyOptGetPriKeyOrder(SSortLogicNode* pSort) { + return ((SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0))->order; +} + +static int32_t sortPriKeyOptGetSequencingNodes(SSortLogicNode* pSort, bool groupSort, SNodeList** pSequencingNodes) { bool notOptimize = false; - int32_t code = sortPriKeyOptGetSequencingNodesImpl(pNode, groupSort, ¬Optimize, pSequencingNodes); + int32_t code = + sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), groupSort, + sortPriKeyOptGetPriKeyOrder(pSort), ¬Optimize, pSequencingNodes); if (TSDB_CODE_SUCCESS != code || notOptimize) { NODES_CLEAR_LIST(*pSequencingNodes); } return code; } -static EOrder sortPriKeyOptGetPriKeyOrder(SSortLogicNode* pSort) { - return ((SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0))->order; -} - static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort, SNodeList* pSequencingNodes) { EOrder order = sortPriKeyOptGetPriKeyOrder(pSort); @@ -1156,6 +1270,7 @@ static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicS if ((ORDER_DESC == order && pScan->scanSeq[0] > 0) || (ORDER_ASC == order && pScan->scanSeq[1] > 0)) { TSWAP(pScan->scanSeq[0], pScan->scanSeq[1]); } + pScan->node.outputTsOrder = order; if (TSDB_SUPER_TABLE == pScan->tableType) { pScan->scanType = SCAN_TYPE_TABLE_MERGE; pScan->node.resultDataOrder = DATA_ORDER_LEVEL_GLOBAL; @@ -1163,9 +1278,9 @@ static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicS } pScan->sortPrimaryKey = true; } else if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pSequencingNode)) { - ((SWindowLogicNode*)pSequencingNode)->outputTsOrder = order; + ((SLogicNode*)pSequencingNode)->outputTsOrder = order; } - optSetParentOrder(((SLogicNode*)pSequencingNode)->pParent, order); + optSetParentOrder(((SLogicNode*)pSequencingNode)->pParent, order, (SLogicNode*)pSort); } SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0); @@ -1183,10 +1298,17 @@ static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicS static int32_t sortPrimaryKeyOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort) { SNodeList* pSequencingNodes = NULL; - int32_t code = sortPriKeyOptGetSequencingNodes((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), - pSort->groupSort, &pSequencingNodes); - if (TSDB_CODE_SUCCESS == code && NULL != pSequencingNodes) { - code = sortPriKeyOptApply(pCxt, pLogicSubplan, pSort, pSequencingNodes); + int32_t code = sortPriKeyOptGetSequencingNodes(pSort, pSort->groupSort, &pSequencingNodes); + if (TSDB_CODE_SUCCESS == code) { + if (pSequencingNodes != NULL) { + code = sortPriKeyOptApply(pCxt, pLogicSubplan, pSort, pSequencingNodes); + } else { + // if we decided not to push down sort info to children, we should propagate output ts order to parents of pSort + optSetParentOrder(pSort->node.pParent, sortPriKeyOptGetPriKeyOrder(pSort), 0); + // we need to prevent this pSort from being chosen to do optimization again + pSort->skipPKSortOpt = true; + pCxt->optimized = true; + } } nodesClearList(pSequencingNodes); return code; @@ -1259,8 +1381,8 @@ static bool smaIndexOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pW .sliding = pIndex->sliding, .slidingUnit = pIndex->slidingUnit, .precision = pScan->node.precision}; - return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval, pScan->node.precision)) && - (pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval, pScan->node.precision)); + return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval)) && + (pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval)); } return true; } @@ -1292,6 +1414,16 @@ static int32_t smaIndexOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { return -1; } +static SNode* smaIndexOptFindWStartFunc(SNodeList* pSmaFuncs) { + SNode* pSmaFunc = NULL; + FOREACH(pSmaFunc, pSmaFuncs) { + if (QUERY_NODE_FUNCTION == nodeType(pSmaFunc) && FUNCTION_TYPE_WSTART == ((SFunctionNode*)pSmaFunc)->funcType) { + return pSmaFunc; + } + } + return NULL; +} + static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput) { SNodeList* pCols = NULL; @@ -1299,6 +1431,7 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo int32_t code = TSDB_CODE_SUCCESS; int32_t index = 0; int32_t smaFuncIndex = -1; + bool hasWStart = false; FOREACH(pFunc, pFuncs) { smaFuncIndex = smaIndexOptFindSmaFunc(pFunc, pSmaFuncs); if (smaFuncIndex < 0) { @@ -1308,11 +1441,35 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo if (TSDB_CODE_SUCCESS != code) { break; } + if (!hasWStart) { + if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pCols->pTail->pNode)->colId) { + hasWStart = true; + } + } } ++index; } if (TSDB_CODE_SUCCESS == code && smaFuncIndex >= 0) { + if (!hasWStart) { + SNode* pWsNode = smaIndexOptFindWStartFunc(pSmaFuncs); + if (!pWsNode) { + nodesDestroyList(pCols); + code = TSDB_CODE_APP_ERROR; + qError("create sma cols failed since %s(_wstart not exist)", tstrerror(code)); + return code; + } + SExprNode exprNode; + exprNode.resType = ((SExprNode*)pWsNode)->resType; + sprintf(exprNode.aliasName, "#expr_%d", index + 1); + SNode* pkNode = smaIndexOptCreateSmaCol((SNode*)&exprNode, tableId, PRIMARYKEY_TIMESTAMP_COL_ID); + code = nodesListPushFront(pCols, pkNode); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pkNode); + nodesDestroyList(pCols); + return code; + } + } *pOutput = pCols; } else { nodesDestroyList(pCols); @@ -1405,11 +1562,33 @@ static bool planOptNodeListHasTbname(SNodeList* pKeys) { } static bool partTagsIsOptimizableNode(SLogicNode* pNode) { - return ((QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode) || - (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) && NULL != ((SAggLogicNode*)pNode)->pGroupKeys && - NULL != ((SAggLogicNode*)pNode)->pAggFuncs)) && - 1 == LIST_LENGTH(pNode->pChildren) && - QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(nodesListGetNode(pNode->pChildren, 0))); + bool ret = 1 == LIST_LENGTH(pNode->pChildren) && + QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(nodesListGetNode(pNode->pChildren, 0)); + if (!ret) return ret; + switch (nodeType(pNode)) { + case QUERY_NODE_LOGIC_PLAN_PARTITION: { + if (pNode->pParent && nodeType(pNode->pParent) == QUERY_NODE_LOGIC_PLAN_WINDOW) { + SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode->pParent; + if (pWindow->winType == WINDOW_TYPE_INTERVAL) { + // if interval has slimit, we push down partition node to scan, and scan will set groupOrderScan to true + // we want to skip groups of blocks after slimit satisfied + // if interval only has limit, we do not push down partition node to scan + // we want to get grouped output from partition node and make use of limit + // if no slimit and no limit, we push down partition node and groupOrderScan is false, cause we do not need + // group ordered output + if (!pWindow->node.pSlimit && pWindow->node.pLimit) ret = false; + } + } + } break; + case QUERY_NODE_LOGIC_PLAN_AGG: { + SAggLogicNode* pAgg = (SAggLogicNode*)pNode; + ret = pAgg->pGroupKeys && pAgg->pAggFuncs; + } break; + default: + ret = false; + break; + } + return ret; } static SNodeList* partTagsGetPartKeys(SLogicNode* pNode) { @@ -1547,8 +1726,11 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub if (TSDB_CODE_SUCCESS == code) { if (QUERY_NODE_LOGIC_PLAN_AGG == pNode->pParent->type) { SAggLogicNode* pParent = (SAggLogicNode*)(pNode->pParent); + scanPathOptSetGroupOrderScan(pScan); pParent->hasGroupKeyOptimized = true; } + if (pNode->pParent->pSlimit) + pScan->groupOrderScan = true; NODES_CLEAR_LIST(pNode->pChildren); nodesDestroyNode((SNode*)pNode); @@ -2195,6 +2377,13 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) { SFunctionNode* pAggFunc = (SFunctionNode*)pFunc; if (FUNCTION_TYPE_LAST == pAggFunc->funcType) { + SNode* pPar = nodesListGetNode(pAggFunc->pParameterList, 0); + if (QUERY_NODE_COLUMN == nodeType(pPar)) { + SColumnNode* pCol = (SColumnNode*)pPar; + if (pCol->colType != COLUMN_TYPE_COLUMN) { + return false; + } + } if (hasSelectFunc || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) { return false; } @@ -2295,6 +2484,7 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic if (NULL != cxt.pLastCols) { cxt.doAgg = false; lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols, true); + nodesWalkExprs(pScan->pScanPseudoCols, lastRowScanOptSetColDataType, &cxt); lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols, false); nodesClearList(cxt.pLastCols); } @@ -2440,7 +2630,7 @@ static void tagScanOptCloneAncestorSlimit(SLogicNode* pTableScanNode) { SLogicNode* pNode = tagScanOptFindAncestorWithSlimit(pTableScanNode); if (NULL != pNode) { - //TODO: only set the slimit now. push down slimit later + // TODO: only set the slimit now. push down slimit later pTableScanNode->pSlimit = nodesCloneNode(pNode->pSlimit); ((SLimitNode*)pTableScanNode->pSlimit)->limit += ((SLimitNode*)pTableScanNode->pSlimit)->offset; ((SLimitNode*)pTableScanNode->pSlimit)->offset = 0; @@ -2494,13 +2684,79 @@ static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp } static bool pushDownLimitOptShouldBeOptimized(SLogicNode* pNode) { - if (NULL == pNode->pLimit || 1 != LIST_LENGTH(pNode->pChildren) || - QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0))) { + if ((NULL == pNode->pLimit && pNode->pSlimit == NULL) || 1 != LIST_LENGTH(pNode->pChildren)) { return false; } + + SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pNode->pChildren, 0); + if (pChild->pLimit || pChild->pSlimit) return false; return true; } +static void swapLimit(SLogicNode* pParent, SLogicNode* pChild) { + pChild->pLimit = pParent->pLimit; + pParent->pLimit = NULL; +} + +static void cloneLimit(SLogicNode* pParent, SLogicNode* pChild) { + SLimitNode* pLimit = NULL; + if (pParent->pLimit) { + pChild->pLimit = nodesCloneNode(pParent->pLimit); + pLimit = (SLimitNode*)pChild->pLimit; + pLimit->limit += pLimit->offset; + pLimit->offset = 0; + } + + if (pParent->pSlimit) { + pChild->pSlimit = nodesCloneNode(pParent->pSlimit); + pLimit = (SLimitNode*)pChild->pSlimit; + pLimit->limit += pLimit->offset; + pLimit->offset = 0; + } +} + +static bool pushDownLimitHow(SLogicNode* pNodeWithLimit, SLogicNode* pNodeLimitPushTo); +static bool pushDownLimitTo(SLogicNode* pNodeWithLimit, SLogicNode* pNodeLimitPushTo) { + switch (nodeType(pNodeLimitPushTo)) { + case QUERY_NODE_LOGIC_PLAN_WINDOW: { + SWindowLogicNode* pWindow = (SWindowLogicNode*)pNodeLimitPushTo; + if (pWindow->winType != WINDOW_TYPE_INTERVAL) break; + cloneLimit(pNodeWithLimit, pNodeLimitPushTo); + return true; + } + case QUERY_NODE_LOGIC_PLAN_FILL: + case QUERY_NODE_LOGIC_PLAN_SORT: { + cloneLimit(pNodeWithLimit, pNodeLimitPushTo); + SNode* pChild = NULL; + FOREACH(pChild, pNodeLimitPushTo->pChildren) { pushDownLimitHow(pNodeLimitPushTo, (SLogicNode*)pChild); } + return true; + } + case QUERY_NODE_LOGIC_PLAN_SCAN: + if (nodeType(pNodeWithLimit) == QUERY_NODE_LOGIC_PLAN_PROJECT && pNodeWithLimit->pLimit) { + swapLimit(pNodeWithLimit, pNodeLimitPushTo); + return true; + } + default: + break; + } + return false; +} + +static bool pushDownLimitHow(SLogicNode* pNodeWithLimit, SLogicNode* pNodeLimitPushTo) { + switch (nodeType(pNodeWithLimit)) { + case QUERY_NODE_LOGIC_PLAN_PROJECT: + case QUERY_NODE_LOGIC_PLAN_FILL: + return pushDownLimitTo(pNodeWithLimit, pNodeLimitPushTo); + case QUERY_NODE_LOGIC_PLAN_SORT: { + SSortLogicNode* pSort = (SSortLogicNode*)pNodeWithLimit; + if (sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys)) return pushDownLimitTo(pNodeWithLimit, pNodeLimitPushTo); + } + default: + break; + } + return false; +} + static int32_t pushDownLimitOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SLogicNode* pNode = optFindPossibleNode(pLogicSubplan->pNode, pushDownLimitOptShouldBeOptimized); if (NULL == pNode) { @@ -2509,10 +2765,9 @@ static int32_t pushDownLimitOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pNode->pChildren, 0); nodesDestroyNode(pChild->pLimit); - pChild->pLimit = pNode->pLimit; - pNode->pLimit = NULL; - pCxt->optimized = true; - + if (pushDownLimitHow(pNode, pChild)) { + pCxt->optimized = true; + } return TSDB_CODE_SUCCESS; } @@ -2755,12 +3010,65 @@ static int32_t tableCountScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLo return code; } +static SSortLogicNode* sortNonPriKeySatisfied(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SORT != nodeType(pNode)) { + return NULL; + } + SSortLogicNode* pSort = (SSortLogicNode*)pNode; + if (sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys)) { + return NULL; + } + SNode *pSortKeyNode = NULL, *pSortKeyExpr = NULL; + FOREACH(pSortKeyNode, pSort->pSortKeys) { + pSortKeyExpr = ((SOrderByExprNode*)pSortKeyNode)->pExpr; + switch (nodeType(pSortKeyExpr)) { + case QUERY_NODE_COLUMN: + break; + case QUERY_NODE_VALUE: + continue; + default: + return NULL; + } + } + + if (!pSortKeyExpr || ((SColumnNode*)pSortKeyExpr)->projIdx != 1 || + ((SColumnNode*)pSortKeyExpr)->node.resType.type != TSDB_DATA_TYPE_TIMESTAMP) { + return NULL; + } + return pSort; +} + +static bool sortNonPriKeyShouldOptimize(SLogicNode* pNode, void* pInfo) { + SSortLogicNode* pSort = sortNonPriKeySatisfied(pNode); + if (!pSort) return false; + SNodeList* pSortNodeList = pInfo; + nodesListAppend(pSortNodeList, (SNode*)pSort); + return false; +} + +static int32_t sortNonPriKeyOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SNodeList* pNodeList = nodesMakeList(); + optFindEligibleNode(pLogicSubplan->pNode, sortNonPriKeyShouldOptimize, pNodeList); + SNode* pNode = NULL; + FOREACH(pNode, pNodeList) { + SSortLogicNode* pSort = (SSortLogicNode*)pNode; + SOrderByExprNode* pOrderByExpr = (SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0); + pSort->node.outputTsOrder = pOrderByExpr->order; + optSetParentOrder(pSort->node.pParent, pOrderByExpr->order, NULL); + } + pCxt->optimized = false; + nodesClearList(pNodeList); + return TSDB_CODE_SUCCESS; +} + // clang-format off static const SOptimizeRule optimizeRuleSet[] = { {.pName = "ScanPath", .optimizeFunc = scanPathOptimize}, {.pName = "PushDownCondition", .optimizeFunc = pushDownCondOptimize}, + {.pName = "sortNonPriKeyOptimize", .optimizeFunc = sortNonPriKeyOptimize}, {.pName = "SortPrimaryKey", .optimizeFunc = sortPrimaryKeyOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaIndexOptimize}, + {.pName = "PushDownLimit", .optimizeFunc = pushDownLimitOptimize}, {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, {.pName = "MergeProjects", .optimizeFunc = mergeProjectsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize}, @@ -2769,7 +3077,6 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "RewriteUnique", .optimizeFunc = rewriteUniqueOptimize}, {.pName = "LastRowScan", .optimizeFunc = lastRowScanOptimize}, {.pName = "TagScan", .optimizeFunc = tagScanOptimize}, - {.pName = "PushDownLimit", .optimizeFunc = pushDownLimitOptimize}, {.pName = "TableCountScan", .optimizeFunc = tableCountScanOptimize}, }; // clang-format on diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index be43bb008c7ece11cd2cf1eb9695b2a6a410415d..06859e195dd8fcb2d2077916e6048a1f0ae7168c 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -366,6 +366,8 @@ static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode TSWAP(pPhysiNode->pLimit, pLogicNode->pLimit); TSWAP(pPhysiNode->pSlimit, pLogicNode->pSlimit); + pPhysiNode->inputTsOrder = pLogicNode->inputTsOrder; + pPhysiNode->outputTsOrder = pLogicNode->outputTsOrder; int32_t code = createDataBlockDesc(pCxt, pLogicNode->pTargets, &pPhysiNode->pOutputDataBlockDesc); if (TSDB_CODE_SUCCESS != code) { @@ -445,6 +447,7 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS pScanPhysiNode->uid = pScanLogicNode->tableId; pScanPhysiNode->suid = pScanLogicNode->stableId; pScanPhysiNode->tableType = pScanLogicNode->tableType; + pScanPhysiNode->groupOrderScan = pScanLogicNode->groupOrderScan; memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName)); if (NULL != pScanLogicNode->pTagCond) { pSubplan->pTagCond = nodesCloneNode(pScanLogicNode->pTagCond); @@ -676,7 +679,7 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren int32_t code = TSDB_CODE_SUCCESS; pJoin->joinType = pJoinLogicNode->joinType; - pJoin->inputTsOrder = pJoinLogicNode->inputTsOrder; + pJoin->node.inputTsOrder = pJoinLogicNode->node.inputTsOrder; setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pMergeCondition, &pJoin->pMergeCondition); if (TSDB_CODE_SUCCESS == code) { @@ -705,6 +708,9 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren pJoinLogicNode->pOnConditions, &pJoin->pOnConditions); } + if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pColEqualOnConditions) { + code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqualOnConditions, &pJoin->pColEqualOnConditions); + } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); } @@ -875,6 +881,7 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, pAgg->mergeDataBlock = (GROUP_ACTION_KEEP == pAggLogicNode->node.groupAction ? false : true); pAgg->groupKeyOptimized = pAggLogicNode->hasGroupKeyOptimized; + pAgg->node.forceCreateNonBlockingOptr = pAggLogicNode->node.forceCreateNonBlockingOptr; SNodeList* pPrecalcExprs = NULL; SNodeList* pGroupKeys = NULL; @@ -936,6 +943,11 @@ static int32_t createIndefRowsFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* SNodeList* pFuncs = NULL; int32_t code = rewritePrecalcExprs(pCxt, pFuncLogicNode->pFuncs, &pPrecalcExprs, &pFuncs); + if (pIdfRowsFunc->node.inputTsOrder == 0) { + // default to asc + pIdfRowsFunc->node.inputTsOrder = TSDB_ORDER_ASC; + } + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); // push down expression to pOutputDataBlockDesc of child node if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { @@ -1147,15 +1159,18 @@ static int32_t createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNo } } -static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWinodwPhysiNode* pWindow, +static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowPhysiNode* pWindow, SWindowLogicNode* pWindowLogicNode) { pWindow->triggerType = pWindowLogicNode->triggerType; pWindow->watermark = pWindowLogicNode->watermark; pWindow->deleteMark = pWindowLogicNode->deleteMark; pWindow->igExpired = pWindowLogicNode->igExpired; - pWindow->inputTsOrder = pWindowLogicNode->inputTsOrder; - pWindow->outputTsOrder = pWindowLogicNode->outputTsOrder; pWindow->mergeDataBlock = (GROUP_ACTION_KEEP == pWindowLogicNode->node.groupAction ? false : true); + pWindow->node.inputTsOrder = pWindowLogicNode->node.inputTsOrder; + pWindow->node.outputTsOrder = pWindowLogicNode->node.outputTsOrder; + if (nodeType(pWindow) == QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL) { + pWindow->node.inputTsOrder = pWindowLogicNode->node.outputTsOrder; + } SNodeList* pPrecalcExprs = NULL; SNodeList* pFuncs = NULL; @@ -1288,9 +1303,9 @@ static int32_t createStateWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC if (TSDB_CODE_SUCCESS == code) { code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pStateKey, &pState->pStateKey); - if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockSlot(pCxt, &pState->pStateKey, pState->window.node.pOutputDataBlockDesc); - } + // if (TSDB_CODE_SUCCESS == code) { + // code = addDataBlockSlot(pCxt, &pState->pStateKey, pState->window.node.pOutputDataBlockDesc); + // } } if (TSDB_CODE_SUCCESS == code) { @@ -1489,7 +1504,7 @@ static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren pFill->mode = pFillNode->mode; pFill->timeRange = pFillNode->timeRange; - pFill->inputTsOrder = pFillNode->inputTsOrder; + pFill->node.inputTsOrder = pFillNode->node.inputTsOrder; SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); int32_t code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pFillNode->pFillExprs, &pFill->pFillExprs); @@ -1644,6 +1659,9 @@ static int32_t createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pChildren, (SNode*)pChild); } + if (TSDB_CODE_SUCCESS != code) { + break; + } } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 504db0d07b8ce4988b3cd40b4089d24dc88dc03f..3f6c73b4e5e5002f27aa85ced327140a9164645e 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -306,54 +306,6 @@ static bool stbSplIsTableCountQuery(SLogicNode* pNode) { return QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && SCAN_TYPE_TABLE_COUNT == ((SScanLogicNode*)pChild)->scanType; } -static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) { - if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - return ((SScanLogicNode*)pNode)->pGroupTags; - } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { - return ((SPartitionLogicNode*)pNode)->pPartitionKeys; - } else { - return NULL; - } -} - -static bool stbSplHasPartTbname(SNodeList* pPartKeys) { - if (NULL == pPartKeys) { - return false; - } - SNode* pPartKey = NULL; - FOREACH(pPartKey, pPartKeys) { - if (QUERY_NODE_GROUPING_SET == nodeType(pPartKey)) { - pPartKey = nodesListGetNode(((SGroupingSetNode*)pPartKey)->pParameterList, 0); - } - if ((QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType) || - (QUERY_NODE_COLUMN == nodeType(pPartKey) && COLUMN_TYPE_TBNAME == ((SColumnNode*)pPartKey)->colType)) { - return true; - } - } - return false; -} - -static bool stbSplNotSystemScan(SLogicNode* pNode) { - if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - return SCAN_TYPE_SYSTEM_TABLE != ((SScanLogicNode*)pNode)->scanType; - } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { - return stbSplNotSystemScan((SLogicNode*)nodesListGetNode(pNode->pChildren, 0)); - } else { - return true; - } -} - -static bool stbSplIsPartTableAgg(SAggLogicNode* pAgg) { - if (1 != LIST_LENGTH(pAgg->node.pChildren)) { - return false; - } - if (NULL != pAgg->pGroupKeys) { - return stbSplHasPartTbname(pAgg->pGroupKeys) && - stbSplNotSystemScan((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0)); - } - return stbSplHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))); -} - static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: @@ -364,7 +316,7 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { return streamQuery ? false : stbSplIsMultiTbScanChild(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_AGG: return (!stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) || - stbSplIsPartTableAgg((SAggLogicNode*)pNode)) && + isPartTableAgg((SAggLogicNode*)pNode)) && stbSplHasMultiTbScan(streamQuery, pNode) && !stbSplIsTableCountQuery(pNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: return stbSplNeedSplitWindow(streamQuery, pNode); @@ -534,7 +486,9 @@ static int32_t stbSplGetNumOfVgroups(SLogicNode* pNode) { static int32_t stbSplRewriteFromMergeNode(SMergeLogicNode* pMerge, SLogicNode* pNode) { int32_t code = TSDB_CODE_SUCCESS; - + pMerge->node.inputTsOrder = pNode->outputTsOrder; + pMerge->node.outputTsOrder = pNode->outputTsOrder; + switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_PROJECT: { SProjectLogicNode *pLogicNode = (SProjectLogicNode*)pNode; @@ -544,6 +498,18 @@ static int32_t stbSplRewriteFromMergeNode(SMergeLogicNode* pMerge, SLogicNode* p } break; } + case QUERY_NODE_LOGIC_PLAN_WINDOW: { + SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode; + if (pMerge->node.pLimit) { + nodesDestroyNode(pMerge->node.pLimit); + pMerge->node.pLimit = NULL; + } + if (pMerge->node.pSlimit) { + nodesDestroyNode(pMerge->node.pSlimit); + pMerge->node.pSlimit = NULL; + } + break; + } default: break; } @@ -631,7 +597,7 @@ static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo ((SWindowLogicNode*)pInfo->pSplitNode)->windowAlgo = INTERVAL_ALGO_MERGE; SNodeList* pMergeKeys = NULL; code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk, - ((SWindowLogicNode*)pInfo->pSplitNode)->outputTsOrder, &pMergeKeys); + ((SWindowLogicNode*)pInfo->pSplitNode)->node.outputTsOrder, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow, true); } @@ -721,7 +687,7 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl SNodeList* pMergeKeys = NULL; int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, - ((SWindowLogicNode*)pWindow)->inputTsOrder, &pMergeKeys); + ((SWindowLogicNode*)pWindow)->node.inputTsOrder, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild, true); @@ -776,10 +742,6 @@ static int32_t stbSplSplitEvent(SSplitContext* pCxt, SStableSplitInfo* pInfo) { } } -static bool stbSplIsPartTableWinodw(SWindowLogicNode* pWindow) { - return stbSplHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); -} - static int32_t stbSplSplitWindowForCrossTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { switch (((SWindowLogicNode*)pInfo->pSplitNode)->winType) { case WINDOW_TYPE_INTERVAL: @@ -832,7 +794,7 @@ static int32_t stbSplSplitWindowForPartTable(SSplitContext* pCxt, SStableSplitIn } static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { - if (stbSplIsPartTableWinodw((SWindowLogicNode*)pInfo->pSplitNode)) { + if (isPartTableWinodw((SWindowLogicNode*)pInfo->pSplitNode)) { return stbSplSplitWindowForPartTable(pCxt, pInfo); } else { return stbSplSplitWindowForCrossTable(pCxt, pInfo); @@ -918,7 +880,7 @@ static int32_t stbSplSplitAggNodeForCrossTable(SSplitContext* pCxt, SStableSplit } static int32_t stbSplSplitAggNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { - if (stbSplIsPartTableAgg((SAggLogicNode*)pInfo->pSplitNode)) { + if (isPartTableAgg((SAggLogicNode*)pInfo->pSplitNode)) { return stbSplSplitAggNodeForPartTable(pCxt, pInfo); } return stbSplSplitAggNodeForCrossTable(pCxt, pInfo); diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index 4b8009347d5e44ce236f99652b4814e36b8af459..88086cde1d0edb91e2918a26935495be0b3120ce 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -51,6 +51,7 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { } return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } + case QUERY_NODE_VALUE: case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_FUNCTION: @@ -247,8 +248,7 @@ static int32_t adjustPartitionDataRequirement(SPartitionLogicNode* pPart, EDataO return TSDB_CODE_PLAN_INTERNAL_ERROR; } pPart->node.resultDataOrder = requirement; - pPart->node.requireDataOrder = - (requirement >= DATA_ORDER_LEVEL_IN_BLOCK ? DATA_ORDER_LEVEL_GLOBAL : DATA_ORDER_LEVEL_NONE); + pPart->node.requireDataOrder = requirement; return TSDB_CODE_SUCCESS; } @@ -321,3 +321,57 @@ int32_t adjustLogicNodeDataRequirement(SLogicNode* pNode, EDataOrderLevel requir } return code; } + +static bool stbNotSystemScan(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + return SCAN_TYPE_SYSTEM_TABLE != ((SScanLogicNode*)pNode)->scanType; + } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { + return stbNotSystemScan((SLogicNode*)nodesListGetNode(pNode->pChildren, 0)); + } else { + return true; + } +} + +static bool stbHasPartTbname(SNodeList* pPartKeys) { + if (NULL == pPartKeys) { + return false; + } + SNode* pPartKey = NULL; + FOREACH(pPartKey, pPartKeys) { + if (QUERY_NODE_GROUPING_SET == nodeType(pPartKey)) { + pPartKey = nodesListGetNode(((SGroupingSetNode*)pPartKey)->pParameterList, 0); + } + if ((QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType) || + (QUERY_NODE_COLUMN == nodeType(pPartKey) && COLUMN_TYPE_TBNAME == ((SColumnNode*)pPartKey)->colType)) { + return true; + } + } + return false; +} + +static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + return ((SScanLogicNode*)pNode)->pGroupTags; + } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { + return ((SPartitionLogicNode*)pNode)->pPartitionKeys; + } else { + return NULL; + } +} + +bool isPartTableAgg(SAggLogicNode* pAgg) { + if (1 != LIST_LENGTH(pAgg->node.pChildren)) { + return false; + } + if (NULL != pAgg->pGroupKeys) { + return stbHasPartTbname(pAgg->pGroupKeys) && + stbNotSystemScan((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0)); + } + return stbHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))); +} + +bool isPartTableWinodw(SWindowLogicNode* pWindow) { + return stbHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); +} + + diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 58b8e53478ef48dfb7fb4198cc50426a8387cf77..2fcc8510d4860e4f7b35cee2a29c9be9095ca777 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -97,6 +97,12 @@ static int32_t setSubplanExecutionNode(SPhysiNode* pNode, int32_t groupId, SDown return TSDB_CODE_SUCCESS; } +int32_t qContinuePlanPostQuery(void *pPostPlan) { + //TODO + return TSDB_CODE_SUCCESS; +} + + int32_t qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstreamSourceNode* pSource) { planDebug("QID:0x%" PRIx64 " set subplan execution node, groupId:%d", subplan->id.queryId, groupId); return setSubplanExecutionNode(subplan->pNode, groupId, pSource); diff --git a/source/libs/planner/test/CMakeLists.txt b/source/libs/planner/test/CMakeLists.txt index b9d5c85717c71cdcb73cf660ac796541896c85e0..73aca8572a0ff5c0dd916ed65a168eedf2424f95 100644 --- a/source/libs/planner/test/CMakeLists.txt +++ b/source/libs/planner/test/CMakeLists.txt @@ -12,10 +12,17 @@ IF(NOT TD_DARWIN) "${SOURCE_LIST}/../../../parser/test/mockCatalogService.cpp" ) - TARGET_LINK_LIBRARIES( - plannerTest - PUBLIC os util common nodes planner parser catalog transport gtest function qcom - ) + IF (TD_GRANT) + TARGET_LINK_LIBRARIES( + plannerTest + PUBLIC os util common nodes planner parser catalog transport gtest function qcom grant + ) + ELSE () + TARGET_LINK_LIBRARIES( + plannerTest + PUBLIC os util common nodes planner parser catalog transport gtest function qcom + ) + ENDIF() TARGET_INCLUDE_DIRECTORIES( plannerTest diff --git a/source/libs/planner/test/planStmtTest.cpp b/source/libs/planner/test/planStmtTest.cpp index bab38797cc84b044c98fbe1f23a019f2bf8eae0a..df8a509d12556137b3040961226d3388f1696a5f 100644 --- a/source/libs/planner/test/planStmtTest.cpp +++ b/source/libs/planner/test/planStmtTest.cpp @@ -108,6 +108,7 @@ class PlanStmtTest : public PlannerTestBase { switch (type) { case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_GEOMETRY: strncpy((char*)pBindParam->buffer, pVal, bytes); break; case TSDB_DATA_TYPE_TIMESTAMP: diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index d89e669a90230894c5a7955a2eece9b1d2cc84e3..3b432b9890e1bb5bd903b2da61ef6b044618ee48 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -441,6 +441,16 @@ class PlannerTestBaseImpl { pCxt->topicQuery = true; } else if (QUERY_NODE_CREATE_INDEX_STMT == nodeType(pQuery->pRoot)) { SMCreateSmaReq req = {0}; + SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pQuery->pRoot; + SCmdMsgInfo* pCmdMsg = (SCmdMsgInfo*)taosMemoryMalloc(sizeof(SCmdMsgInfo)); + if (NULL == pCmdMsg) FAIL(); + pCmdMsg->msgType = TDMT_MND_CREATE_SMA; + pCmdMsg->msgLen = tSerializeSMCreateSmaReq(NULL, 0, pStmt->pReq); + pCmdMsg->pMsg = taosMemoryMalloc(pCmdMsg->msgLen); + if (!pCmdMsg->pMsg) FAIL(); + tSerializeSMCreateSmaReq(pCmdMsg->pMsg, pCmdMsg->msgLen, pStmt->pReq); + ((SQuery*)pQuery)->pCmdMsg = pCmdMsg; + tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req); g_mockCatalogService->createSmaIndex(&req); nodesStringToNode(req.ast, &pCxt->pAstRoot); diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 9d8c17000345a7aa73761c1e3f9af5ae4a4a3363..38d7c9da3bb5a01848c2055d28a982272173a289 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -52,6 +52,10 @@ static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen if (pSchema[i].bytes > TSDB_MAX_NCHAR_LEN) { return false; } + } else if (pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) { + if (pSchema[i].bytes > TSDB_MAX_GEOMETRY_LEN) { + return false; + } } else { if (pSchema[i].bytes != tDataTypes[pSchema[i].type].bytes) { return false; @@ -137,7 +141,7 @@ void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { if (NULL == pMsgBody) { return; } - + taosMemoryFreeClear(pMsgBody->target.dbFName); taosMemoryFreeClear(pMsgBody->msgInfo.pData); if (pMsgBody->paramFreeFp) { @@ -169,7 +173,7 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra .contLen = pInfo->msgInfo.len, .info.ahandle = (void*)pInfo, .info.handle = pInfo->msgInfo.handle, - .info.persistHandle = persistHandle, + .info.persistHandle = persistHandle, .code = 0 }; TRACE_SET_ROOTID(&rpcMsg.info.traceId, pInfo->requestId); @@ -248,7 +252,7 @@ void destroyQueryExecRes(SExecResult* pRes) { taosMemoryFreeClear(pRes->res); break; } - case TDMT_SCH_QUERY: + case TDMT_SCH_QUERY: case TDMT_SCH_MERGE_QUERY: { taosArrayDestroy((SArray*)pRes->res); break; @@ -297,6 +301,7 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_GEOMETRY: if (bufSize < 0) { // tscError("invalid buf size"); return TSDB_CODE_TSC_INVALID_VALUE; @@ -449,6 +454,18 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } +void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType) { + int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; + for (int32_t i = 0; i < nums; ++i) { + if (0 == strcmp(pName, pMeta->schema[i].name)) { + *pType = (i < pMeta->tableInfo.numOfColumns) ? TCOL_TYPE_COLUMN : TCOL_TYPE_TAG; + return; + } + } + + *pType = TCOL_TYPE_NONE; +} + void freeVgInfo(SDBVgInfo* vgInfo) { if (NULL == vgInfo) { return; @@ -515,7 +532,7 @@ int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) { (*pDst)->name = taosStrdup(pSrc->name); } (*pDst)->uid = pSrc->uid; - (*pDst)->ctime = pSrc->ctime; + (*pDst)->btime = pSrc->btime; (*pDst)->ttl = pSrc->ttl; (*pDst)->commentLen = pSrc->commentLen; if (pSrc->comment) { @@ -548,3 +565,11 @@ int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) { return TSDB_CODE_SUCCESS; } + +void freeDbCfgInfo(SDbCfgInfo *pInfo) { + if (pInfo) { + taosArrayDestroy(pInfo->pRetensions); + } + taosMemoryFree(pInfo); +} + diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt index 92ccde31634eb8051952ff313c86056fba08f075..7a984cd000caf314f59e74baad09d898800bf19d 100644 --- a/source/libs/qworker/CMakeLists.txt +++ b/source/libs/qworker/CMakeLists.txt @@ -7,8 +7,8 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(qworker - PRIVATE os util transport nodes planner qcom executor +TARGET_LINK_LIBRARIES(qworker + PRIVATE os util transport nodes planner qcom executor index ) if(${BUILD_TEST}) diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index f198b73c7d880c032064d0ea5edece3908af8351..4bc357d7dda790e833908acaa6bb20b3cd5a9b35 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -120,7 +120,8 @@ typedef struct SQWTaskCtx { int8_t explain; int8_t needFetch; int8_t localExec; - int32_t msgType; + int32_t queryMsgType; + int32_t fetchMsgType; int32_t level; uint64_t sId; diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 59e63e9eae2636ae1676ac612083e3857e3a7510..f2e48918ab053ec3f789b621e5e78fb645df80fa 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -126,10 +126,10 @@ void qwDbgDumpTasksInfo(SQWorker *mgmt) { void *key = taosHashGetKey(pIter, NULL); QW_GET_QTID(key, qId, tId, eId); - QW_TASK_DLOG("%p lock:%x, phase:%d, type:%d, explain:%d, needFetch:%d, localExec:%d, msgType:%d, " + QW_TASK_DLOG("%p lock:%x, phase:%d, type:%d, explain:%d, needFetch:%d, localExec:%d, queryMsgType:%d, " "sId:%" PRId64 ", level:%d, queryGotData:%d, queryRsped:%d, queryEnd:%d, queryContinue:%d, queryInQueue:%d, " "rspCode:%x, affectedRows:%" PRId64 ", taskHandle:%p, sinkHandle:%p, tbFName:%s, sver:%d, tver:%d, events:%d,%d,%d,%d,%d", - ctx, ctx->lock, ctx->phase, ctx->taskType, ctx->explain, ctx->needFetch, ctx->localExec, ctx->msgType, + ctx, ctx->lock, ctx->phase, ctx->taskType, ctx->explain, ctx->needFetch, ctx->localExec, ctx->queryMsgType, ctx->sId, ctx->level, ctx->queryGotData, ctx->queryRsped, ctx->queryEnd, ctx->queryContinue, ctx->queryInQueue, ctx->rspCode, ctx->affectedRows, ctx->taskHandle, ctx->sinkHandle, ctx->tbInfo.tbFName, ctx->tbInfo.sversion, ctx->tbInfo.tversion, ctx->events[QW_EVENT_CANCEL], ctx->events[QW_EVENT_READY], @@ -259,9 +259,9 @@ void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped) { static int32_t ignoreTime = 0; if (++ignoreTime > 10 && 0 == taosRand() % 9) { - if (ctx->msgType == TDMT_SCH_FETCH) { + if (ctx->fetchMsgType == TDMT_SCH_FETCH) { qwBuildAndSendErrorRsp(TDMT_SCH_LINK_BROKEN, &ctx->ctrlConnInfo, TSDB_CODE_RPC_BROKEN_LINK); - qwBuildAndSendErrorRsp(ctx->msgType + 1, &ctx->dataConnInfo, TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); + qwBuildAndSendErrorRsp(ctx->fetchMsgType + 1, &ctx->dataConnInfo, TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); *rsped = true; taosSsleep(3); diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 1a3a740b34918c695dfb79848f8df2c3500b0f3a..508e957e265d295b51db83aedb22ac92575bec9c 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -366,7 +366,7 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - if (chkGrant && (!TEST_SHOW_REWRITE_MASK(msg.msgMask)) && (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS)) { + if (chkGrant && (!TEST_SHOW_REWRITE_MASK(msg.msgMask)) && !taosGranted()) { QW_ELOG("query failed cause of grant expired, msgMask:%d", msg.msgMask); tFreeSSubQueryMsg(&msg); QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); @@ -440,11 +440,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int int64_t rId = msg.refId; int32_t eId = msg.execId; - SQWMsg qwMsg = {.node = node, - .msg = msg.msg, - .msgLen = msg.msgLen, - .connInfo = pMsg->info, - .msgType = pMsg->msgType}; + SQWMsg qwMsg = {.node = node, .msg = msg.msg, .msgLen = msg.msgLen, .connInfo = pMsg->info, .msgType = pMsg->msgType}; qwMsg.msgInfo.explain = msg.explain; qwMsg.msgInfo.taskType = msg.taskType; qwMsg.msgInfo.needFetch = msg.needFetch; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 92c7852dbca5ba9bd2bf0542e31e7b63b15aa632..6f641e677a7edacbf862b095854772567431a7ba 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -254,6 +254,7 @@ bool qwTaskNotInExec(SQWTaskCtx *ctx) { return false; } + int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { int32_t taskNum = 0; @@ -426,6 +427,7 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes pRes->suid = pDelRes->suid; pRes->uidList = pDelRes->uidList; + pRes->ctimeMs = taosGetTimestampMs(); pRes->skey = pDelRes->skey; pRes->ekey = pDelRes->ekey; pRes->affectedRows = pDelRes->affectedRows; @@ -436,6 +438,40 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes return TSDB_CODE_SUCCESS; } + +int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx * ctx, SQWMsg *qwMsg, int32_t code) { + if (QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy && ctx != NULL) { + if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { + void *rsp = NULL; + int32_t dataLen = 0; + SOutputData sOutput = {0}; + if (qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)) { + return TSDB_CODE_SUCCESS; + } + + if (rsp) { + bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); + + qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); + if (qComplete) { + atomic_store_8((int8_t *)&ctx->queryEnd, true); + } + + qwMsg->connInfo = ctx->dataConnInfo; + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); + + qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code); + rsp = NULL; + + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), + dataLen); + } + } + } + + return TSDB_CODE_SUCCESS; +} + int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { int32_t code = 0; SQWTaskCtx *ctx = NULL; @@ -584,7 +620,7 @@ _return: if (QW_PHASE_POST_QUERY == phase && ctx && !ctx->queryRsped) { bool rsped = false; - SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; + SQWMsg qwMsg = {.msgType = ctx->queryMsgType, .connInfo = ctx->ctrlConnInfo}; qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); if (!rsped) { @@ -634,6 +670,8 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT)); + qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); + _return: if (ctx) { @@ -660,7 +698,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { ctx->taskType = qwMsg->msgInfo.taskType; ctx->explain = qwMsg->msgInfo.explain; ctx->needFetch = qwMsg->msgInfo.needFetch; - ctx->msgType = qwMsg->msgType; + ctx->queryMsgType = qwMsg->msgType; ctx->localExec = false; // QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); @@ -684,7 +722,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { QW_ERR_JRET(TSDB_CODE_APP_ERROR); } - qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); + //qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); ctx->level = plan->level; atomic_store_ptr(&ctx->taskHandle, pTaskInfo); @@ -701,32 +739,7 @@ _return: input.msgType = qwMsg->msgType; code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); - if (QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy && ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { - void *rsp = NULL; - int32_t dataLen = 0; - SOutputData sOutput = {0}; - if (qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)) { - return TSDB_CODE_SUCCESS; - } - - if (rsp) { - bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); - - qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); - if (qComplete) { - atomic_store_8((int8_t *)&ctx->queryEnd, true); - } - - qwMsg->connInfo = ctx->dataConnInfo; - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - - qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); - rsp = NULL; - - QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), - dataLen); - } - } + qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code); QW_RET(TSDB_CODE_SUCCESS); } @@ -750,8 +763,10 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { atomic_store_8((int8_t *)&ctx->queryInQueue, 0); atomic_store_8((int8_t *)&ctx->queryContinue, 0); - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryStop)); - + if (!queryStop) { + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryStop)); + } + if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { SOutputData sOutput = {0}; QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); @@ -774,7 +789,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { qwMsg->connInfo = ctx->dataConnInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); + qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code); rsp = NULL; QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, @@ -796,7 +811,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { rsp = NULL; qwMsg->connInfo = ctx->dataConnInfo; - qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, NULL, 0, code); + qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, NULL, 0, code); QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), 0); } @@ -810,6 +825,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { break; } QW_UNLOCK(QW_WRITE, &ctx->lock); + queryStop = false; } while (true); input.code = code; @@ -830,7 +846,7 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - ctx->msgType = qwMsg->msgType; + ctx->fetchMsgType = qwMsg->msgType; ctx->dataConnInfo = qwMsg->connInfo; SOutputData sOutput = {0}; @@ -896,6 +912,8 @@ _return: qwFreeFetchRsp(rsp); rsp = NULL; } + } else { + //qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code); } QW_RET(TSDB_CODE_SUCCESS); @@ -1307,7 +1325,7 @@ int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64 ctx->taskType = qwMsg->msgInfo.taskType; ctx->explain = qwMsg->msgInfo.explain; ctx->needFetch = qwMsg->msgInfo.needFetch; - ctx->msgType = qwMsg->msgType; + ctx->queryMsgType = qwMsg->msgType; ctx->localExec = true; ctx->explainRes = explainRes; @@ -1362,7 +1380,7 @@ int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64 QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - ctx->msgType = TDMT_SCH_MERGE_FETCH; + ctx->fetchMsgType = TDMT_SCH_MERGE_FETCH; ctx->explainRes = explainRes; SOutputData sOutput = {0}; diff --git a/source/libs/qworker/test/CMakeLists.txt b/source/libs/qworker/test/CMakeLists.txt index 780f5ae84be12a8b1feea24c25bfa148a1f9de5b..22870ea94dc1df36b8a50ebafef204c1a614d23a 100644 --- a/source/libs/qworker/test/CMakeLists.txt +++ b/source/libs/qworker/test/CMakeLists.txt @@ -8,7 +8,7 @@ IF(NOT TD_DARWIN) ADD_EXECUTABLE(qworkerTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( qworkerTest - PUBLIC os util common transport gtest qcom nodes planner qworker executor + PUBLIC os util common transport gtest qcom nodes planner qworker executor index ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/scalar/CMakeLists.txt b/source/libs/scalar/CMakeLists.txt index 193a6971e54b52b1c59749c0b41eefe6b9bbb2fb..1fe0f9a18db695cf1fc4ad6b36420cdeca0dccf3 100644 --- a/source/libs/scalar/CMakeLists.txt +++ b/source/libs/scalar/CMakeLists.txt @@ -8,12 +8,14 @@ target_include_directories( ) target_link_libraries(scalar - PRIVATE os - PRIVATE util + PRIVATE os + PRIVATE util PRIVATE common PRIVATE nodes PRIVATE function PRIVATE qcom + PRIVATE parser + PRIVATE geometry ) if(${BUILD_TEST}) diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 2023d387773aa294b6949f35365060dc8661c322..5fb7b0e90c30521c3af10f596c87db50448eb3aa 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -227,8 +227,10 @@ typedef struct SFltTreeStat { SFilterInfo *info; } SFltTreeStat; + typedef struct SFltScalarCtx { SNode *node; + SArray* fltSclRange; } SFltScalarCtx; typedef struct SFltBuildGroupCtx { @@ -237,6 +239,11 @@ typedef struct SFltBuildGroupCtx { int32_t code; } SFltBuildGroupCtx; +typedef struct { + SColumnNode *colNode; + SArray *points; +} SFltSclColumnRange; + struct SFilterInfo { bool scalarMode; SFltScalarCtx sclCtx; @@ -264,8 +271,9 @@ struct SFilterInfo { SFilterPCtx pctx; }; -#define FILTER_NO_MERGE_DATA_TYPE(t) \ - ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON) +#define FILTER_NO_MERGE_DATA_TYPE(t) \ + ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON || \ + (t) == TSDB_DATA_TYPE_GEOMETRY) #define FILTER_NO_MERGE_OPTR(o) ((o) == OP_TYPE_IS_NULL || (o) == OP_TYPE_IS_NOT_NULL || (o) == FILTER_DUMMY_EMPTY_OPTR) #define MR_EMPTY_RES(ctx) (ctx->rs == NULL) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 02a21b66edc1d12a09300f2204ab914327c06775..892fd588b6f274dd4418a03063986b1463491dd1 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -14,6 +14,7 @@ */ #include #include "os.h" +#include "tglobal.h" #include "thash.h" // #include "queryLog.h" #include "filter.h" @@ -22,6 +23,7 @@ #include "sclInt.h" #include "tcompare.h" #include "tdatablock.h" +#include "tsimplehash.h" #include "ttime.h" bool filterRangeCompGi(const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { @@ -131,7 +133,7 @@ __compar_fn_t gDataCompare[] = { setChkInBytes2, setChkInBytes4, setChkInBytes8, comparestrRegexMatch, comparestrRegexNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8, compareChkNotInString, comparestrPatternNMatch, comparewcsPatternNMatch, - comparewcsRegexMatch, comparewcsRegexNMatch, + comparewcsRegexMatch, comparewcsRegexNMatch, compareLenBinaryVal }; __compar_fn_t gInt8SignCompare[] = {compareInt8Val, compareInt8Int16, compareInt8Int32, @@ -178,7 +180,7 @@ __compar_fn_t gUint64UsignCompare[] = {compareUint64Uint8, compareUint64Uint16, int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { int8_t comparFn = 0; - if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -204,7 +206,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { } } - if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -260,7 +262,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { comparFn = 19; } else if (optr == OP_TYPE_NMATCH) { comparFn = 20; - } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ + } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ comparFn = 7; } else if (optr == OP_TYPE_NOT_LIKE) { /* wildcard query using like operator */ comparFn = 26; @@ -294,6 +296,21 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { break; } + case TSDB_DATA_TYPE_GEOMETRY: { + if (optr == OP_TYPE_EQUAL || optr == OP_TYPE_NOT_EQUAL || optr == OP_TYPE_IS_NULL || + optr == OP_TYPE_IS_NOT_NULL) { + comparFn = 30; + } else if (optr == OP_TYPE_IN) { + comparFn = 8; + } else if (optr == OP_TYPE_NOT_IN) { + comparFn = 25; + } else { + terrno = TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR; + return 0; + } + break; + } + case TSDB_DATA_TYPE_UTINYINT: comparFn = 11; break; @@ -448,8 +465,9 @@ static FORCE_INLINE SFilterRangeNode *filterNewRange(SFilterRangeCtx *ctx, SFilt } void *filterInitRangeCtx(int32_t type, int32_t options) { - if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_BINARY || - type == TSDB_DATA_TYPE_NCHAR) { + if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || + type == TSDB_DATA_TYPE_BINARY || + type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) { qError("not supported range type:%d", type); return NULL; } @@ -1038,12 +1056,12 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *fid) { if (node == NULL) { fltDebug("empty node"); - FLT_ERR_RET(TSDB_CODE_APP_ERROR); + goto _return; } if (nodeType(node) != QUERY_NODE_COLUMN && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) { - FLT_ERR_RET(TSDB_CODE_APP_ERROR); + goto _return; } int32_t type; @@ -1059,6 +1077,7 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f filterAddField(info, v, NULL, type, fid, 0, true, NULL); +_return: return TSDB_CODE_SUCCESS; } @@ -1567,6 +1586,7 @@ int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: if (bufSize < 0) { // tscError("invalid buf size"); return TSDB_CODE_TSC_INVALID_VALUE; @@ -1633,7 +1653,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) SDataType *dType = &var->node.resType; qDebug("VAL%d => [type:%d][val:%" PRIx64 "]", i, dType->type, var->datum.i); // TODO } else if (field->data) { - qDebug("VAL%d => [type:NIL][val:NIL]", i); // TODO + qDebug("VAL%d => [type:NIL][val:NIL]", i); // TODO } } @@ -1840,6 +1860,13 @@ void filterFreeInfo(SFilterInfo *info) { return; } + for (int32_t i = 0; i < taosArrayGetSize(info->sclCtx.fltSclRange); ++i) { + SFltSclColumnRange *colRange = taosArrayGet(info->sclCtx.fltSclRange, i); + nodesDestroyNode((SNode *)colRange->colNode); + taosArrayDestroy(colRange->points); + } + taosArrayDestroy(info->sclCtx.fltSclRange); + taosMemoryFreeClear(info->cunits); taosMemoryFreeClear(info->blkUnitRes); taosMemoryFreeClear(info->blkUnits); @@ -1936,33 +1963,15 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } SDataType *dType = &var->node.resType; - size_t bytes = 0; - - if (type == TSDB_DATA_TYPE_BINARY) { - size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes - : MAX_NUM_STR_SIZE; - bytes = len + 1 + VARSTR_HEADER_SIZE; - - fi->data = taosMemoryCalloc(1, bytes); - } else if (type == TSDB_DATA_TYPE_NCHAR) { - size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes - : MAX_NUM_STR_SIZE; - bytes = (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - - fi->data = taosMemoryCalloc(1, bytes); - } else { - fi->data = taosMemoryCalloc(1, sizeof(int64_t)); - } - if (dType->type == type) { + size_t bufBytes = TMAX(dType->bytes, sizeof(int64_t)); + fi->data = taosMemoryCalloc(1, bufBytes); assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type); } else { SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; out.columnData->info.type = type; out.columnData->info.precision = precision; - if (IS_VAR_DATA_TYPE(type)) { - out.columnData->info.bytes = bytes; - } else { + if (!IS_VAR_DATA_TYPE(type)) { out.columnData->info.bytes = tDataTypes[type].bytes; } @@ -1970,10 +1979,16 @@ int32_t fltInitValFieldData(SFilterInfo *info) { int32_t code = sclConvertValueToSclParam(var, &out, NULL); if (code != TSDB_CODE_SUCCESS) { qError("convert value to type[%d] failed", type); - return TSDB_CODE_TSC_INVALID_OPERATION; + return code; } - memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes); + size_t bufBytes = IS_VAR_DATA_TYPE(type) ? varDataTLen(out.columnData->pData) + : TMAX(out.columnData->info.bytes, sizeof(int64_t)); + fi->data = taosMemoryCalloc(1, bufBytes); + + size_t valBytes = IS_VAR_DATA_TYPE(type) ? varDataTLen(out.columnData->pData) : out.columnData->info.bytes; + memcpy(fi->data, out.columnData->pData, valBytes); + colDataDestroy(out.columnData); taosMemoryFree(out.columnData); } @@ -2739,6 +2754,7 @@ int32_t filterPostProcessRange(SFilterInfo *info) { } int32_t filterGenerateComInfo(SFilterInfo *info) { + terrno = 0; info->cunits = taosMemoryMalloc(info->unitNum * sizeof(*info->cunits)); info->blkUnitRes = taosMemoryMalloc(sizeof(*info->blkUnitRes) * info->unitNum); info->blkUnits = taosMemoryMalloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); @@ -2746,7 +2762,7 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; - info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); + info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); // set terrno if err info->cunits[i].rfunc = filterGetRangeCompFuncFromOptrs(unit->compare.optr, unit->compare.optr2); info->cunits[i].optr = FILTER_UNIT_OPTR(unit); info->cunits[i].colData = NULL; @@ -2767,7 +2783,7 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit); } - return TSDB_CODE_SUCCESS; + return terrno; } int32_t filterUpdateComUnits(SFilterInfo *info) { @@ -3324,6 +3340,7 @@ int32_t filterSetExecFunc(SFilterInfo *info) { } int32_t filterPreprocess(SFilterInfo *info) { + int32_t code = TSDB_CODE_SUCCESS; SFilterGroupCtx **gRes = taosMemoryCalloc(info->groupNum, sizeof(SFilterGroupCtx *)); int32_t gResNum = 0; @@ -3349,7 +3366,7 @@ int32_t filterPreprocess(SFilterInfo *info) { filterRewrite(info, gRes, gResNum); - filterGenerateComInfo(info); + FLT_ERR_JRET(filterGenerateComInfo(info)); _return: @@ -3361,7 +3378,7 @@ _return: taosMemoryFreeClear(gRes); - return TSDB_CODE_SUCCESS; + return code; } int32_t fltSetColFieldDataImpl(SFilterInfo *info, void *param, filer_get_col_from_id fp, bool fromColId) { @@ -3423,8 +3440,356 @@ _return: return code; } +// compare ranges, null < min < val < max. null=null, min=min, max=max +typedef enum { + FLT_SCL_DATUM_KIND_NULL, + FLT_SCL_DATUM_KIND_MIN, + FLT_SCL_DATUM_KIND_INT64, + FLT_SCL_DATUM_KIND_UINT64, + FLT_SCL_DATUM_KIND_FLOAT64, + FLT_SCL_DATUM_KIND_VARCHAR, + FLT_SCL_DATUM_KIND_NCHAR, + FLT_SCL_DATUM_KIND_MAX, +} SFltSclDatumKind; + +typedef struct { + SFltSclDatumKind kind; + union { + int64_t i; // for int and bool (1 true, 0 false) and ts + uint64_t u; // for uint + double d; // for double + uint8_t *pData; // for varchar, nchar, len prefixed + }; + SDataType type; // TODO: original data type, may not be used? +} SFltSclDatum; + +typedef struct { + SFltSclDatum val; + bool excl; + bool start; +} SFltSclPoint; + +int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2->kind == float64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Double(&val1->u, &val2->d); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Double(&val1->i, &val2->d); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleVal(&val1->d, &val2->d); + } + // TODO: varchar, nchar + default: + qError("not supported comparsion. kind1 %d, kind2 %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2->kind == int64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Int64(&val1->u, &val2->i); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Val(&val1->i, &val2->i); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleInt64(&val1->d, &val2->i); + } + // TODO: varchar, nchar + default: + qError("not supported comparsion. kind1 %d, kind2 %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2 kind == uint64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Val(&val1->u, &val2->u); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Uint64(&val1->i, &val2->u); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleUint64(&val1->d, &val2->u); + } + // TODO: varchar, nchar + default: + qError("not supported comparsion. kind1 %d, kind2 %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { + if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || + val2->kind == FLT_SCL_DATUM_KIND_MAX) { + return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); + } + + switch (val2->kind) { + case FLT_SCL_DATUM_KIND_UINT64: { + return fltSclCompareWithUInt64(val1, val2); + } + case FLT_SCL_DATUM_KIND_INT64: { + return fltSclCompareWithInt64(val1, val2); + } + case FLT_SCL_DATUM_KIND_FLOAT64: { + return fltSclCompareWithFloat64(val1, val2); + } + // TODO: varchar/nchar + default: + qError("not supported kind when compare datum. kind2 : %d", val2->kind); + return 0; + break; + } + return 0; +} + +bool fltSclLessPoint(SFltSclPoint *pt1, SFltSclPoint *pt2) { + // first value compare + int32_t cmp = fltSclCompareDatum(&pt1->val, &pt2->val); + if (cmp != 0) { + return cmp < 0; + } + + if (pt1->start && pt2->start) { + return !pt1->excl && pt2->excl; + } else if (pt1->start) { + return !pt1->excl && !pt2->excl; + } else if (pt2->start) { + return pt1->excl || pt2->excl; + } + return pt1->excl && !pt2->excl; +} + +int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { + size_t len1 = taosArrayGetSize(pts1); + size_t len2 = taosArrayGetSize(pts2); + size_t i = 0; + size_t j = 0; + while (i < len1 && j < len2) { + SFltSclPoint *pt1 = taosArrayGet(pts1, i); + SFltSclPoint *pt2 = taosArrayGet(pts2, j); + bool less = fltSclLessPoint(pt1, pt2); + if (less) { + taosArrayPush(result, pt1); + ++i; + } else { + taosArrayPush(result, pt2); + ++j; + } + } + if (i < len1) { + for (; i < len1; ++i) { + SFltSclPoint *pt1 = taosArrayGet(pts1, i); + taosArrayPush(result, pt1); + } + } + if (j < len2) { + for (; j < len2; ++j) { + SFltSclPoint *pt2 = taosArrayGet(pts2, j); + taosArrayPush(result, pt2); + } + } + return 0; +} + +int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { + size_t len1 = taosArrayGetSize(pts1); + size_t len2 = taosArrayGetSize(pts2); + // first merge sort pts1 and pts2 + SArray *all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); + fltSclMergeSort(pts1, pts2, all); + int32_t countRequired = (isUnion) ? 1 : 2; + int32_t count = 0; + for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { + SFltSclPoint *pt = taosArrayGet(all, i); + if (pt->start) { + ++count; + if (count == countRequired) { + taosArrayPush(merged, pt); + } + } else { + if (count == countRequired) { + taosArrayPush(merged, pt); + } + --count; + } + } + taosArrayDestroy(all); + return 0; +} + +int32_t fltSclIntersect(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, false, merged); } + +int32_t fltSclUnion(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, true, merged); } + +typedef struct { + SColumnNode *colNode; + SValueNode *valNode; + EOperatorType type; +} SFltSclOperator; + +SFltSclColumnRange *fltSclGetOrCreateColumnRange(SColumnNode *colNode, SArray *colRangeList) { + for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { + SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i); + if (nodesEqualNode((SNode *)colRange->colNode, (SNode *)colNode)) { + return colRange; + } + } + SColumnNode *pColumnNode = (SColumnNode *)nodesCloneNode((SNode *)colNode); + SFltSclColumnRange newColRange = {.colNode = pColumnNode, .points = taosArrayInit(4, sizeof(SFltSclPoint))}; + taosArrayPush(colRangeList, &newColRange); + return taosArrayGetLast(colRangeList); +} + +int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) { + datum->type = valNode->node.resType; + + if (valNode->isNull) { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + } else { + switch (valNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + break; + } + case TSDB_DATA_TYPE_BOOL: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->i = (valNode->datum.b) ? 0 : 1; + break; + } + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->i = valNode->datum.i; + break; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: { + datum->kind = FLT_SCL_DATUM_KIND_UINT64; + datum->u = valNode->datum.u; + break; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; + datum->d = valNode->datum.d; + break; + } + // TODO:varchar/nchar/json + default: { + qError("not supported type %d when build datum from value node", valNode->node.resType.type); + break; + } + } + } + return TSDB_CODE_SUCCESS; +} + +int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int64_t val) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->i = val; + break; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: { + datum->kind = FLT_SCL_DATUM_KIND_UINT64; + datum->u = *(uint64_t *)&val; + break; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; + datum->d = *(double *)&val; + break; + } + // TODO:varchar/nchar/json + default: { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + qError("not supported type %d when build datum from block sma value", type); + break; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange *colRange, SColumnDataAgg *pAgg, int32_t numOfRows, + SArray *points) { + if (pAgg->numOfNull == numOfRows) { + SFltSclDatum datum = {.kind = FLT_SCL_DATUM_KIND_NULL}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = datum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = datum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + return TSDB_CODE_SUCCESS; + } + if (pAgg->numOfNull > 0) { + SFltSclDatum nullDatum = {.kind = FLT_SCL_DATUM_KIND_NULL}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = nullDatum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = nullDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } + SFltSclDatum min = {0}; + fltSclBuildDatumFromBlockSmaValue(&min, colRange->colNode->node.resType.type, pAgg->min); + SFltSclPoint minPt = {.excl = false, .start = true, .val = min}; + SFltSclDatum max = {0}; + fltSclBuildDatumFromBlockSmaValue(&max, colRange->colNode->node.resType.type, pAgg->max); + SFltSclPoint maxPt = {.excl = false, .start = false, .val = max}; + taosArrayPush(points, &minPt); + taosArrayPush(points, &maxPt); + return TSDB_CODE_SUCCESS; +} + bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t numOfCols, int32_t numOfRows) { if (info->scalarMode) { + SArray *colRanges = info->sclCtx.fltSclRange; + for (int32_t i = 0; i < taosArrayGetSize(colRanges); ++i) { + SFltSclColumnRange *colRange = taosArrayGet(colRanges, i); + bool foundCol = false; + int32_t j = 0; + for (; j < numOfCols; ++j) { + if (pDataStatis[j] != NULL && pDataStatis[j]->colId == colRange->colNode->colId) { + foundCol = true; + break; + } + } + if (foundCol) { + SColumnDataAgg *pAgg = pDataStatis[j]; + SArray *points = taosArrayInit(2, sizeof(SFltSclPoint)); + fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); + qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min %" PRId64, pAgg->numOfNull, numOfRows, + pAgg->max, pAgg->min); + + SArray *merged = taosArrayInit(8, sizeof(SFltSclPoint)); + fltSclIntersect(points, colRange->points, merged); + bool isIntersect = taosArrayGetSize(merged) != 0; + qDebug("filter range execute, scalar mode, column range found. colId: %d colName: %s has overlap: %d", + colRange->colNode->colId, colRange->colNode->colName, isIntersect); + + taosArrayDestroy(merged); + taosArrayDestroy(points); + if (!isIntersect) { + return false; + } + } + } return true; } @@ -3604,6 +3969,31 @@ _return: return code; } +static int32_t fltSclGetDatumValueFromPoint(SFltSclPoint *point, SFltSclDatum *d) { + *d = point->val; + if (point->val.kind == FLT_SCL_DATUM_KIND_NULL) { + return TSDB_CODE_SUCCESS; + } + if (point->val.kind == FLT_SCL_DATUM_KIND_MAX) { + getDataMax(d->type.type, &(d->i)); + } else if (point->val.kind == FLT_SCL_DATUM_KIND_MIN) { + getDataMin(d->type.type, &(d->i)); + } + + if (IS_INTEGER_TYPE(d->type.type) || IS_TIMESTAMP_TYPE(d->type.type)) { + if (point->excl) { + if (point->start) { + ++d->i; + } else { + --d->i; + } + } + } else { + qError("not supported type %d when get datum from point", d->type.type); + } + return TSDB_CODE_SUCCESS; +} + int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { SFilterInfo *info = NULL; int32_t code = 0; @@ -3613,6 +4003,26 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { FLT_ERR_RET(filterInitFromNode(pNode, &info, FLT_OPTION_NO_REWRITE | FLT_OPTION_TIMESTAMP)); if (info->scalarMode) { + SArray *colRanges = info->sclCtx.fltSclRange; + if (taosArrayGetSize(colRanges) == 1) { + SFltSclColumnRange *colRange = taosArrayGet(colRanges, 0); + SArray *points = colRange->points; + if (taosArrayGetSize(points) == 2) { + SFltSclPoint *startPt = taosArrayGet(points, 0); + SFltSclPoint *endPt = taosArrayGet(points, 1); + SFltSclDatum start; + SFltSclDatum end; + fltSclGetDatumValueFromPoint(startPt, &start); + fltSclGetDatumValueFromPoint(endPt, &end); + win->skey = start.i; + win->ekey = end.i; + *isStrict = true; + goto _return; + } else if (taosArrayGetSize(points) == 0) { + *win = TSWINDOW_DESC_INITIALIZER; + goto _return; + } + } *win = TSWINDOW_INITIALIZER; *isStrict = false; goto _return; @@ -3739,7 +4149,8 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { } if (TSDB_DATA_TYPE_BINARY != valueNode->node.resType.type && TSDB_DATA_TYPE_NCHAR != - valueNode->node.resType.type) { return DEAL_RES_CONTINUE; + valueNode->node.resType.type && + TSDB_DATA_TYPE_GEOMETRY != valueNode->node.resType.type) { return DEAL_RES_CONTINUE; } if (stat->precision < 0) { @@ -3884,30 +4295,27 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { return DEAL_RES_ERROR; } + SColumnNode *refNode = (SColumnNode *)node->pLeft; + SExprNode *exprNode = NULL; if (OP_TYPE_IN != node->opType) { - SColumnNode *refNode = (SColumnNode *)node->pLeft; SValueNode *valueNode = (SValueNode *)node->pRight; if (FILTER_GET_FLAG(stat->info->options, FLT_OPTION_TIMESTAMP) && TSDB_DATA_TYPE_UBIGINT == valueNode->node.resType.type && valueNode->datum.u <= INT64_MAX) { valueNode->node.resType.type = TSDB_DATA_TYPE_BIGINT; } - int32_t type = vectorGetConvertType(refNode->node.resType.type, valueNode->node.resType.type); - if (0 != type && type != refNode->node.resType.type) { - stat->scalarMode = true; - return DEAL_RES_CONTINUE; - } + exprNode = &valueNode->node; } else { - SColumnNode *refNode = (SColumnNode *)node->pLeft; SNodeListNode *listNode = (SNodeListNode *)node->pRight; if (LIST_LENGTH(listNode->pNodeList) > 10) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } - int32_t type = vectorGetConvertType(refNode->node.resType.type, listNode->node.resType.type); - if (0 != type && type != refNode->node.resType.type) { - stat->scalarMode = true; - return DEAL_RES_CONTINUE; - } + exprNode = &listNode->node; + } + int32_t type = vectorGetConvertType(refNode->node.resType.type, exprNode->resType.type); + if (0 != type && type != refNode->node.resType.type) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; } } @@ -3942,8 +4350,204 @@ _return: FLT_RET(code); } +int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { + switch (oper->type) { + case OP_TYPE_GREATER_THAN: { + SFltSclDatum start; + fltSclBuildDatumFromValueNode(&start, oper->valNode); + SFltSclPoint startPt = {.start = true, .excl = true, .val = start}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_GREATER_EQUAL: { + SFltSclDatum start; + fltSclBuildDatumFromValueNode(&start, oper->valNode); + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_LOWER_THAN: { + SFltSclDatum end; + fltSclBuildDatumFromValueNode(&end, oper->valNode); + SFltSclPoint endPt = {.start = false, .excl = true, .val = end}; + SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_LOWER_EQUAL: { + SFltSclDatum end; + fltSclBuildDatumFromValueNode(&end, oper->valNode); + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_EQUAL: { + SFltSclDatum valDatum; + fltSclBuildDatumFromValueNode(&valDatum, oper->valNode); + SFltSclPoint startPt = {.start = true, .excl = false, .val = valDatum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = valDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_NOT_EQUAL: { + SFltSclDatum valDatum; + fltSclBuildDatumFromValueNode(&valDatum, oper->valNode); + { + SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + SFltSclPoint endPt = {.start = false, .excl = true, .val = valDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } + { + SFltSclPoint startPt = {.start = true, .excl = true, .val = valDatum}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } + break; + } + case OP_TYPE_IS_NULL: { + SFltSclDatum nullDatum = {.kind = FLT_SCL_DATUM_KIND_NULL}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = nullDatum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = nullDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_IS_NOT_NULL: { + SFltSclDatum minDatum = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = minDatum}; + SFltSclDatum maxDatum = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = maxDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + default: { + qError("not supported operator type : %d when build range points", oper->type); + break; + } + } + return TSDB_CODE_SUCCESS; +} + +// TODO: process DNF composed of CNF +int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { + size_t sz = taosArrayGetSize(sclOpListCNF); + for (int32_t i = 0; i < sz; ++i) { + SFltSclOperator *sclOper = taosArrayGet(sclOpListCNF, i); + SFltSclColumnRange *colRange = fltSclGetOrCreateColumnRange(sclOper->colNode, colRangeList); + SArray *points = taosArrayInit(4, sizeof(SFltSclPoint)); + fltSclBuildRangePoints(sclOper, points); + if (taosArrayGetSize(colRange->points) != 0) { + SArray *merged = taosArrayInit(4, sizeof(SFltSclPoint)); + int32_t code = fltSclIntersect(colRange->points, points, merged); + taosArrayDestroy(colRange->points); + taosArrayDestroy(points); + colRange->points = merged; + } else { + taosArrayDestroy(colRange->points); + colRange->points = points; + } + } + return TSDB_CODE_SUCCESS; +} + +static bool fltSclIsCollectableNode(SNode *pNode) { + if (nodeType(pNode) != QUERY_NODE_OPERATOR) { + return false; + } + + SOperatorNode *pOper = (SOperatorNode *)pNode; + if (pOper->pLeft == NULL || pOper->pRight == NULL) { + return false; + } + + if (!(pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || + pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || + pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { + return false; + } + + if (!(nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE)) { + return false; + } + return true; +} + +static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { + if (!fltSclIsCollectableNode(pNode)) { + return TSDB_CODE_SUCCESS; + } + + SOperatorNode *pOper = (SOperatorNode *)pNode; + + SValueNode *valNode = (SValueNode *)pOper->pRight; + if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { + SFltSclOperator sclOp = {.colNode = (SColumnNode *)nodesCloneNode(pOper->pLeft), + .valNode = (SValueNode *)nodesCloneNode(pOper->pRight), + .type = pOper->opType}; + taosArrayPush(sclOpList, &sclOp); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t fltSclCollectOperatorsFromLogicCond(SNode *pNode, SArray *sclOpList) { + if (nodeType(pNode) != QUERY_NODE_LOGIC_CONDITION) { + return TSDB_CODE_SUCCESS; + } + SLogicConditionNode *pLogicCond = (SLogicConditionNode *)pNode; + // TODO: support LOGIC_COND_TYPE_OR + if (pLogicCond->condType != LOGIC_COND_TYPE_AND) { + return TSDB_CODE_SUCCESS; + } + SNode *pExpr = NULL; + FOREACH(pExpr, pLogicCond->pParameterList) { + if (!fltSclIsCollectableNode(pExpr)) { + return TSDB_CODE_SUCCESS; + } + } + FOREACH(pExpr, pLogicCond->pParameterList) { fltSclCollectOperatorFromNode(pExpr, sclOpList); } + return TSDB_CODE_SUCCESS; +} + +static int32_t fltSclCollectOperators(SNode *pNode, SArray *sclOpList) { + if (nodeType(pNode) == QUERY_NODE_OPERATOR) { + fltSclCollectOperatorFromNode(pNode, sclOpList); + } else if (nodeType(pNode) == QUERY_NODE_LOGIC_CONDITION) { + fltSclCollectOperatorsFromLogicCond(pNode, sclOpList); + } + return TSDB_CODE_SUCCESS; +} + int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { - // TODO + SArray *sclOpList = taosArrayInit(16, sizeof(SFltSclOperator)); + fltSclCollectOperators(*pNode, sclOpList); + SArray *colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); + fltSclProcessCNF(sclOpList, colRangeList); + pInfo->sclCtx.fltSclRange = colRangeList; + + for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { + SFltSclOperator *sclOp = taosArrayGet(sclOpList, i); + nodesDestroyNode((SNode *)sclOp->colNode); + nodesDestroyNode((SNode *)sclOp->valNode); + } + taosArrayDestroy(sclOpList); return TSDB_CODE_SUCCESS; } @@ -4017,8 +4621,11 @@ int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pInfo, uint32_t options) stat.info = info; FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); - - info->scalarMode = stat.scalarMode; + if (tsFilterScalarMode) { + info->scalarMode = true; + } else { + info->scalarMode = stat.scalarMode; + } fltDebug("scalar mode: %d", info->scalarMode); if (!info->scalarMode) { @@ -4037,11 +4644,11 @@ _return: FLT_RET(code); } -bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, SColumnDataAgg *statis, int16_t numOfCols, - int32_t *pResultStatus) { +int32_t filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, SColumnDataAgg *statis, + int16_t numOfCols, int32_t *pResultStatus) { if (NULL == info) { *pResultStatus = FILTER_RESULT_ALL_QUALIFIED; - return false; + return TSDB_CODE_SUCCESS; } SScalarParam output = {0}; @@ -4049,7 +4656,7 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, SC int32_t code = sclCreateColumnInfoData(&type, pSrc->info.rows, &output); if (code != TSDB_CODE_SUCCESS) { - return false; + return code; } if (info->scalarMode) { @@ -4070,18 +4677,23 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, SC } else { *pResultStatus = FILTER_RESULT_PARTIAL_QUALIFIED; } - return false; - } else { - *p = output.columnData; - output.numOfRows = pSrc->info.rows; + return TSDB_CODE_SUCCESS; + } - if (*p == NULL) { - return false; - } + ASSERT(false == info->scalarMode); + *p = output.columnData; + output.numOfRows = pSrc->info.rows; - bool keep = (*info->func)(info, pSrc->info.rows, *p, statis, numOfCols, &output.numOfQualified); + if (*p == NULL) { + return TSDB_CODE_APP_ERROR; + } - // todo this should be return during filter procedure + bool keepAll = (*info->func)(info, pSrc->info.rows, *p, statis, numOfCols, &output.numOfQualified); + + // todo this should be return during filter procedure + if (keepAll) { + *pResultStatus = FILTER_RESULT_ALL_QUALIFIED; + } else { int32_t num = 0; for (int32_t i = 0; i < output.numOfRows; ++i) { if (((int8_t *)((*p)->pData))[i] == 1) { @@ -4096,9 +4708,9 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, SC } else { *pResultStatus = FILTER_RESULT_PARTIAL_QUALIFIED; } - - return keep; } + + return TSDB_CODE_SUCCESS; } typedef struct SClassifyConditionCxt { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index d9295656e8c4b882e5af1d735d7399b7dfb5a332..4eb0f0e1bce3e9f21ce0b4891e7c7c65e3c439cd 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1694,7 +1694,8 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { SCL_ERR_JRET(TSDB_CODE_APP_ERROR); } - if (1 == res->numOfRows) { + SSDataBlock *pb = taosArrayGetP(pBlockList, 0); + if (1 == res->numOfRows && pb->info.rows > 0) { SCL_ERR_JRET(sclExtendResRows(pDst, res, pBlockList)); } else { colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows, true); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 1afcf4495840675c428c25f2c0ab4554cd02a882..b9af71692907875fc9702ed22ca8ec0f5d29f03d 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -289,7 +289,7 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP static VarDataLenT tlength(char *input, int32_t type) { return varDataLen(input); } static VarDataLenT tcharlength(char *input, int32_t type) { - if (type == TSDB_DATA_TYPE_VARCHAR) { + if (type == TSDB_DATA_TYPE_VARCHAR || type == TSDB_DATA_TYPE_GEOMETRY) { return varDataLen(input); } else { // NCHAR return varDataLen(input) / TSDB_NCHAR_SIZE; @@ -935,7 +935,8 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } break; } - case TSDB_DATA_TYPE_BINARY: { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_GEOMETRY: { if (inputType == TSDB_DATA_TYPE_BOOL) { // NOTE: sprintf will append '\0' at the end of string int32_t len = sprintf(varDataVal(output), "%.*s", (int32_t)(outputLen - VARSTR_HEADER_SIZE), diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 4d803cb6388c75211dcea9db6a0dc353b91c71aa..0246724c5be21ee9eda89c1da2f2605341ccb1af 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -26,6 +26,7 @@ #include "tdataformat.h" #include "ttime.h" #include "ttypes.h" +#include "geosWrapper.h" #define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void *)pLeftCol : pLeftCol->pData)) #define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void *)pRightCol : pRightCol->pData)) @@ -230,6 +231,8 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { p = getVectorValueAddr_VAR; } else if (srcType == TSDB_DATA_TYPE_NCHAR) { p = getVectorValueAddr_VAR; + }else if(srcType == TSDB_DATA_TYPE_GEOMETRY) { + p = getVectorValueAddr_VAR; } else { p = getVectorValueAddr_default; } @@ -237,15 +240,20 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { } static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { + terrno = TSDB_CODE_SUCCESS; + int64_t value = 0; if (taosParseTime(buf, &value, strlen(buf), pOut->columnData->info.precision, tsDaylight) != TSDB_CODE_SUCCESS) { value = 0; + terrno = TSDB_CODE_SCALAR_CONVERT_ERROR; } colDataSetInt64(pOut->columnData, rowIndex, &value); } static FORCE_INLINE void varToSigned(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { + terrno = TSDB_CODE_SUCCESS; + if (overflow) { int64_t minValue = tDataTypes[pOut->columnData->info.type].minValue; int64_t maxValue = tDataTypes[pOut->columnData->info.type].maxValue; @@ -287,6 +295,8 @@ static FORCE_INLINE void varToSigned(char *buf, SScalarParam *pOut, int32_t rowI } static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { + terrno = TSDB_CODE_SUCCESS; + if (overflow) { uint64_t minValue = (uint64_t)tDataTypes[pOut->columnData->info.type].minValue; uint64_t maxValue = (uint64_t)tDataTypes[pOut->columnData->info.type].maxValue; @@ -327,6 +337,8 @@ static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam *pOut, int32_t ro } static FORCE_INLINE void varToFloat(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { + terrno = TSDB_CODE_SUCCESS; + if (TSDB_DATA_TYPE_FLOAT == pOut->columnData->info.type) { float value = taosStr2Float(buf, NULL); colDataSetFloat(pOut->columnData, rowIndex, &value); @@ -338,6 +350,8 @@ static FORCE_INLINE void varToFloat(char *buf, SScalarParam *pOut, int32_t rowIn } static FORCE_INLINE void varToBool(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { + terrno = TSDB_CODE_SUCCESS; + int64_t value = taosStr2Int64(buf, NULL, 10); bool v = (value != 0) ? true : false; colDataSetInt8(pOut->columnData, rowIndex, (int8_t *)&v); @@ -345,6 +359,8 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam *pOut, int32_t rowInd // todo remove this malloc static FORCE_INLINE void varToNchar(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { + terrno = TSDB_CODE_SUCCESS; + int32_t len = 0; int32_t inputLen = varDataLen(buf); int32_t outputMaxLen = (inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; @@ -354,6 +370,7 @@ static FORCE_INLINE void varToNchar(char *buf, SScalarParam *pOut, int32_t rowIn taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4 *)varDataVal(t), outputMaxLen - VARSTR_HEADER_SIZE, &len); if (!ret) { sclError("failed to convert to NCHAR"); + terrno = TSDB_CODE_SCALAR_CONVERT_ERROR; } varDataSetLen(t, len); @@ -362,11 +379,14 @@ static FORCE_INLINE void varToNchar(char *buf, SScalarParam *pOut, int32_t rowIn } static FORCE_INLINE void ncharToVar(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { + terrno = TSDB_CODE_SUCCESS; + int32_t inputLen = varDataLen(buf); char *t = taosMemoryCalloc(1, inputLen + VARSTR_HEADER_SIZE); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t)); if (len < 0) { + terrno = TSDB_CODE_SCALAR_CONVERT_ERROR; taosMemoryFree(t); return; } @@ -376,8 +396,46 @@ static FORCE_INLINE void ncharToVar(char *buf, SScalarParam *pOut, int32_t rowIn taosMemoryFree(t); } +static FORCE_INLINE void varToGeometry(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { + //[ToDo] support to parse WKB as well as WKT + terrno = TSDB_CODE_SUCCESS; + + size_t len = 0; + unsigned char *t = NULL; + char *output = NULL; + + if (initCtxGeomFromText()) { + sclError("failed to init geometry ctx, %s", getThreadLocalGeosCtx()->errMsg); + terrno = TSDB_CODE_APP_ERROR; + goto _err; + } + if (doGeomFromText(buf, &t, &len)) { + sclInfo("failed to convert text to geometry, %s", getThreadLocalGeosCtx()->errMsg); + terrno = TSDB_CODE_SCALAR_CONVERT_ERROR; + goto _err; + } + + output = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE); + memcpy(output + VARSTR_HEADER_SIZE, t, len); + varDataSetLen(output, len); + + colDataSetVal(pOut->columnData, rowIndex, output, false); + + taosMemoryFree(output); + geosFreeBuffer(t); + + return; + +_err: + ASSERT(t == NULL && len == 0); + VarDataLenT dummyHeader = 0; + colDataSetVal(pOut->columnData, rowIndex, (const char *)&dummyHeader, false); +} + // TODO opt performance, tmp is not needed. int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { + terrno = TSDB_CODE_SUCCESS; + bool vton = false; _bufConverteFunc func = NULL; @@ -399,9 +457,12 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { vton = true; } else if (TSDB_DATA_TYPE_TIMESTAMP == pCtx->outType) { func = varToTimestamp; + } else if (TSDB_DATA_TYPE_GEOMETRY == pCtx->outType) { + func = varToGeometry; } else { sclError("invalid convert outType:%d, inType:%d", pCtx->outType, pCtx->inType); - return TSDB_CODE_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; + return terrno; } pCtx->pOut->numOfRows = pCtx->pIn->numOfRows; @@ -421,7 +482,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { convertType = TSDB_DATA_TYPE_NCHAR; } else if (tTagIsJson(data) || *data == TSDB_DATA_TYPE_NULL) { terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR; - return terrno; + goto _err; } else { convertNumberToNumber(data + CHAR_BYTES, colDataGetNumData(pCtx->pOut->columnData, i), *data, pCtx->outType); continue; @@ -433,29 +494,30 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { tmp = taosMemoryMalloc(bufSize); if (tmp == NULL) { sclError("out of memory in vectorConvertFromVarData"); - return TSDB_CODE_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } } if (vton) { memcpy(tmp, data, varDataTLen(data)); } else { - if (TSDB_DATA_TYPE_VARCHAR == convertType) { + if (TSDB_DATA_TYPE_VARCHAR == convertType || TSDB_DATA_TYPE_GEOMETRY == convertType) { memcpy(tmp, varDataVal(data), varDataLen(data)); tmp[varDataLen(data)] = 0; } else if (TSDB_DATA_TYPE_NCHAR == convertType) { // we need to convert it to native char string, and then perform the string to numeric data if (varDataLen(data) > bufSize) { sclError("castConvert convert buffer size too small"); - taosMemoryFreeClear(tmp); - return TSDB_CODE_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; + goto _err; } int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp); if (len < 0) { sclError("castConvert taosUcs4ToMbs error 1"); - taosMemoryFreeClear(tmp); - return TSDB_CODE_APP_ERROR; + terrno = TSDB_CODE_SCALAR_CONVERT_ERROR; + goto _err; } tmp[len] = 0; @@ -463,12 +525,16 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { } (*func)(tmp, pCtx->pOut, i, overflow); + if (terrno != TSDB_CODE_SUCCESS) { + goto _err; + } } +_err: if (tmp != NULL) { taosMemoryFreeClear(tmp); } - return TSDB_CODE_SUCCESS; + return terrno; } double getVectorDoubleValue_JSON(void *src, int32_t index) { @@ -531,7 +597,7 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t } if (optr == OP_TYPE_LIKE || optr == OP_TYPE_NOT_LIKE || optr == OP_TYPE_MATCH || optr == OP_TYPE_NMATCH) { - if (typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY) { + if (typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY && typeLeft != TSDB_DATA_TYPE_GEOMETRY) { return false; } } @@ -560,7 +626,8 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t if (IS_NUMERIC_TYPE(type)) { if (typeLeft == TSDB_DATA_TYPE_NCHAR || - typeLeft == TSDB_DATA_TYPE_VARCHAR) { + typeLeft == TSDB_DATA_TYPE_VARCHAR || + typeLeft == TSDB_DATA_TYPE_GEOMETRY) { return false; } else if (typeLeft != type) { convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type); @@ -568,13 +635,14 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t } if (typeRight == TSDB_DATA_TYPE_NCHAR || - typeRight == TSDB_DATA_TYPE_VARCHAR) { + typeRight == TSDB_DATA_TYPE_VARCHAR || + typeRight == TSDB_DATA_TYPE_GEOMETRY) { return false; } else if (typeRight != type) { convertNumberToNumber(*pRightData, pRightOut, typeRight, type); *pRightData = pRightOut; } - } else if (type == TSDB_DATA_TYPE_BINARY) { + } else if (type == TSDB_DATA_TYPE_BINARY || typeLeft == TSDB_DATA_TYPE_GEOMETRY) { if (typeLeft == TSDB_DATA_TYPE_NCHAR) { *pLeftData = ncharTobinary(*pLeftData); *freeLeft = true; @@ -864,7 +932,8 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, break; } case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: { return vectorConvertToVarData(&cCtx); } default: @@ -875,27 +944,29 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, return TSDB_CODE_SUCCESS; } -int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB + 1][TSDB_DATA_TYPE_BLOB + 1] = { - /* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB */ - /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 7, 11, 12, 13, 14, 0, 7, 0, 0, - /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, - /*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, - /*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 7, 4, 4, 5, 7, 0, 7, 0, 0, - /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 7, 5, 5, 5, 7, 0, 7, 0, 0, - /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, - /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, - /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 0, 0, 0, - /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, 7, 0, 0, - /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, - /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, - /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0, - /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 0, - /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, - /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +int8_t gConvertTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = { + /* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/ + /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 7, 11, 12, 13, 14, 0, 7, 0, 0, 0, -1, + /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, 0, -1, + /*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, 0, -1, + /*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 7, 4, 4, 5, 7, 0, 7, 0, 0, 0, -1, + /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 7, 5, 5, 5, 7, 0, 7, 0, 0, 0, -1, + /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, 0, -1, + /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, 0, -1, + /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 0, 0, 0, 0, 20, + /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, 7, 0, 0, 0, -1, + /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, 0, -1, + /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, 0, -1, + /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0, 0, -1, + /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 0, 0, -1, + /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, -1, + /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, + /*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, + /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, + /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, + /*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, + /*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; int32_t vectorGetConvertType(int32_t type1, int32_t type2) { if (type1 == type2) { @@ -975,6 +1046,11 @@ int32_t vectorConvertCols(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara if (0 == type) { return TSDB_CODE_SUCCESS; } + if (-1 == type) { + sclError("invalid convert type1:%d, type2:%d", GET_PARAM_TYPE(param1), GET_PARAM_TYPE(param2)); + terrno = TSDB_CODE_SCALAR_CONVERT_ERROR; + return TSDB_CODE_SCALAR_CONVERT_ERROR; + } } if (type != GET_PARAM_TYPE(param1)) { @@ -1718,7 +1794,9 @@ void vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam * param1 = pLeft; param2 = pRight; } else { - vectorConvertCols(pLeft, pRight, &pLeftOut, &pRightOut, startIndex, numOfRows); + if (vectorConvertCols(pLeft, pRight, &pLeftOut, &pRightOut, startIndex, numOfRows)) { + return; + } param1 = (pLeftOut.columnData != NULL) ? &pLeftOut : pLeft; param2 = (pRightOut.columnData != NULL) ? &pRightOut : pRight; } @@ -1784,7 +1862,11 @@ void vectorNotMatch(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOu void vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { for (int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 1 : 0; + if (v) { + ++pOut->numOfQualified; + } colDataSetInt8(pOut->columnData, i, &v); + colDataClearNull_f(pOut->columnData->nullbitmap, i); } pOut->numOfRows = pLeft->numOfRows; } @@ -1792,7 +1874,11 @@ void vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, void vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { for (int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 0 : 1; + if (v) { + ++pOut->numOfQualified; + } colDataSetInt8(pOut->columnData, i, &v); + colDataClearNull_f(pOut->columnData->nullbitmap, i); } pOut->numOfRows = pLeft->numOfRows; } @@ -1805,6 +1891,13 @@ void vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, colDataSetInt8(pOut->columnData, i, &v); colDataClearNull_f(pOut->columnData->nullbitmap, i); } + { + bool v = false; + GET_TYPED_DATA(v, bool, pOut->columnData->info.type, colDataGetData(pOut->columnData, i)); + if (v) { + ++pOut->numOfQualified; + } + } } pOut->columnData->hasNull = false; } @@ -1844,7 +1937,9 @@ void vectorJsonContains(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam char *pLeftData = colDataGetVarData(pLeft->columnData, i); getJsonValue(pLeftData, jsonKey, &isExist); } - + if (isExist) { + ++pOut->numOfQualified; + } colDataSetVal(pOutputCol, i, (const char *)(&isExist), false); } taosMemoryFree(jsonKey); diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index 3288120b67518aa532db7579a7677086899514c7..fafc2a27e0f48740926d250d47d5f34fc88e203c 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories( target_link_libraries( scheduler - PUBLIC os util nodes planner qcom common catalog transport command qworker executor + PUBLIC os util nodes planner qcom common catalog transport command qworker executor index ) if(${BUILD_TEST}) diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 7840fe2017a6f384e70f1553105c7293a4abb2d7..aecf3d5d916681687986ec1771d7f02e79304a8b 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -57,7 +57,7 @@ typedef enum { #define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ #define SCHEDULE_DEFAULT_MAX_NODE_NUM 20 -#define SCH_DEFAULT_TASK_TIMEOUT_USEC 60000000 +#define SCH_DEFAULT_TASK_TIMEOUT_USEC 5000000 #define SCH_MAX_TASK_TIMEOUT_USEC 300000000 #define SCH_DEFAULT_MAX_RETRY_NUM 6 #define SCH_MIN_AYSNC_EXEC_NUM 3 @@ -239,7 +239,7 @@ typedef struct SSchTask { int32_t lastMsgType; // last sent msg type int64_t timeoutUsec; // task timeout useconds before reschedule SQueryNodeAddr succeedAddr; // task executed success node address - int8_t candidateIdx; // current try condidation index + int32_t candidateIdx; // current try condidation index SArray *candidateAddrs; // condidate node addresses, element is SQueryNodeAddr SHashObj *execNodes; // all tried node for current task, element is SSchNodeInfo SSchTaskProfile profile; // task execution profile diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index e7bfe95795f44f61ee4d1cc02873a21b80438369..78e08077753da5a6d090ff3c8a1938ff6e55cafc 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -135,6 +135,7 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { break; case JOB_TASK_STATUS_DROP: SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); + break; default: SCH_JOB_ELOG("invalid job status:%s", jobTaskStatusStr(oriStatus)); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 80fdc7594cc63b8e893de9e4e7cf193592deb885..01b4e7e9e67b563caa34507b42dde9b2c9e913cf 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -392,6 +392,7 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD // NEVER REACH HERE SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:0x%" PRIx64, pJob->refId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + break; } case TDMT_SCH_LINK_BROKEN: SCH_TASK_ELOG("link broken received, error:%x - %s", rspCode, tstrerror(rspCode)); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 207753ae2571fe292ca009cb6af51030bf0667f0..9985e7d6a1260a7c6f8220022f85bbdbaa89e25e 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -745,7 +745,6 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - pTask->candidateIdx = 0; pTask->candidateAddrs = taosArrayInit(SCHEDULE_DEFAULT_MAX_NODE_NUM, sizeof(SQueryNodeAddr)); if (NULL == pTask->candidateAddrs) { SCH_TASK_ELOG("taosArrayInit %d condidate addrs failed", SCHEDULE_DEFAULT_MAX_NODE_NUM); @@ -765,11 +764,13 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_DATA_BIND_TASK(pTask)) { SCH_TASK_ELOG("no execNode specifed for data src task, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps); - SCH_ERR_RET(TSDB_CODE_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_MND_INVALID_SCHEMA_VER); } SCH_ERR_RET(schSetAddrsFromNodeList(pJob, pTask)); + pTask->candidateIdx = taosRand() % taosArrayGetSize(pTask->candidateAddrs); + /* for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); @@ -961,7 +962,6 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { localRsp->rsp.numOfPlans = 0; localRsp->rsp.subplanInfo = NULL; pTask = NULL; - pJob = NULL; } _return: diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index e7561ccb7ecf68c5030e12a19c04f3d1a9661cb5..841066a4c9caebc3c69ea86a8fc8533f17289999 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -35,7 +35,7 @@ int32_t schedulerInit() { schMgmt.cfg.schPolicy = SCHEDULE_DEFAULT_POLICY; schMgmt.cfg.enableReSchedule = true; - qDebug("schedule init, policy: %d, maxNodeTableNum: %" PRId64", reSchedule:%d", + qDebug("schedule init, policy: %d, maxNodeTableNum: %" PRId64", reSchedule:%d", schMgmt.cfg.schPolicy, schMgmt.cfg.maxNodeTableNum, schMgmt.cfg.enableReSchedule); schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl); @@ -57,11 +57,11 @@ int32_t schedulerInit() { } if (taosGetSystemUUID((char *)&schMgmt.sId, sizeof(schMgmt.sId))) { - qError("generate schdulerId failed, errno:%d", errno); + qError("generate schedulerId failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_SYS_ERROR); } - qInfo("scheduler 0x%" PRIx64 " initizlized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum); + qInfo("scheduler 0x%" PRIx64 " initialized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/test/CMakeLists.txt b/source/libs/scheduler/test/CMakeLists.txt index ce928862215d3efe7dc14c1c50e581416cb18777..703bd5932b2301d0006960f94a0131f792501c43 100644 --- a/source/libs/scheduler/test/CMakeLists.txt +++ b/source/libs/scheduler/test/CMakeLists.txt @@ -7,10 +7,18 @@ IF(NOT TD_DARWIN) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(schedulerTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES( - schedulerTest - PUBLIC os util common catalog transport gtest qcom taos_static planner scheduler - ) + + IF (TD_GRANT) + TARGET_LINK_LIBRARIES( + schedulerTest + PUBLIC os util common catalog transport gtest qcom taos_static planner scheduler grant + ) + ELSE () + TARGET_LINK_LIBRARIES( + schedulerTest + PUBLIC os util common catalog transport gtest qcom taos_static planner scheduler + ) + ENDIF() TARGET_INCLUDE_DIRECTORIES( schedulerTest diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index 2edbc44aaeb8e87c0dc51971a4c92c292549a87b..b63a8b39009187714d44d95f353ba4baa7cf8d6d 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -6,18 +6,45 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -if(${BUILD_WITH_ROCKSDB}) - target_link_libraries( - stream - PUBLIC rocksdb tdb - PRIVATE os util transport qcom executor wal - ) - - target_include_directories( - stream - PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include" - ) +if(${BUILD_WITH_ROCKSDB}) + if (${BUILD_CONTRIB}) + target_link_libraries( + stream + PUBLIC rocksdb tdb + PRIVATE os util transport qcom executor wal index + ) + target_include_directories( + stream + PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include" + ) + else() + if (TD_LINUX) + target_include_directories( + stream + PUBLIC "${TD_SOURCE_DIR}/deps/${TD_DEPS_DIR}/rocksdb_static" + ) + target_link_directories( + stream + PUBLIC "${TD_SOURCE_DIR}/deps/${TD_DEPS_DIR}/rocksdb_static" + ) + target_link_libraries( + stream + PUBLIC rocksdb tdb + PRIVATE os util transport qcom executor wal index + ) + else() + target_link_libraries( + stream + PUBLIC rocksdb tdb + PRIVATE os util transport qcom executor wal index + ) + target_include_directories( + stream + PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include" + ) + endif() + endif() add_definitions(-DUSE_ROCKSDB) endif(${BUILD_WITH_ROCKSDB}) diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 0f39cf817b1bbd191d9ab49d3456be9e1bfa1c66..b6bc9c888bec1136f82e13d5f072c0acbed9d937 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -17,7 +17,6 @@ #define _STREAM_BACKEDN_ROCKSDB_H_ #include "rocksdb/c.h" -// #include "streamInc.h" #include "streamState.h" #include "tcoding.h" #include "tcommon.h" @@ -43,10 +42,11 @@ typedef struct { TdThreadMutex cfMutex; SHashObj* cfInst; int64_t defaultCfInit; -} SBackendHandle; +} SBackendWrapper; void* streamBackendInit(const char* path); void streamBackendCleanup(void* arg); +void streamBackendHandleCleanup(void* arg); SListNode* streamBackendAddCompare(void* backend, void* arg); void streamBackendDelCompare(void* backend, void* arg); @@ -123,12 +123,17 @@ char* streamDefaultIterKey_rocksdb(void* iter, int32_t* len); char* streamDefaultIterVal_rocksdb(void* iter, int32_t* len); // batch func +int streamStateGetCfIdx(SStreamState* pState, const char* funcName); void* streamStateCreateBatch(); int32_t streamStateGetBatchSize(void* pBatch); void streamStateClearBatch(void* pBatch); void streamStateDestroyBatch(void* pBatch); int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_writebatch_t* pBatch, void* key, void* val, int32_t vlen, int64_t ttl); + +int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb_writebatch_t* pBatch, void* key, + void* val, int32_t vlen, int64_t ttl, void* tmpBuf); + int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch); // int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); #endif \ No newline at end of file diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInt.h similarity index 56% rename from source/libs/stream/inc/streamInc.h rename to source/libs/stream/inc/streamInt.h index c471bc2bd8f41723067de1d0ae98d504aa4a1aa9..32d6dc65d93ab1ddf1a6e383645652b70110f351 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInt.h @@ -31,24 +31,42 @@ typedef struct { void* timer; } SStreamGlobalEnv; -static SStreamGlobalEnv streamEnv; +typedef struct { + SEpSet epset; + int32_t taskId; + SRpcMsg msg; +} SStreamContinueExecInfo; -int32_t streamDispatch(SStreamTask* pTask); -int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData); -int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData); -int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* data); +extern SStreamGlobalEnv streamEnv; + +void streamRetryDispatchStreamBlock(SStreamTask* pTask, int64_t waitDuration); +int32_t streamDispatchStreamBlock(SStreamTask* pTask); + +SStreamDataBlock* createStreamDataFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg); +SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, + SArray* pRes); +void destroyStreamDataBlock(SStreamDataBlock* pBlock); +int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData); int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock); int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); +int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pData); int32_t streamDispatchCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet); -int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, - SEpSet* pEpSet); +int32_t streamDoDispatchScanHistoryFinishMsg(SStreamTask* pTask, const SStreamScanHistoryFinishReq* pReq, int32_t vgId, + SEpSet* pEpSet); SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem); +int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo, SStreamScanHistoryFinishReq* pReq); +int32_t streamNotifyUpstreamContinue(SStreamTask* pTask); +int32_t streamTaskFillHistoryFinished(SStreamTask* pTask); + +extern int32_t streamBackendId; +extern int32_t streamBackendCfWrapperId; + #ifdef __cplusplus } #endif diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 635024519e8cb665e08a05ddf665638033edda3a..f85ade591ca48f5403c4c1e0ee59f87a5760462a 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -13,11 +13,15 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" #include "ttimer.h" -#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 20480 -#define STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE (100) +#define STREAM_TASK_INPUT_QUEUE_CAPACITY 20480 +#define STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE (30) +#define ONE_MB_F (1048576.0) +#define QUEUE_MEM_SIZE_IN_MB(_q) (taosQueueMemorySize(_q) / ONE_MB_F) + +SStreamGlobalEnv streamEnv; int32_t streamInit() { int8_t old; @@ -34,6 +38,7 @@ int32_t streamInit() { } atomic_store_8(&streamEnv.inited, 1); } + return 0; } @@ -50,107 +55,121 @@ void streamCleanUp() { } } -void streamSchedByTimer(void* param, void* tmrId) { +char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { + char buf[128] = {0}; + sprintf(buf, "0x%" PRIx64 "-0x%x", streamId, taskId); + return taosStrdup(buf); +} + +static void streamSchedByTimer(void* param, void* tmrId) { SStreamTask* pTask = (void*)param; + int8_t status = atomic_load_8(&pTask->triggerStatus); + qDebug("s-task:%s in scheduler, trigger status:%d, next:%dms", pTask->id.idStr, status, (int32_t)pTask->triggerParam); + if (streamTaskShouldStop(&pTask->status) || streamTaskShouldPause(&pTask->status)) { streamMetaReleaseTask(NULL, pTask); + qDebug("s-task:%s jump out of schedTimer", pTask->id.idStr); return; } - if (atomic_load_8(&pTask->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) { - SStreamTrigger* trigger = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM, 0); - if (trigger == NULL) return; - trigger->type = STREAM_INPUT__GET_RES; - trigger->pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - if (trigger->pBlock == NULL) { - taosFreeQitem(trigger); + if (status == TASK_TRIGGER_STATUS__ACTIVE) { + SStreamTrigger* pTrigger = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM, 0); + if (pTrigger == NULL) { return; } - trigger->pBlock->info.type = STREAM_GET_ALL; - atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE); + pTrigger->type = STREAM_INPUT__GET_RES; + pTrigger->pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + if (pTrigger->pBlock == NULL) { + taosFreeQitem(pTrigger); + return; + } - if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)trigger) < 0) { - taosFreeQitem(trigger); - taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); + atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE); + pTrigger->pBlock->info.type = STREAM_GET_ALL; + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pTrigger) < 0) { + taosFreeQitem(pTrigger); + taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->schedTimer); return; } streamSchedExec(pTask); } - taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); + taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->schedTimer); } -int32_t streamSetupTrigger(SStreamTask* pTask) { - if (pTask->triggerParam != 0) { +int32_t streamSetupScheduleTrigger(SStreamTask* pTask) { + if (pTask->triggerParam != 0 && pTask->info.fillHistory == 0) { int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1); - ASSERT(ref == 2); - pTask->timer = taosTmrStart(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer); + ASSERT(ref == 2 && pTask->schedTimer == NULL); + + qDebug("s-task:%s setup scheduler trigger, delay:%"PRId64" ms", pTask->id.idStr, pTask->triggerParam); + + pTask->schedTimer = taosTmrStart(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer); pTask->triggerStatus = TASK_TRIGGER_STATUS__INACTIVE; } + return 0; } int32_t streamSchedExec(SStreamTask* pTask) { - int8_t schedStatus = - atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__WAITING); + int8_t schedStatus = atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE, + TASK_SCHED_STATUS__WAITING); if (schedStatus == TASK_SCHED_STATUS__INACTIVE) { SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); if (pRunReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + qError("failed to create msg to aunch s-task:%s, reason out of memory", pTask->id.idStr); return -1; } - pRunReq->head.vgId = pTask->nodeId; + pRunReq->head.vgId = pTask->info.nodeId; pRunReq->streamId = pTask->id.streamId; pRunReq->taskId = pTask->id.taskId; - SRpcMsg msg = { .msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq) }; - tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &msg); qDebug("trigger to run s-task:%s", pTask->id.idStr); + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &msg); + } else { + qDebug("s-task:%s not launch task since sched status:%d", pTask->id.idStr, pTask->status.schedStatus); } return 0; } int32_t streamTaskEnqueueBlocks(SStreamTask* pTask, const SStreamDispatchReq* pReq, SRpcMsg* pRsp) { - SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); - int8_t status; + int8_t status = 0; - // enqueue data block - if (pData != NULL) { - pData->type = STREAM_INPUT__DATA_BLOCK; - pData->srcVgId = pReq->dataSrcVgId; - // decode - /*pData->blocks = pReq->data;*/ - /*pBlock->sourceVer = pReq->sourceVer;*/ - streamDispatchReqToData(pReq, pData); - if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pData) == 0) { - status = TASK_INPUT_STATUS__NORMAL; - } else { // input queue is full, upstream is blocked now - status = TASK_INPUT_STATUS__BLOCKED; - } - } else { + SStreamDataBlock* pBlock = createStreamDataFromDispatchMsg(pReq, STREAM_INPUT__DATA_BLOCK, pReq->dataSrcVgId); + if (pBlock == NULL) { streamTaskInputFail(pTask); status = TASK_INPUT_STATUS__FAILED; + qError("vgId:%d, s-task:%s failed to receive dispatch msg, reason: out of memory", pTask->pMeta->vgId, + pTask->id.idStr); + } else { + int32_t code = tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pBlock); + // input queue is full, upstream is blocked now + status = (code == TSDB_CODE_SUCCESS)? TASK_INPUT_STATUS__NORMAL:TASK_INPUT_STATUS__BLOCKED; } // rsp by input status void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp)); ((SMsgHead*)buf)->vgId = htonl(pReq->upstreamNodeId); - SStreamDispatchRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead)); - pCont->inputStatus = status; - pCont->streamId = htobe64(pReq->streamId); - pCont->upstreamNodeId = htonl(pReq->upstreamNodeId); - pCont->upstreamTaskId = htonl(pReq->upstreamTaskId); - pCont->downstreamNodeId = htonl(pTask->nodeId); - pCont->downstreamTaskId = htonl(pTask->id.taskId); - pRsp->pCont = buf; + SStreamDispatchRsp* pDispatchRsp = POINTER_SHIFT(buf, sizeof(SMsgHead)); + pDispatchRsp->inputStatus = status; + pDispatchRsp->streamId = htobe64(pReq->streamId); + pDispatchRsp->upstreamNodeId = htonl(pReq->upstreamNodeId); + pDispatchRsp->upstreamTaskId = htonl(pReq->upstreamTaskId); + pDispatchRsp->downstreamNodeId = htonl(pTask->info.nodeId); + pDispatchRsp->downstreamTaskId = htonl(pTask->id.taskId); + + pRsp->pCont = buf; pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); tmsgSendRsp(pRsp); @@ -163,21 +182,18 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, // enqueue if (pData != NULL) { - qDebug("task %d(child %d) recv retrieve req from task %d, reqId %" PRId64, pTask->id.taskId, pTask->selfChildId, - pReq->srcTaskId, pReq->reqId); + qDebug("s-task:%s (child %d) recv retrieve req from task:0x%x(vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr, pTask->info.selfChildId, + pReq->srcTaskId, pReq->srcNodeId, pReq->reqId); pData->type = STREAM_INPUT__DATA_RETRIEVE; pData->srcVgId = 0; - // decode - /*pData->blocks = pReq->data;*/ - /*pBlock->sourceVer = pReq->sourceVer;*/ streamRetrieveReqToData(pReq, pData); if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pData) == 0) { status = TASK_INPUT_STATUS__NORMAL; } else { status = TASK_INPUT_STATUS__FAILED; } - } else { + } else { // todo handle oom /*streamTaskInputFail(pTask);*/ /*status = TASK_INPUT_STATUS__FAILED;*/ } @@ -192,33 +208,36 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, pRsp->pCont = buf; pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp); tmsgSendRsp(pRsp); + return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1; } -int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) { +// todo add log +int32_t streamTaskOutputResultBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { int32_t code = 0; - if (pTask->outputType == TASK_OUTPUT__TABLE) { + int32_t type = pTask->outputInfo.type; + if (type == TASK_OUTPUT__TABLE) { pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks); - taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); - taosFreeQitem(pBlock); - } else if (pTask->outputType == TASK_OUTPUT__SMA) { + destroyStreamDataBlock(pBlock); + } else if (type == TASK_OUTPUT__SMA) { pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks); - taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); - taosFreeQitem(pBlock); + destroyStreamDataBlock(pBlock); } else { - ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH); - code = taosWriteQitem(pTask->outputQueue->queue, pBlock); - if (code != 0) { + ASSERT(type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH); + code = taosWriteQitem(pTask->outputInfo.queue->queue, pBlock); + if (code != 0) { // todo failed to add it into the output queue, free it. return code; } - streamDispatch(pTask); + + streamDispatchStreamBlock(pTask); } + return 0; } -int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp, bool exec) { - qDebug("s-task:%s receive dispatch msg from taskId:%d(vgId:%d)", pTask->id.idStr, pReq->upstreamTaskId, - pReq->upstreamNodeId); +int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp, bool exec) { + qDebug("s-task:%s receive dispatch msg from taskId:0x%x(vgId:%d), msgLen:%" PRId64, pTask->id.idStr, + pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen); // todo add the input queue buffer limitation streamTaskEnqueueBlocks(pTask, pReq, pRsp); @@ -235,28 +254,68 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S return 0; } +// todo record the idle time for dispatch data int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) { - ASSERT(pRsp->inputStatus == TASK_OUTPUT_STATUS__NORMAL || pRsp->inputStatus == TASK_OUTPUT_STATUS__BLOCKED); - qDebug("s-task:%s receive dispatch rsp, code: %x", pTask->id.idStr, code); + if (code != TSDB_CODE_SUCCESS) { + // dispatch message failed: network error, or node not available. + // in case of the input queue is full, the code will be TSDB_CODE_SUCCESS, the and pRsp>inputStatus will be set + // flag. here we need to retry dispatch this message to downstream task immediately. handle the case the failure + // happened too fast. todo handle the shuffle dispatch failure + if (code == TSDB_CODE_STREAM_TASK_NOT_EXIST) { + qError("s-task:%s failed to dispatch msg to task:0x%x, code:%s, no-retry", pTask->id.idStr, + pRsp->downstreamTaskId, tstrerror(code)); + return code; + } else { + qError("s-task:%s failed to dispatch msg to task:0x%x, code:%s, retry cnt:%d", pTask->id.idStr, + pRsp->downstreamTaskId, tstrerror(code), ++pTask->msgInfo.retryCount); + return streamDispatchAllBlocks(pTask, pTask->msgInfo.pData); + } + } + + qDebug("s-task:%s receive dispatch rsp, output status:%d code:%d", pTask->id.idStr, pRsp->inputStatus, code); - if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + // there are other dispatch message not response yet + if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); - qDebug("task %d is shuffle, left waiting rsp %d", pTask->id.taskId, leftRsp); + qDebug("s-task:%s is shuffle, left waiting rsp %d", pTask->id.idStr, leftRsp); if (leftRsp > 0) { return 0; } } - int8_t old = atomic_exchange_8(&pTask->outputStatus, pRsp->inputStatus); - ASSERT(old == TASK_OUTPUT_STATUS__WAIT); + pTask->msgInfo.retryCount = 0; + ASSERT(pTask->outputInfo.status == TASK_OUTPUT_STATUS__WAIT); + + qDebug("s-task:%s output status is set to:%d", pTask->id.idStr, pTask->outputInfo.status); + + // the input queue of the (down stream) task that receive the output data is full, + // so the TASK_INPUT_STATUS_BLOCKED is rsp + // todo blocking the output status if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { - // TODO: init recover timer - ASSERT(0); - return 0; + pTask->msgInfo.blockingTs = taosGetTimestampMs(); // record the blocking start time + + int32_t waitDuration = 300; // 300 ms + qError("s-task:%s inputQ of downstream task:0x%x is full, time:%" PRId64 "wait for %dms and retry dispatch data", + pTask->id.idStr, pRsp->downstreamTaskId, pTask->msgInfo.blockingTs, waitDuration); + streamRetryDispatchStreamBlock(pTask, waitDuration); + } else { // pipeline send data in output queue + // this message has been sent successfully, let's try next one. + destroyStreamDataBlock(pTask->msgInfo.pData); + pTask->msgInfo.pData = NULL; + + if (pTask->msgInfo.blockingTs != 0) { + int64_t el = taosGetTimestampMs() - pTask->msgInfo.blockingTs; + qDebug("s-task:%s resume to normal from inputQ blocking, idle time:%"PRId64"ms", pTask->id.idStr, el); + pTask->msgInfo.blockingTs = 0; + } + + // now ready for next data output + atomic_store_8(&pTask->outputInfo.status, TASK_OUTPUT_STATUS__NORMAL); + + // otherwise, continue dispatch the first block to down stream task in pipeline + streamDispatchStreamBlock(pTask); } - // continue dispatch - streamDispatch(pTask); return 0; } @@ -265,103 +324,113 @@ int32_t streamProcessRunReq(SStreamTask* pTask) { return -1; } - /*if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {*/ - /*streamDispatch(pTask);*/ - /*}*/ return 0; } int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) { - qDebug("s-task:%s receive retrieve req from node %d taskId:%d", pTask->id.idStr, pReq->srcNodeId, pReq->srcTaskId); streamTaskEnqueueRetrieve(pTask, pReq, pRsp); - - ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); + ASSERT(pTask->info.taskLevel != TASK_LEVEL__SINK); streamSchedExec(pTask); return 0; } bool tInputQueueIsFull(const SStreamTask* pTask) { - return taosQueueItemSize((pTask->inputQueue->queue)) >= STREAM_TASK_INPUT_QUEUEU_CAPACITY; + bool isFull = taosQueueItemSize((pTask->inputQueue->queue)) >= STREAM_TASK_INPUT_QUEUE_CAPACITY; + double size = QUEUE_MEM_SIZE_IN_MB(pTask->inputQueue->queue); + return (isFull || size >= STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE); } int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { - int8_t type = pItem->type; + int8_t type = pItem->type; + int32_t total = taosQueueItemSize(pTask->inputQueue->queue) + 1; + double size = QUEUE_MEM_SIZE_IN_MB(pTask->inputQueue->queue); if (type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit2* pSubmitBlock = streamSubmitBlockClone((SStreamDataSubmit2*)pItem); - if (pSubmitBlock == NULL) { - qDebug("task %d %p submit enqueue failed since out of memory", pTask->id.taskId, pTask); - terrno = TSDB_CODE_OUT_OF_MEMORY; - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); + SStreamDataSubmit* px = (SStreamDataSubmit*)pItem; + if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && tInputQueueIsFull(pTask)) { + qError("s-task:%s input queue is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) stop to push data", + pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, + size); + streamDataSubmitDestroy(px); + taosFreeQitem(pItem); return -1; } - int32_t numOfBlocks = taosQueueItemSize(pTask->inputQueue->queue) + 1; - double size = taosQueueMemorySize(pTask->inputQueue->queue) / 1048576.0; + int32_t msgLen = px->submit.msgLen; + int64_t ver = px->submit.ver; - qDebug("s-task:%s submit enqueue msgLen:%d ver:%" PRId64 ", total in queue:%d, size:%.2fMiB", pTask->id.idStr, - pSubmitBlock->submit.msgLen, pSubmitBlock->submit.ver, numOfBlocks, size); - - if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && - (numOfBlocks > STREAM_TASK_INPUT_QUEUEU_CAPACITY || (size >= STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE))) { - qError("s-task:%s input queue is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) abort", pTask->id.idStr, - STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, - numOfBlocks, size); - streamDataSubmitDestroy(pSubmitBlock); - return -1; + int32_t code = taosWriteQitem(pTask->inputQueue->queue, pItem); + if (code != TSDB_CODE_SUCCESS) { + streamDataSubmitDestroy(px); + taosFreeQitem(pItem); + return code; } - taosWriteQitem(pTask->inputQueue->queue, pSubmitBlock); + // use the local variable to avoid the pItem be freed by other threads, since it has been put into queue already. + qDebug("s-task:%s submit enqueue msgLen:%d ver:%" PRId64 ", total in queue:%d, size:%.2fMiB", pTask->id.idStr, + msgLen, ver, total, size + msgLen/1048576.0); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { - int32_t numOfBlocks = taosQueueItemSize(pTask->inputQueue->queue) + 1; - double size = taosQueueMemorySize(pTask->inputQueue->queue) / 1048576.0; - - if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && - (numOfBlocks > STREAM_TASK_INPUT_QUEUEU_CAPACITY || (size >= STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE))) { + if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && (tInputQueueIsFull(pTask))) { qError("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", - pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, numOfBlocks, + pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); + destroyStreamDataBlock((SStreamDataBlock*) pItem); return -1; } - qDebug("s-task:%s data block enqueue, total in queue:%d", pTask->id.idStr, numOfBlocks); - taosWriteQitem(pTask->inputQueue->queue, pItem); + qDebug("s-task:%s blockdata enqueue, total in queue:%d, size:%.2fMiB", pTask->id.idStr, total, size); + int32_t code = taosWriteQitem(pTask->inputQueue->queue, pItem); + if (code != TSDB_CODE_SUCCESS) { + destroyStreamDataBlock((SStreamDataBlock*) pItem); + return code; + } } else if (type == STREAM_INPUT__CHECKPOINT) { taosWriteQitem(pTask->inputQueue->queue, pItem); } else if (type == STREAM_INPUT__GET_RES) { + // use the default memory limit, refactor later. taosWriteQitem(pTask->inputQueue->queue, pItem); + qDebug("s-task:%s data res enqueue, current(blocks:%d, size:%.2fMiB)", pTask->id.idStr, total, size); } if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE); + qDebug("s-task:%s new data arrived, active the trigger, trigerStatus:%d", pTask->id.idStr, pTask->triggerStatus); } -#if 0 - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL); -#endif - return 0; } static void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; } -void* streamQueueNextItem(SStreamQueue* queue) { - int8_t dequeueFlag = atomic_exchange_8(&queue->status, STREAM_QUEUE__PROCESSING); - if (dequeueFlag == STREAM_QUEUE__FAILED) { - ASSERT(queue->qItem != NULL); - return streamQueueCurItem(queue); +void* streamQueueNextItem(SStreamQueue* pQueue) { + int8_t flag = atomic_exchange_8(&pQueue->status, STREAM_QUEUE__PROCESSING); + + if (flag == STREAM_QUEUE__FAILED) { + ASSERT(pQueue->qItem != NULL); + return streamQueueCurItem(pQueue); } else { - queue->qItem = NULL; - taosGetQitem(queue->qall, &queue->qItem); - if (queue->qItem == NULL) { - taosReadAllQitems(queue->queue, queue->qall); - taosGetQitem(queue->qall, &queue->qItem); + pQueue->qItem = NULL; + taosGetQitem(pQueue->qall, &pQueue->qItem); + if (pQueue->qItem == NULL) { + taosReadAllQitems(pQueue->queue, pQueue->qall); + taosGetQitem(pQueue->qall, &pQueue->qItem); } - return streamQueueCurItem(queue); + + return streamQueueCurItem(pQueue); } } -void streamTaskInputFail(SStreamTask* pTask) { - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); +void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); } + +SStreamChildEpInfo * streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId) { + int32_t num = taosArrayGetSize(pTask->pUpstreamEpInfoList); + for(int32_t i = 0; i < num; ++i) { + SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->pUpstreamEpInfoList, i); + if (pInfo->taskId == taskId) { + return pInfo; + } + } + + return NULL; } \ No newline at end of file diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 16ba81c74a610c761d6ad697b14a2db2ff5cba24..571aca9935e50e1e280eefa7d33b746e8c0da7eb 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -16,12 +16,17 @@ #include "streamBackendRocksdb.h" #include "executor.h" #include "query.h" +#include "streamInt.h" #include "tcommon.h" +#include "tref.h" typedef struct SCompactFilteFactory { void* status; } SCompactFilteFactory; +typedef struct { + void* tableOpt; +} RocksdbCfParam; typedef struct { rocksdb_t* db; rocksdb_column_family_handle_t** pHandle; @@ -29,12 +34,14 @@ typedef struct { rocksdb_readoptions_t* rOpt; rocksdb_options_t** cfOpt; rocksdb_options_t* dbOpt; - void* param; - void* pBackendHandle; + RocksdbCfParam* param; + void* pBackend; SListNode* pCompareNode; + rocksdb_comparator_t** pCompares; } RocksdbCfInst; -int32_t streamStateOpenBackendCf(void* backend, char* name, SHashObj* ids); +uint32_t nextPow2(uint32_t x); +int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf); void destroyRocksdbCfInst(RocksdbCfInst* inst); @@ -46,9 +53,6 @@ unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, c char** newval, size_t* newvlen, unsigned char* value_changed); rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx); -typedef struct { - void* tableOpt; -} RocksdbCfParam; const char* cfName[] = {"default", "state", "fill", "sess", "func", "parname", "partag"}; typedef int (*EncodeFunc)(void* key, char* buf); @@ -59,7 +63,22 @@ typedef int (*BackendCmpFunc)(void* state, const char* aBuf, size_t aLen, const typedef void (*DestroyFunc)(void* state); typedef int32_t (*EncodeValueFunc)(void* value, int32_t vlen, int64_t ttl, char** dest); typedef int32_t (*DecodeValueFunc)(void* value, int32_t vlen, int64_t* ttl, char** dest); +typedef struct { + const char* key; + int32_t len; + int idx; + BackendCmpFunc cmpFunc; + EncodeFunc enFunc; + DecodeFunc deFunc; + ToStringFunc toStrFunc; + CompareName cmpName; + DestroyFunc detroyFunc; + EncodeValueFunc enValueFunc; + DecodeValueFunc deValueFunc; +} SCfInit; + +#define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX)); const char* compareDefaultName(void* name); const char* compareStateName(void* name); const char* compareWinKeyName(void* name); @@ -68,29 +87,90 @@ const char* compareFuncKeyName(void* name); const char* compareParKeyName(void* name); const char* comparePartagKeyName(void* name); +int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); +int defaultKeyEncode(void* k, char* buf); +int defaultKeyDecode(void* k, char* buf); +int defaultKeyToString(void* k, char* buf); + +int stateKeyDBComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); +int stateKeyEncode(void* k, char* buf); +int stateKeyDecode(void* k, char* buf); +int stateKeyToString(void* k, char* buf); + +int stateSessionKeyDBComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); +int stateSessionKeyEncode(void* ses, char* buf); +int stateSessionKeyDecode(void* ses, char* buf); +int stateSessionKeyToString(void* k, char* buf); + +int winKeyDBComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); +int winKeyEncode(void* k, char* buf); +int winKeyDecode(void* k, char* buf); +int winKeyToString(void* k, char* buf); + +int tupleKeyDBComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); +int tupleKeyEncode(void* k, char* buf); +int tupleKeyDecode(void* k, char* buf); +int tupleKeyToString(void* k, char* buf); + +int parKeyDBComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); +int parKeyEncode(void* k, char* buf); +int parKeyDecode(void* k, char* buf); +int parKeyToString(void* k, char* buf); + +int stremaValueEncode(void* k, char* buf); +int streamValueDecode(void* k, char* buf); +int32_t streamValueToString(void* k, char* buf); +int32_t streaValueIsStale(void* k, int64_t ts); +void destroyFunc(void* arg); + +int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest); +int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest); + +SCfInit ginitDict[] = { + {"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName, + destroyFunc, encodeValueFunc, decodeValueFunc}, + {"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyFunc, + encodeValueFunc, decodeValueFunc}, + {"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc}, + {"sess", 4, 3, stateSessionKeyDBComp, stateSessionKeyEncode, stateSessionKeyDecode, stateSessionKeyToString, + compareSessionKeyName, destroyFunc, encodeValueFunc, decodeValueFunc}, + {"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc}, + {"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc}, + {"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc}, +}; + void* streamBackendInit(const char* path) { - qDebug("init stream backend"); - SBackendHandle* pHandle = calloc(1, sizeof(SBackendHandle)); + uint32_t dbMemLimit = nextPow2(tsMaxStreamBackendCache) << 20; + + qDebug("start to init stream backend at %s", path); + SBackendWrapper* pHandle = taosMemoryCalloc(1, sizeof(SBackendWrapper)); pHandle->list = tdListNew(sizeof(SCfComparator)); taosThreadMutexInit(&pHandle->mutex, NULL); taosThreadMutexInit(&pHandle->cfMutex, NULL); pHandle->cfInst = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); rocksdb_env_t* env = rocksdb_create_default_env(); // rocksdb_envoptions_create(); - rocksdb_env_set_low_priority_background_threads(env, 4); - rocksdb_env_set_high_priority_background_threads(env, 2); - rocksdb_cache_t* cache = rocksdb_cache_create_lru(128 << 20); + int32_t nBGThread = tsNumOfSnodeStreamThreads <= 2 ? 1 : tsNumOfSnodeStreamThreads / 2; + rocksdb_env_set_low_priority_background_threads(env, nBGThread); + rocksdb_env_set_high_priority_background_threads(env, nBGThread); + + rocksdb_cache_t* cache = rocksdb_cache_create_lru(dbMemLimit / 2); rocksdb_options_t* opts = rocksdb_options_create(); rocksdb_options_set_env(opts, env); rocksdb_options_set_create_if_missing(opts, 1); rocksdb_options_set_create_missing_column_families(opts, 1); - rocksdb_options_set_write_buffer_size(opts, 128 << 20); - rocksdb_options_set_max_total_wal_size(opts, 128 << 20); + rocksdb_options_set_max_total_wal_size(opts, dbMemLimit); rocksdb_options_set_recycle_log_file_num(opts, 6); rocksdb_options_set_max_write_buffer_number(opts, 3); rocksdb_options_set_info_log_level(opts, 0); + rocksdb_options_set_db_write_buffer_size(opts, dbMemLimit); + rocksdb_options_set_write_buffer_size(opts, dbMemLimit / 2); pHandle->env = env; pHandle->dbOpt = opts; @@ -109,32 +189,18 @@ void* streamBackendInit(const char* path) { if (err != NULL) { qError("failed to open rocksdb, path:%s, reason:%s", path, err); taosMemoryFreeClear(err); + goto _EXIT; } } else { /* list all cf and get prefix */ - int64_t streamId; - int32_t taskId, dummpy = 0; - SHashObj* tbl = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (size_t i = 0; i < nCf; i++) { - char* cf = cfs[i]; - char suffix[64] = {0}; - if (3 == sscanf(cf, "0x%" PRIx64 "-%d_%s", &streamId, &taskId, suffix)) { - char idstr[128] = {0}; - sprintf(idstr, "0x%" PRIx64 "-%d", streamId, taskId); - // qError("make cf name %s", idstr); - if (taosHashGet(tbl, idstr, strlen(idstr) + 1) == NULL) { - taosHashPut(tbl, idstr, strlen(idstr) + 1, &dummpy, sizeof(dummpy)); - } - } else { - continue; - } - } - streamStateOpenBackendCf(pHandle, (char*)path, tbl); - taosHashCleanup(tbl); + streamStateOpenBackendCf(pHandle, (char*)path, cfs, nCf); } - rocksdb_list_column_families_destroy(cfs, nCf); + if (cfs != NULL) { + rocksdb_list_column_families_destroy(cfs, nCf); + } + qDebug("succ to init stream backend at %s, backend:%p", path, pHandle); return (void*)pHandle; _EXIT: @@ -146,59 +212,124 @@ _EXIT: taosHashCleanup(pHandle->cfInst); rocksdb_compactionfilterfactory_destroy(pHandle->filterFactory); tdListFree(pHandle->list); - free(pHandle); + taosMemoryFree(pHandle); + qDebug("failed to init stream backend at %s", path); return NULL; } void streamBackendCleanup(void* arg) { - SBackendHandle* pHandle = (SBackendHandle*)arg; - RocksdbCfInst** pIter = (RocksdbCfInst**)taosHashIterate(pHandle->cfInst, NULL); + SBackendWrapper* pHandle = (SBackendWrapper*)arg; + void* pIter = taosHashIterate(pHandle->cfInst, NULL); while (pIter != NULL) { - RocksdbCfInst* inst = *pIter; + RocksdbCfInst* inst = *(RocksdbCfInst**)pIter; destroyRocksdbCfInst(inst); - taosHashIterate(pHandle->cfInst, pIter); + pIter = taosHashIterate(pHandle->cfInst, pIter); } taosHashCleanup(pHandle->cfInst); - rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); - char* err = NULL; - rocksdb_flush(pHandle->db, flushOpt, &err); - if (err != NULL) { - qError("failed to flush db before streamBackend clean up, reason:%s", err); - taosMemoryFree(err); + if (pHandle->db) { + char* err = NULL; + rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); + rocksdb_flush(pHandle->db, flushOpt, &err); + if (err != NULL) { + qError("failed to flush db before streamBackend clean up, reason:%s", err); + taosMemoryFree(err); + } + rocksdb_flushoptions_destroy(flushOpt); + rocksdb_close(pHandle->db); } - rocksdb_flushoptions_destroy(flushOpt); - - rocksdb_close(pHandle->db); rocksdb_options_destroy(pHandle->dbOpt); rocksdb_env_destroy(pHandle->env); rocksdb_cache_destroy(pHandle->cache); - taosThreadMutexDestroy(&pHandle->mutex); SListNode* head = tdListPopHead(pHandle->list); while (head != NULL) { streamStateDestroyCompar(head->data); taosMemoryFree(head); head = tdListPopHead(pHandle->list); } - // rocksdb_compactionfilterfactory_destroy(pHandle->filterFactory); + tdListFree(pHandle->list); + taosThreadMutexDestroy(&pHandle->mutex); + taosThreadMutexDestroy(&pHandle->cfMutex); + qDebug("destroy stream backend backend:%p", pHandle); taosMemoryFree(pHandle); + return; +} +void streamBackendHandleCleanup(void* arg) { + SBackendCfWrapper* wrapper = arg; + bool remove = wrapper->remove; + qDebug("start to do-close backendwrapper %p, %s", wrapper, wrapper->idstr); + if (wrapper->rocksdb == NULL) { + return; + } + + int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); + char* err = NULL; + if (remove) { + for (int i = 0; i < cfLen; i++) { + if (wrapper->pHandle[i] != NULL) + rocksdb_drop_column_family(wrapper->rocksdb, ((rocksdb_column_family_handle_t**)wrapper->pHandle)[i], &err); + if (err != NULL) { + // qError("failed to create cf:%s_%s, reason:%s", wrapper->idstr, ginitDict[i].key, err); + taosMemoryFreeClear(err); + } + } + } else { + rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); + for (int i = 0; i < cfLen; i++) { + if (wrapper->pHandle[i] != NULL) rocksdb_flush_cf(wrapper->rocksdb, flushOpt, wrapper->pHandle[i], &err); + if (err != NULL) { + qError("failed to create cf:%s_%s, reason:%s", wrapper->idstr, ginitDict[i].key, err); + taosMemoryFreeClear(err); + } + } + rocksdb_flushoptions_destroy(flushOpt); + } + + for (int i = 0; i < cfLen; i++) { + if (wrapper->pHandle[i] != NULL) { + rocksdb_column_family_handle_destroy(wrapper->pHandle[i]); + } + } + taosMemoryFreeClear(wrapper->pHandle); + for (int i = 0; i < cfLen; i++) { + rocksdb_options_destroy(wrapper->cfOpts[i]); + rocksdb_block_based_options_destroy(((RocksdbCfParam*)wrapper->param)[i].tableOpt); + } + + if (remove) { + streamBackendDelCompare(wrapper->pBackend, wrapper->pComparNode); + } + rocksdb_writeoptions_destroy(wrapper->writeOpts); + wrapper->writeOpts = NULL; + + rocksdb_readoptions_destroy(wrapper->readOpts); + wrapper->readOpts = NULL; + taosMemoryFreeClear(wrapper->cfOpts); + taosMemoryFreeClear(wrapper->param); + + taosThreadRwlockDestroy(&wrapper->rwLock); + wrapper->rocksdb = NULL; + taosReleaseRef(streamBackendId, wrapper->backendId); + + qDebug("end to do-close backendwrapper %p, %s", wrapper, wrapper->idstr); + taosMemoryFree(wrapper); return; } SListNode* streamBackendAddCompare(void* backend, void* arg) { - SBackendHandle* pHandle = (SBackendHandle*)backend; - SListNode* node = NULL; + SBackendWrapper* pHandle = (SBackendWrapper*)backend; + SListNode* node = NULL; taosThreadMutexLock(&pHandle->mutex); node = tdListAdd(pHandle->list, arg); taosThreadMutexUnlock(&pHandle->mutex); return node; } void streamBackendDelCompare(void* backend, void* arg) { - SBackendHandle* pHandle = (SBackendHandle*)backend; - SListNode* node = NULL; + SBackendWrapper* pHandle = (SBackendWrapper*)backend; + SListNode* node = NULL; taosThreadMutexLock(&pHandle->mutex); node = tdListPopNode(pHandle->list, arg); taosThreadMutexUnlock(&pHandle->mutex); @@ -209,7 +340,6 @@ void streamBackendDelCompare(void* backend, void* arg) { } void streamStateDestroy_rocksdb(SStreamState* pState, bool remove) { streamStateCloseBackend(pState, remove); } static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len); -int streamGetInit(const char* funcName); // |key|-----value------| // |key|ttl|len|userData| @@ -537,33 +667,22 @@ void destroyFunc(void* arg) { return; } -typedef struct { - const char* key; - int32_t len; - int idx; - BackendCmpFunc cmpFunc; - EncodeFunc enFunc; - DecodeFunc deFunc; - ToStringFunc toStrFunc; - CompareName cmpName; - DestroyFunc detroyFunc; - EncodeValueFunc enValueFunc; - DecodeValueFunc deValueFunc; - -} SCfInit; - -#define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX)); - int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest) { SStreamValue key = {.unixTimestamp = ttl, .len = vlen, .data = (char*)(value)}; - - char* p = taosMemoryCalloc(1, sizeof(int64_t) + sizeof(int32_t) + key.len); - char* buf = p; - int32_t len = 0; - len += taosEncodeFixedI64((void**)&buf, key.unixTimestamp); - len += taosEncodeFixedI32((void**)&buf, key.len); - len += taosEncodeBinary((void**)&buf, (char*)value, vlen); - *dest = p; + int32_t len = 0; + if (*dest == NULL) { + char* p = taosMemoryCalloc(1, sizeof(int64_t) + sizeof(int32_t) + key.len); + char* buf = p; + len += taosEncodeFixedI64((void**)&buf, key.unixTimestamp); + len += taosEncodeFixedI32((void**)&buf, key.len); + len += taosEncodeBinary((void**)&buf, (char*)value, vlen); + *dest = p; + } else { + char* buf = *dest; + len += taosEncodeFixedI64((void**)&buf, key.unixTimestamp); + len += taosEncodeFixedI32((void**)&buf, key.len); + len += taosEncodeBinary((void**)&buf, (char*)value, vlen); + } return len; } /* @@ -577,9 +696,14 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { *dest = NULL; return -1; } - int64_t now = taosGetTimestampMs(); p = taosDecodeFixedI64(p, &key.unixTimestamp); p = taosDecodeFixedI32(p, &key.len); + if (vlen != (sizeof(int64_t) + sizeof(int32_t) + key.len)) { + if (dest != NULL) *dest = NULL; + qError("vlen: %d, read len: %d", vlen, key.len); + return -1; + } + if (key.len == 0) { key.data = NULL; } else { @@ -587,6 +711,7 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { } if (ttl != NULL) { + int64_t now = taosGetTimestampMs(); *ttl = key.unixTimestamp == 0 ? 0 : key.unixTimestamp - now; } if (dest != NULL) { @@ -596,22 +721,6 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { } return key.len; } -SCfInit ginitDict[] = { - {"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName, - destroyFunc, encodeValueFunc, decodeValueFunc}, - {"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"sess", 4, 3, stateSessionKeyDBComp, stateSessionKeyEncode, stateSessionKeyDecode, stateSessionKeyToString, - compareSessionKeyName, destroyFunc, encodeValueFunc, decodeValueFunc}, - {"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, -}; const char* compareDefaultName(void* arg) { (void)arg; @@ -667,7 +776,7 @@ rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_c void destroyRocksdbCfInst(RocksdbCfInst* inst) { int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); for (int i = 0; i < cfLen; i++) { - rocksdb_column_family_handle_destroy(inst->pHandle[i]); + if (inst->pHandle[i]) rocksdb_column_family_handle_destroy((inst->pHandle)[i]); } rocksdb_writeoptions_destroy(inst->wOpt); @@ -675,139 +784,162 @@ void destroyRocksdbCfInst(RocksdbCfInst* inst) { rocksdb_readoptions_destroy(inst->rOpt); taosMemoryFree(inst->cfOpt); - taosMemoryFree(inst->param); taosMemoryFreeClear(inst->param); taosMemoryFree(inst); } -int32_t streamStateOpenBackendCf(void* backend, char* name, SHashObj* ids) { - SBackendHandle* handle = backend; - char* err = NULL; - size_t nSize = taosHashGetSize(ids); - int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); - - char** cfNames = taosMemoryCalloc(nSize * cfLen + 1, sizeof(char*)); - void* pIter = taosHashIterate(ids, NULL); - size_t keyLen = 0; - char* idstr = taosHashGetKey(pIter, &keyLen); - for (int i = 0; i < nSize * cfLen + 1; i++) { - cfNames[i] = (char*)taosMemoryCalloc(1, 128); - if (i == 0) { - memcpy(cfNames[0], "default", strlen("default")); - continue; - } +int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf) { + SBackendWrapper* handle = backend; + char* err = NULL; + int64_t streamId; + int32_t taskId, dummy = 0; + char suffix[64] = {0}; - GEN_COLUMN_FAMILY_NAME(cfNames[i], idstr, ginitDict[(i - 1) % (cfLen)].key); - if (i % cfLen == 0) { - pIter = taosHashIterate(ids, pIter); - if (pIter != NULL) idstr = taosHashGetKey(pIter, &keyLen); - } - } - rocksdb_options_t** cfOpts = taosMemoryCalloc(nSize * cfLen + 1, sizeof(rocksdb_options_t*)); - RocksdbCfParam* params = taosMemoryCalloc(nSize * cfLen + 1, sizeof(RocksdbCfParam*)); - for (int i = 0; i < nSize * cfLen + 1; i++) { + rocksdb_options_t** cfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); + RocksdbCfParam* params = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam)); + rocksdb_comparator_t** pCompare = taosMemoryCalloc(nCf, sizeof(rocksdb_comparator_t*)); + rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); + + for (int i = 0; i < nCf; i++) { + char* cf = cfs[i]; + char funcname[64] = {0}; cfOpts[i] = rocksdb_options_create_copy(handle->dbOpt); - if (i == 0) { - continue; - } - // refactor later - rocksdb_block_based_table_options_t* tableOpt = rocksdb_block_based_options_create(); - rocksdb_block_based_options_set_block_cache(tableOpt, handle->cache); + if (i == 0) continue; + if (3 == sscanf(cf, "0x%" PRIx64 "-%d_%s", &streamId, &taskId, funcname)) { + rocksdb_block_based_table_options_t* tableOpt = rocksdb_block_based_options_create(); + rocksdb_block_based_options_set_block_cache(tableOpt, handle->cache); - rocksdb_filterpolicy_t* filter = rocksdb_filterpolicy_create_bloom(15); - rocksdb_block_based_options_set_filter_policy(tableOpt, filter); + rocksdb_filterpolicy_t* filter = rocksdb_filterpolicy_create_bloom(15); + rocksdb_block_based_options_set_filter_policy(tableOpt, filter); - rocksdb_options_set_block_based_table_factory((rocksdb_options_t*)cfOpts[i], tableOpt); - params[i].tableOpt = tableOpt; - }; + rocksdb_options_set_block_based_table_factory((rocksdb_options_t*)cfOpts[i], tableOpt); + params[i].tableOpt = tableOpt; - rocksdb_comparator_t** pCompare = taosMemoryCalloc(nSize * cfLen + 1, sizeof(rocksdb_comparator_t**)); - for (int i = 0; i < nSize * cfLen + 1; i++) { - if (i == 0) { - continue; - } - SCfInit* cf = &ginitDict[(i - 1) % cfLen]; + int idx = streamStateGetCfIdx(NULL, funcname); + SCfInit* cfPara = &ginitDict[idx]; - rocksdb_comparator_t* compare = rocksdb_comparator_create(NULL, cf->detroyFunc, cf->cmpFunc, cf->cmpName); - rocksdb_options_set_comparator((rocksdb_options_t*)cfOpts[i], compare); - pCompare[i] = compare; + rocksdb_comparator_t* compare = + rocksdb_comparator_create(NULL, cfPara->detroyFunc, cfPara->cmpFunc, cfPara->cmpName); + rocksdb_options_set_comparator((rocksdb_options_t*)cfOpts[i], compare); + pCompare[i] = compare; + } } - rocksdb_column_family_handle_t** cfHandle = - taosMemoryCalloc(nSize * cfLen + 1, sizeof(rocksdb_column_family_handle_t*)); - rocksdb_t* db = rocksdb_open_column_families(handle->dbOpt, name, nSize * cfLen + 1, (const char* const*)cfNames, + rocksdb_t* db = rocksdb_open_column_families(handle->dbOpt, name, nCf, (const char* const*)cfs, (const rocksdb_options_t* const*)cfOpts, cfHandle, &err); if (err != NULL) { qError("failed to open rocksdb cf, reason:%s", err); taosMemoryFree(err); } else { - qDebug("succ to open rocksdb cf, reason:%s", err); - } - - pIter = taosHashIterate(ids, NULL); - idstr = taosHashGetKey(pIter, &keyLen); - for (int i = 0; i < nSize; i++) { - RocksdbCfInst* inst = taosMemoryCalloc(1, sizeof(RocksdbCfInst)); - rocksdb_column_family_handle_t** subCf = taosMemoryCalloc(cfLen, sizeof(rocksdb_column_family_handle_t*)); - rocksdb_comparator_t** subCompare = taosMemoryCalloc(cfLen, sizeof(rocksdb_comparator_t*)); - RocksdbCfParam* subParam = taosMemoryCalloc(cfLen, sizeof(RocksdbCfParam)); - rocksdb_options_t** subOpt = taosMemoryCalloc(cfLen, sizeof(rocksdb_options_t*)); - for (int j = 0; j < cfLen; j++) { - subCf[j] = cfHandle[i * cfLen + j + 1]; - subCompare[j] = pCompare[i * cfLen + j + 1]; - subParam[j] = params[i * cfLen + j + 1]; - subOpt[j] = cfOpts[i * cfLen + j + 1]; + qDebug("succ to open rocksdb cf"); + } + // close default cf + if (((rocksdb_column_family_handle_t**)cfHandle)[0] != 0) { + rocksdb_column_family_handle_destroy(cfHandle[0]); + cfHandle[0] = NULL; + } + rocksdb_options_destroy(cfOpts[0]); + handle->db = db; + + static int32_t cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); + for (int i = 0; i < nCf; i++) { + char* cf = cfs[i]; + if (i == 0) continue; + char funcname[64] = {0}; + if (3 == sscanf(cf, "0x%" PRIx64 "-%d_%s", &streamId, &taskId, funcname)) { + char idstr[128] = {0}; + sprintf(idstr, "0x%" PRIx64 "-%d", streamId, taskId); + + int idx = streamStateGetCfIdx(NULL, funcname); + + RocksdbCfInst* inst = NULL; + RocksdbCfInst** pInst = taosHashGet(handle->cfInst, idstr, strlen(idstr) + 1); + if (pInst == NULL || *pInst == NULL) { + inst = taosMemoryCalloc(1, sizeof(RocksdbCfInst)); + inst->pHandle = taosMemoryCalloc(cfLen, sizeof(rocksdb_column_family_handle_t*)); + inst->cfOpt = taosMemoryCalloc(cfLen, sizeof(rocksdb_options_t*)); + inst->wOpt = rocksdb_writeoptions_create(); + inst->rOpt = rocksdb_readoptions_create(); + inst->param = taosMemoryCalloc(cfLen, sizeof(RocksdbCfParam)); + inst->pBackend = handle; + inst->db = db; + inst->pCompares = taosMemoryCalloc(cfLen, sizeof(rocksdb_comparator_t*)); + + inst->dbOpt = handle->dbOpt; + rocksdb_writeoptions_disable_WAL(inst->wOpt, 1); + taosHashPut(handle->cfInst, idstr, strlen(idstr) + 1, &inst, sizeof(void*)); + } else { + inst = *pInst; + } + inst->cfOpt[idx] = cfOpts[i]; + inst->pCompares[idx] = pCompare[i]; + memcpy(&(inst->param[idx]), &(params[i]), sizeof(RocksdbCfParam)); + inst->pHandle[idx] = cfHandle[i]; } - inst->db = db; - inst->pHandle = subCf; - inst->wOpt = rocksdb_writeoptions_create(); - inst->rOpt = rocksdb_readoptions_create(); - inst->cfOpt = (rocksdb_options_t**)subOpt; - inst->dbOpt = handle->dbOpt; - inst->param = subParam; - inst->pBackendHandle = handle; - handle->db = db; - SCfComparator compare = {.comp = subCompare, .numOfComp = cfLen}; - inst->pCompareNode = streamBackendAddCompare(handle, &compare); - rocksdb_writeoptions_disable_WAL(inst->wOpt, 1); + } + void** pIter = taosHashIterate(handle->cfInst, NULL); + while (pIter) { + RocksdbCfInst* inst = *pIter; - taosHashPut(handle->cfInst, idstr, keyLen, &inst, sizeof(void*)); + for (int i = 0; i < cfLen; i++) { + if (inst->cfOpt[i] == NULL) { + rocksdb_options_t* opt = rocksdb_options_create_copy(handle->dbOpt); + rocksdb_block_based_table_options_t* tableOpt = rocksdb_block_based_options_create(); + rocksdb_block_based_options_set_block_cache(tableOpt, handle->cache); - pIter = taosHashIterate(ids, pIter); - if (pIter != NULL) idstr = taosHashGetKey(pIter, &keyLen); - } - rocksdb_column_family_handle_destroy(cfHandle[0]); - rocksdb_options_destroy(cfOpts[0]); + rocksdb_filterpolicy_t* filter = rocksdb_filterpolicy_create_bloom(15); + rocksdb_block_based_options_set_filter_policy(tableOpt, filter); - for (int i = 0; i < nSize * cfLen + 1; i++) { - taosMemoryFree(cfNames[i]); + rocksdb_options_set_block_based_table_factory((rocksdb_options_t*)opt, tableOpt); + + SCfInit* cfPara = &ginitDict[i]; + + rocksdb_comparator_t* compare = + rocksdb_comparator_create(NULL, cfPara->detroyFunc, cfPara->cmpFunc, cfPara->cmpName); + rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); + + inst->pCompares[i] = compare; + inst->cfOpt[i] = opt; + inst->param[i].tableOpt = tableOpt; + } + } + SCfComparator compare = {.comp = inst->pCompares, .numOfComp = cfLen}; + inst->pCompareNode = streamBackendAddCompare(handle, &compare); + pIter = taosHashIterate(handle->cfInst, pIter); } - taosMemoryFree(cfNames); + taosMemoryFree(cfHandle); taosMemoryFree(pCompare); taosMemoryFree(params); taosMemoryFree(cfOpts); - return 0; } int streamStateOpenBackend(void* backend, SStreamState* pState) { - qInfo("start to open backend, %p 0x%" PRIx64 "-%d", pState, pState->streamId, pState->taskId); - SBackendHandle* handle = backend; - - sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-%d", pState->streamId, pState->taskId); + qInfo("start to open state %p on backend %p 0x%" PRIx64 "-%d", pState, backend, pState->streamId, pState->taskId); + taosAcquireRef(streamBackendId, pState->streamBackendRid); + SBackendWrapper* handle = backend; + SBackendCfWrapper* pBackendCfWrapper = taosMemoryCalloc(1, sizeof(SBackendCfWrapper)); taosThreadMutexLock(&handle->cfMutex); + RocksdbCfInst** ppInst = taosHashGet(handle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); if (ppInst != NULL && *ppInst != NULL) { RocksdbCfInst* inst = *ppInst; - pState->pTdbState->rocksdb = inst->db; - pState->pTdbState->pHandle = inst->pHandle; - pState->pTdbState->writeOpts = inst->wOpt; - pState->pTdbState->readOpts = inst->rOpt; - pState->pTdbState->cfOpts = inst->cfOpt; - pState->pTdbState->dbOpt = handle->dbOpt; - pState->pTdbState->param = inst->param; - pState->pTdbState->pBackendHandle = handle; - pState->pTdbState->pComparNode = inst->pCompareNode; + pBackendCfWrapper->rocksdb = inst->db; + pBackendCfWrapper->pHandle = (void**)inst->pHandle; + pBackendCfWrapper->writeOpts = inst->wOpt; + pBackendCfWrapper->readOpts = inst->rOpt; + pBackendCfWrapper->cfOpts = (void**)(inst->cfOpt); + pBackendCfWrapper->dbOpt = handle->dbOpt; + pBackendCfWrapper->param = inst->param; + pBackendCfWrapper->pBackend = handle; + pBackendCfWrapper->pComparNode = inst->pCompareNode; taosThreadMutexUnlock(&handle->cfMutex); + pBackendCfWrapper->backendId = pState->streamBackendRid; + memcpy(pBackendCfWrapper->idstr, pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr)); + + int64_t id = taosAddRef(streamBackendCfWrapperId, pBackendCfWrapper); + pState->pTdbState->backendCfWrapperId = id; + pState->pTdbState->pBackendCfWrapper = pBackendCfWrapper; + qInfo("succ to open state %p on backendWrapper, %p, %s", pState, pBackendCfWrapper, pBackendCfWrapper->idstr); return 0; } taosThreadMutexUnlock(&handle->cfMutex); @@ -831,7 +963,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { param[i].tableOpt = tableOpt; }; - rocksdb_comparator_t** pCompare = taosMemoryCalloc(cfLen, sizeof(rocksdb_comparator_t**)); + rocksdb_comparator_t** pCompare = taosMemoryCalloc(cfLen, sizeof(rocksdb_comparator_t*)); for (int i = 0; i < cfLen; i++) { SCfInit* cf = &ginitDict[i]; @@ -839,36 +971,34 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { rocksdb_options_set_comparator((rocksdb_options_t*)cfOpt[i], compare); pCompare[i] = compare; } - rocksdb_column_family_handle_t** cfHandle = taosMemoryMalloc(cfLen * sizeof(rocksdb_column_family_handle_t*)); - for (int i = 0; i < cfLen; i++) { - char buf[128] = {0}; - GEN_COLUMN_FAMILY_NAME(buf, pState->pTdbState->idstr, ginitDict[i].key); - cfHandle[i] = rocksdb_create_column_family(handle->db, cfOpt[i], buf, &err); - if (err != NULL) { - qError("failed to create cf:%s_%s, reason:%s", pState->pTdbState->idstr, ginitDict[i].key, err); - taosMemoryFreeClear(err); - } - } - pState->pTdbState->rocksdb = handle->db; - pState->pTdbState->pHandle = cfHandle; - pState->pTdbState->writeOpts = rocksdb_writeoptions_create(); - pState->pTdbState->readOpts = rocksdb_readoptions_create(); - pState->pTdbState->cfOpts = (rocksdb_options_t**)cfOpt; - pState->pTdbState->dbOpt = handle->dbOpt; - pState->pTdbState->param = param; - pState->pTdbState->pBackendHandle = handle; - + rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(cfLen, sizeof(rocksdb_column_family_handle_t*)); + pBackendCfWrapper->rocksdb = handle->db; + pBackendCfWrapper->pHandle = (void**)cfHandle; + pBackendCfWrapper->writeOpts = rocksdb_writeoptions_create(); + pBackendCfWrapper->readOpts = rocksdb_readoptions_create(); + pBackendCfWrapper->cfOpts = (void**)cfOpt; + pBackendCfWrapper->dbOpt = handle->dbOpt; + pBackendCfWrapper->param = param; + pBackendCfWrapper->pBackend = handle; + pBackendCfWrapper->backendId = pState->streamBackendRid; + taosThreadRwlockInit(&pBackendCfWrapper->rwLock, NULL); SCfComparator compare = {.comp = pCompare, .numOfComp = cfLen}; - pState->pTdbState->pComparNode = streamBackendAddCompare(handle, &compare); - // rocksdb_writeoptions_disable_WAL(pState->pTdbState->writeOpts, 1); - qInfo("succ to open backend, %p, 0x%" PRIx64 "-%d", pState, pState->streamId, pState->taskId); + pBackendCfWrapper->pComparNode = streamBackendAddCompare(handle, &compare); + rocksdb_writeoptions_disable_WAL(pBackendCfWrapper->writeOpts, 1); + memcpy(pBackendCfWrapper->idstr, pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr)); + + int64_t id = taosAddRef(streamBackendCfWrapperId, pBackendCfWrapper); + pState->pTdbState->backendCfWrapperId = id; + pState->pTdbState->pBackendCfWrapper = pBackendCfWrapper; + qInfo("succ to open state %p on backendWrapper %p %s", pState, pBackendCfWrapper, pBackendCfWrapper->idstr); return 0; } void streamStateCloseBackend(SStreamState* pState, bool remove) { - SBackendHandle* pHandle = pState->pTdbState->pBackendHandle; + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SBackendWrapper* pHandle = wrapper->pBackend; taosThreadMutexLock(&pHandle->cfMutex); - RocksdbCfInst** ppInst = taosHashGet(pHandle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); + RocksdbCfInst** ppInst = taosHashGet(pHandle->cfInst, wrapper->idstr, strlen(pState->pTdbState->idstr) + 1); if (ppInst != NULL && *ppInst != NULL) { RocksdbCfInst* inst = *ppInst; taosMemoryFree(inst); @@ -877,72 +1007,53 @@ void streamStateCloseBackend(SStreamState* pState, bool remove) { taosThreadMutexUnlock(&pHandle->cfMutex); char* status[] = {"close", "drop"}; - qInfo("start to %s backend, %p, 0x%" PRIx64 "-%d", status[remove == false ? 0 : 1], pState, pState->streamId, - pState->taskId); - if (pState->pTdbState->rocksdb == NULL) { - return; - } - - int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); - - char* err = NULL; - if (remove) { - for (int i = 0; i < cfLen; i++) { - rocksdb_drop_column_family(pState->pTdbState->rocksdb, pState->pTdbState->pHandle[i], &err); - if (err != NULL) { - qError("failed to create cf:%s_%s, reason:%s", pState->pTdbState->idstr, ginitDict[i].key, err); - taosMemoryFreeClear(err); - } - } - } else { - rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); - for (int i = 0; i < cfLen; i++) { - rocksdb_flush_cf(pState->pTdbState->rocksdb, flushOpt, pState->pTdbState->pHandle[i], &err); - if (err != NULL) { - qError("failed to create cf:%s_%s, reason:%s", pState->pTdbState->idstr, ginitDict[i].key, err); - taosMemoryFreeClear(err); - } - } - rocksdb_flushoptions_destroy(flushOpt); - } - - for (int i = 0; i < cfLen; i++) { - rocksdb_column_family_handle_destroy(pState->pTdbState->pHandle[i]); - } - taosMemoryFreeClear(pState->pTdbState->pHandle); - for (int i = 0; i < cfLen; i++) { - rocksdb_options_destroy(pState->pTdbState->cfOpts[i]); - rocksdb_block_based_options_destroy(((RocksdbCfParam*)pState->pTdbState->param)[i].tableOpt); - } - - if (remove) { - streamBackendDelCompare(pState->pTdbState->pBackendHandle, pState->pTdbState->pComparNode); - } - rocksdb_writeoptions_destroy(pState->pTdbState->writeOpts); - pState->pTdbState->writeOpts = NULL; - - rocksdb_readoptions_destroy(pState->pTdbState->readOpts); - pState->pTdbState->readOpts = NULL; - taosMemoryFreeClear(pState->pTdbState->cfOpts); - taosMemoryFreeClear(pState->pTdbState->param); - pState->pTdbState->rocksdb = NULL; + qInfo("start to close %s state %p on backendWrapper %p %s", status[remove == false ? 0 : 1], pState, wrapper, + wrapper->idstr); + wrapper->remove |= remove; // update by other pState + taosReleaseRef(streamBackendCfWrapperId, pState->pTdbState->backendCfWrapperId); } void streamStateDestroyCompar(void* arg) { SCfComparator* comp = (SCfComparator*)arg; for (int i = 0; i < comp->numOfComp; i++) { - rocksdb_comparator_destroy(comp->comp[i]); + if (comp->comp[i]) rocksdb_comparator_destroy(comp->comp[i]); } taosMemoryFree(comp->comp); } -int streamGetInit(const char* funcName) { +int streamStateGetCfIdx(SStreamState* pState, const char* funcName) { + int idx = -1; size_t len = strlen(funcName); for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { if (len == ginitDict[i].len && strncmp(funcName, ginitDict[i].key, strlen(funcName)) == 0) { - return i; + idx = i; + break; } } - return -1; + if (pState != NULL && idx != -1) { + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + rocksdb_column_family_handle_t* cf = NULL; + taosThreadRwlockRdlock(&wrapper->rwLock); + cf = wrapper->pHandle[idx]; + taosThreadRwlockUnlock(&wrapper->rwLock); + if (cf == NULL) { + char buf[128] = {0}; + GEN_COLUMN_FAMILY_NAME(buf, wrapper->idstr, ginitDict[idx].key); + char* err = NULL; + + taosThreadRwlockWrlock(&wrapper->rwLock); + cf = rocksdb_create_column_family(wrapper->rocksdb, wrapper->cfOpts[idx], buf, &err); + if (err != NULL) { + idx = -1; + qError("failed to to open cf, %p %s_%s, reason:%s", pState, wrapper->idstr, funcName, err); + taosMemoryFree(err); + } else { + wrapper->pHandle[idx] = cf; + } + taosThreadRwlockUnlock(&wrapper->rwLock); + } + } + + return idx; } bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len) { rocksdb_iter_seek(iter, buf, len); @@ -956,123 +1067,122 @@ bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len } rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt) { - int idx = streamGetInit(cfName); + int idx = streamStateGetCfIdx(pState, cfName); - if (snapshot != NULL) { - *snapshot = (rocksdb_snapshot_t*)rocksdb_create_snapshot(pState->pTdbState->rocksdb); - } rocksdb_readoptions_t* rOpt = rocksdb_readoptions_create(); *readOpt = rOpt; - rocksdb_readoptions_set_snapshot(rOpt, *snapshot); - rocksdb_readoptions_set_fill_cache(rOpt, 0); - - return rocksdb_create_iterator_cf(pState->pTdbState->rocksdb, rOpt, pState->pTdbState->pHandle[idx]); -} - -#define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamGetInit(funcname); \ - if (i < 0) { \ - qWarn("streamState failed to get cf name: %s", funcname); \ - code = -1; \ - break; \ - } \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = pState->pTdbState->pHandle[ginitDict[i].idx]; \ - rocksdb_t* db = pState->pTdbState->rocksdb; \ - rocksdb_writeoptions_t* opts = pState->pTdbState->writeOpts; \ - char* ttlV = NULL; \ - int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ - rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ - if (err != NULL) { \ - taosMemoryFree(err); \ - qDebug("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ - code = -1; \ - } else { \ - qDebug("streamState str:%s succ to write to %s, valLen:%d", toString, funcname, vLen); \ - } \ - taosMemoryFree(ttlV); \ - } while (0); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + if (snapshot != NULL) { + *snapshot = (rocksdb_snapshot_t*)rocksdb_create_snapshot(wrapper->rocksdb); + rocksdb_readoptions_set_snapshot(rOpt, *snapshot); + rocksdb_readoptions_set_fill_cache(rOpt, 0); + } + + return rocksdb_create_iterator_cf(wrapper->rocksdb, rOpt, ((rocksdb_column_family_handle_t**)wrapper->pHandle)[idx]); +} -#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ +#define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \ do { \ code = 0; \ char buf[128] = {0}; \ char* err = NULL; \ - int i = streamGetInit(funcname); \ + int i = streamStateGetCfIdx(pState, funcname); \ if (i < 0) { \ qWarn("streamState failed to get cf name: %s", funcname); \ code = -1; \ break; \ } \ - char toString[128] = {0}; \ + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; \ + char toString[128] = {0}; \ if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = pState->pTdbState->pHandle[ginitDict[i].idx]; \ - rocksdb_t* db = pState->pTdbState->rocksdb; \ - rocksdb_readoptions_t* opts = pState->pTdbState->readOpts; \ - size_t len = 0; \ - char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ - if (val == NULL) { \ - if (err == NULL) { \ - qDebug("streamState str: %s failed to read from %s_%s, err: not exist", toString, pState->pTdbState->idstr, \ - funcname); \ - } else { \ - qDebug("streamState str: %s failed to read from %s_%s, err: %s", toString, pState->pTdbState->idstr, funcname, \ - err); \ - taosMemoryFreeClear(err); \ - } \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pHandle)[ginitDict[i].idx]; \ + rocksdb_t* db = wrapper->rocksdb; \ + rocksdb_writeoptions_t* opts = wrapper->writeOpts; \ + char* ttlV = NULL; \ + int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ + rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ + if (err != NULL) { \ + qError("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ + taosMemoryFree(err); \ code = -1; \ } else { \ - char* p = NULL; \ - int32_t len = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ - if (len < 0) { \ - qDebug("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, pState->pTdbState->idstr, \ - funcname); \ - code = -1; \ - } else { \ - qDebug("streamState str: %s succ to read from %s_%s, valLen:%d", toString, pState->pTdbState->idstr, funcname, \ - len); \ - } \ - taosMemoryFree(val); \ - if (vLen != NULL) *vLen = len; \ + qTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d", toString, funcname, vLen, ttlVLen); \ } \ - if (code == 0) \ - qDebug("streamState str: %s succ to read from %s_%s", toString, pState->pTdbState->idstr, funcname); \ + taosMemoryFree(ttlV); \ + } while (0); + +#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamStateGetCfIdx(pState, funcname); \ + if (i < 0) { \ + qWarn("streamState failed to get cf name: %s", funcname); \ + code = -1; \ + break; \ + } \ + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; \ + char toString[128] = {0}; \ + if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pHandle)[ginitDict[i].idx]; \ + rocksdb_t* db = wrapper->rocksdb; \ + rocksdb_readoptions_t* opts = wrapper->readOpts; \ + size_t len = 0; \ + char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ + if (val == NULL || len == 0) { \ + if (err == NULL) { \ + qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, wrapper->idstr, funcname); \ + } else { \ + qError("streamState str: %s failed to read from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ + taosMemoryFreeClear(err); \ + } \ + code = -1; \ + } else { \ + char* p = NULL; \ + int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ + if (tlen <= 0) { \ + qError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, wrapper->idstr, \ + funcname); \ + code = -1; \ + } else { \ + qTrace("streamState str: %s succ to read from %s_%s, valLen:%d", toString, wrapper->idstr, funcname, tlen); \ + } \ + taosMemoryFree(val); \ + if (vLen != NULL) *vLen = tlen; \ + } \ + if (code == 0) qDebug("streamState str: %s succ to read from %s_%s", toString, wrapper->idstr, funcname); \ } while (0); -#define STREAM_STATE_DEL_ROCKSDB(pState, funcname, key) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamGetInit(funcname); \ - if (i < 0) { \ - qWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ - code = -1; \ - break; \ - } \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = pState->pTdbState->pHandle[ginitDict[i].idx]; \ - rocksdb_t* db = pState->pTdbState->rocksdb; \ - rocksdb_writeoptions_t* opts = pState->pTdbState->writeOpts; \ - rocksdb_delete_cf(db, opts, pHandle, (const char*)buf, klen, &err); \ - if (err != NULL) { \ - qError("streamState str: %s failed to del from %s_%s, err: %s", toString, pState->pTdbState->idstr, funcname, \ - err); \ - taosMemoryFree(err); \ - code = -1; \ - } else { \ - qDebug("streamState str: %s succ to del from %s_%s", toString, pState->pTdbState->idstr, funcname); \ - } \ +#define STREAM_STATE_DEL_ROCKSDB(pState, funcname, key) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamStateGetCfIdx(pState, funcname); \ + if (i < 0) { \ + qWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ + code = -1; \ + break; \ + } \ + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; \ + char toString[128] = {0}; \ + if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pHandle)[ginitDict[i].idx]; \ + rocksdb_t* db = wrapper->rocksdb; \ + rocksdb_writeoptions_t* opts = wrapper->writeOpts; \ + rocksdb_delete_cf(db, opts, pHandle, (const char*)buf, klen, &err); \ + if (err != NULL) { \ + qError("streamState str: %s failed to del from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ + taosMemoryFree(err); \ + code = -1; \ + } else { \ + qTrace("streamState str: %s succ to del from %s_%s", toString, wrapper->idstr, funcname); \ + } \ } while (0); // state cf @@ -1098,29 +1208,30 @@ int32_t streamStateDel_rocksdb(SStreamState* pState, const SWinKey* key) { int32_t streamStateClear_rocksdb(SStreamState* pState) { qDebug("streamStateClear_rocksdb"); - SStateKey sKey = {.key = {.ts = 0, .groupId = 0}, .opNum = pState->number}; - SStateKey eKey = {.key = {.ts = INT64_MAX, .groupId = UINT64_MAX}, .opNum = pState->number}; - char sKeyStr[128] = {0}; - char eKeyStr[128] = {0}; + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + char sKeyStr[128] = {0}; + char eKeyStr[128] = {0}; + SStateKey sKey = {.key = {.ts = 0, .groupId = 0}, .opNum = pState->number}; + SStateKey eKey = {.key = {.ts = INT64_MAX, .groupId = UINT64_MAX}, .opNum = pState->number}; int sLen = stateKeyEncode(&sKey, sKeyStr); int eLen = stateKeyEncode(&eKey, eKeyStr); - char toStringStart[128] = {0}; - char toStringEnd[128] = {0}; - if (qDebugFlag & DEBUG_TRACE) { - stateKeyToString(&sKey, toStringStart); - stateKeyToString(&eKey, toStringEnd); - } + if (wrapper->pHandle[1] != NULL) { + char* err = NULL; + rocksdb_delete_range_cf(wrapper->rocksdb, wrapper->writeOpts, wrapper->pHandle[1], sKeyStr, sLen, eKeyStr, eLen, + &err); + if (err != NULL) { + char toStringStart[128] = {0}; + char toStringEnd[128] = {0}; + stateKeyToString(&sKey, toStringStart); + stateKeyToString(&eKey, toStringEnd); - char* err = NULL; - rocksdb_delete_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->writeOpts, pState->pTdbState->pHandle[1], - sKeyStr, sLen, eKeyStr, eLen, &err); - // rocksdb_compact_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->pHandle[0], sKeyStr, sLen, eKeyStr, - // eLen); - if (err != NULL) { - qWarn("failed to delete range cf(state) start: %s, end:%s, reason:%s", toStringStart, toStringEnd, err); - taosMemoryFree(err); + qWarn("failed to delete range cf(state) start: %s, end:%s, reason:%s", toStringStart, toStringEnd, err); + taosMemoryFree(err); + } else { + rocksdb_compact_range_cf(wrapper->rocksdb, wrapper->pHandle[1], sKeyStr, sLen, eKeyStr, eLen); + } } return 0; @@ -1155,6 +1266,8 @@ int32_t streamStateGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, if (pKey->groupId == groupId) { return 0; } + taosMemoryFree((void*)*pVal); + *pVal = NULL; } return -1; } @@ -1213,9 +1326,11 @@ SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWin if (pCur == NULL) { return NULL; } + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; pCur->number = pState->number; - pCur->db = pState->pTdbState->rocksdb; - pCur->iter = streamStateIterCreate(pState, "state", &pCur->snapshot, &pCur->readOpt); + pCur->db = wrapper->rocksdb; + pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot, + (rocksdb_readoptions_t**)&pCur->readOpt); SStateKey sKey = {.key = *key, .opNum = pState->number}; char buf[128] = {0}; @@ -1246,16 +1361,18 @@ SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWin SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateGetCur_rocksdb"); - int32_t code = 0; + int32_t code = 0; + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + const SStateKey maxStateKey = {.key = {.groupId = UINT64_MAX, .ts = INT64_MAX}, .opNum = INT64_MAX}; STREAM_STATE_PUT_ROCKSDB(pState, "state", &maxStateKey, "", 0); - char buf[128] = {0}; - int32_t klen = stateKeyEncode((void*)&maxStateKey, buf); - + char buf[128] = {0}; + int32_t klen = stateKeyEncode((void*)&maxStateKey, buf); SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) return NULL; - pCur->db = pState->pTdbState->rocksdb; - pCur->iter = streamStateIterCreate(pState, "state", &pCur->snapshot, &pCur->readOpt); + pCur->db = wrapper->rocksdb; + pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot, + (rocksdb_readoptions_t**)&pCur->readOpt); rocksdb_iter_seek(pCur->iter, buf, (size_t)klen); rocksdb_iter_prev(pCur->iter); @@ -1273,11 +1390,13 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState, const SWinK SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateGetCur_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) return NULL; - pCur->db = pState->pTdbState->rocksdb; - pCur->iter = streamStateIterCreate(pState, "state", &pCur->snapshot, &pCur->readOpt); + pCur->db = wrapper->rocksdb; + pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot, + (rocksdb_readoptions_t**)&pCur->readOpt); SStateKey sKey = {.key = *key, .opNum = pState->number}; char buf[128] = {0}; @@ -1326,8 +1445,6 @@ int32_t streamStateSessionPut_rocksdb(SStreamState* pState, const SSessionKey* k int code = 0; SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; STREAM_STATE_PUT_ROCKSDB(pState, "sess", &sKey, value, vLen); - if (code == -1) { - } return code; } int32_t streamStateSessionGet_rocksdb(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { @@ -1345,8 +1462,10 @@ int32_t streamStateSessionGet_rocksdb(SStreamState* pState, SSessionKey* key, vo code = -1; } else { *key = resKey; - *pVal = taosMemoryCalloc(1, *pVLen); - memcpy(*pVal, tmp, *pVLen); + if (pVal != NULL && pVLen != NULL) { + *pVal = taosMemoryCalloc(1, *pVLen); + memcpy(*pVal, tmp, *pVLen); + } } } taosMemoryFree(tmp); @@ -1363,13 +1482,16 @@ int32_t streamStateSessionDel_rocksdb(SStreamState* pState, const SSessionKey* k } SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pState, const SSessionKey* key) { qDebug("streamStateSessionSeekKeyCurrentPrev_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } pCur->number = pState->number; - pCur->db = pState->pTdbState->rocksdb; - pCur->iter = streamStateIterCreate(pState, "sess", &pCur->snapshot, &pCur->readOpt); + pCur->db = wrapper->rocksdb; + pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, + (rocksdb_readoptions_t**)&pCur->readOpt); char buf[128] = {0}; SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; @@ -1403,12 +1525,14 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta } SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, SSessionKey* key) { qDebug("streamStateSessionSeekKeyCurrentNext_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } - pCur->db = pState->pTdbState->rocksdb; - pCur->iter = streamStateIterCreate(pState, "sess", &pCur->snapshot, &pCur->readOpt); + pCur->db = wrapper->rocksdb; + pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, + (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; char buf[128] = {0}; @@ -1439,12 +1563,14 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, const SSessionKey* key) { qDebug("streamStateSessionSeekKeyNext_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } - pCur->db = pState->pTdbState->rocksdb; - pCur->iter = streamStateIterCreate(pState, "sess", &pCur->snapshot, &pCur->readOpt); + pCur->db = wrapper->rocksdb; + pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, + (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; @@ -1487,6 +1613,9 @@ int32_t streamStateSessionGetKVByCur_rocksdb(SStreamStateCur* pCur, SSessionKey* const char* curKey = rocksdb_iter_key(pCur->iter, (size_t*)&kLen); stateSessionKeyDecode((void*)&ktmp, (char*)curKey); + if (pVal != NULL) *pVal = NULL; + if (pVLen != NULL) *pVLen = 0; + SStateSessionKey* pKTmp = &ktmp; const char* vval = rocksdb_iter_value(pCur->iter, (size_t*)&vLen); char* val = NULL; @@ -1494,19 +1623,23 @@ int32_t streamStateSessionGetKVByCur_rocksdb(SStreamStateCur* pCur, SSessionKey* if (len < 0) { return -1; } - if (pVal != NULL) { - *pVal = (char*)val; - } else { - taosMemoryFree(val); - } - if (pVLen != NULL) *pVLen = len; if (pKTmp->opNum != pCur->number) { + taosMemoryFree(val); return -1; } if (pKey->groupId != 0 && pKey->groupId != pKTmp->key.groupId) { + taosMemoryFree(val); return -1; } + + if (pVal != NULL) { + *pVal = (char*)val; + } else { + taosMemoryFree(val); + } + + if (pVLen != NULL) *pVLen = len; *pKey = pKTmp->key; return 0; } @@ -1531,12 +1664,14 @@ int32_t streamStateFillDel_rocksdb(SStreamState* pState, const SWinKey* key) { SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillGetCur_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; if (pCur == NULL) return NULL; - pCur->db = pState->pTdbState->rocksdb; - pCur->iter = streamStateIterCreate(pState, "fill", &pCur->snapshot, &pCur->readOpt); + pCur->db = wrapper->rocksdb; + pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot, + (rocksdb_readoptions_t**)&pCur->readOpt); char buf[128] = {0}; int len = winKeyEncode((void*)key, buf); @@ -1589,13 +1724,15 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillSeekKeyNext_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (!pCur) { return NULL; } - pCur->db = pState->pTdbState->rocksdb; - pCur->iter = streamStateIterCreate(pState, "fill", &pCur->snapshot, &pCur->readOpt); + pCur->db = wrapper->rocksdb; + pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot, + (rocksdb_readoptions_t**)&pCur->readOpt); char buf[128] = {0}; int len = winKeyEncode((void*)key, buf); @@ -1624,13 +1761,15 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const } SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillSeekKeyPrev_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } - pCur->db = pState->pTdbState->rocksdb; - pCur->iter = streamStateIterCreate(pState, "fill", &pCur->snapshot, &pCur->readOpt); + pCur->db = wrapper->rocksdb; + pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot, + (rocksdb_readoptions_t**)&pCur->readOpt); char buf[128] = {0}; int len = winKeyEncode((void*)key, buf); @@ -1659,13 +1798,15 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const } int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { qDebug("streamStateSessionGetKeyByRange_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return -1; } pCur->number = pState->number; - pCur->db = pState->pTdbState->rocksdb; - pCur->iter = streamStateIterCreate(pState, "sess", &pCur->snapshot, &pCur->readOpt); + pCur->db = wrapper->rocksdb; + pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, + (rocksdb_readoptions_t**)&pCur->readOpt); SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; int32_t c = 0; @@ -1735,7 +1876,6 @@ int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKe if (sessionRangeKeyCmpr(&searchKey, key) == 0) { memcpy(tmp, *pVal, valSize); taosMemoryFreeClear(*pVal); - streamStateSessionDel_rocksdb(pState, key); goto _end; } taosMemoryFreeClear(*pVal); @@ -1751,7 +1891,6 @@ int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKe if (code == 0) { if (sessionRangeKeyCmpr(&searchKey, key) == 0) { memcpy(tmp, *pVal, valSize); - streamStateSessionDel_rocksdb(pState, key); goto _end; } } @@ -1809,14 +1948,12 @@ int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* if (code == 0) { if (key->win.skey <= tmpKey.win.skey && tmpKey.win.ekey <= key->win.ekey) { memcpy(tmp, *pVal, valSize); - streamStateSessionDel_rocksdb(pState, key); goto _end; } void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); if (fn(pKeyData, stateKey) == true) { memcpy(tmp, *pVal, valSize); - streamStateSessionDel_rocksdb(pState, key); goto _end; } @@ -1832,7 +1969,6 @@ int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); if (fn(pKeyData, stateKey) == true) { memcpy(tmp, *pVal, valSize); - streamStateSessionDel_rocksdb(pState, key); goto _end; } } @@ -1876,17 +2012,17 @@ int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, voi int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen) { int code = 0; - STREAM_STATE_PUT_ROCKSDB(pState, "default", &key, pVal, pVLen); + STREAM_STATE_PUT_ROCKSDB(pState, "default", key, pVal, pVLen); return code; } int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** pVal, int32_t* pVLen) { int code = 0; - STREAM_STATE_GET_ROCKSDB(pState, "default", &key, pVal, pVLen); + STREAM_STATE_GET_ROCKSDB(pState, "default", key, pVal, pVLen); return code; } int32_t streamDefaultDel_rocksdb(SStreamState* pState, const void* key) { int code = 0; - STREAM_STATE_DEL_ROCKSDB(pState, "default", &key); + STREAM_STATE_DEL_ROCKSDB(pState, "default", key); return code; } @@ -1894,6 +2030,7 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co int code = 0; char* err = NULL; + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; rocksdb_snapshot_t* snapshot = NULL; rocksdb_readoptions_t* readopts = NULL; rocksdb_iterator_t* pIter = streamStateIterCreate(pState, "default", &snapshot, &readopts); @@ -1926,16 +2063,18 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co } rocksdb_iter_next(pIter); } - rocksdb_release_snapshot(pState->pTdbState->rocksdb, snapshot); + rocksdb_release_snapshot(wrapper->rocksdb, snapshot); rocksdb_readoptions_destroy(readopts); rocksdb_iter_destroy(pIter); return code; } void* streamDefaultIterCreate_rocksdb(SStreamState* pState) { - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - pCur->db = pState->pTdbState->rocksdb; - pCur->iter = streamStateIterCreate(pState, "default", &pCur->snapshot, &pCur->readOpt); + pCur->db = wrapper->rocksdb; + pCur->iter = streamStateIterCreate(pState, "default", (rocksdb_snapshot_t**)&pCur->snapshot, + (rocksdb_readoptions_t**)&pCur->readOpt); return pCur; } int32_t streamDefaultIterValid_rocksdb(void* iter) { @@ -1980,7 +2119,8 @@ void streamStateClearBatch(void* pBatch) { rocksdb_writebatch_clear((rocksdb_ void streamStateDestroyBatch(void* pBatch) { rocksdb_writebatch_destroy((rocksdb_writebatch_t*)pBatch); } int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_writebatch_t* pBatch, void* key, void* val, int32_t vlen, int64_t ttl) { - int i = streamGetInit(cfName); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + int i = streamStateGetCfIdx(pState, cfName); if (i < 0) { qError("streamState failed to put to cf name:%s", cfName); @@ -1991,14 +2131,32 @@ int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_wr char* ttlV = NULL; int32_t ttlVLen = ginitDict[i].enValueFunc(val, vlen, ttl, &ttlV); - rocksdb_column_family_handle_t* pCf = pState->pTdbState->pHandle[ginitDict[i].idx]; + rocksdb_column_family_handle_t* pCf = wrapper->pHandle[ginitDict[i].idx]; rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen); taosMemoryFree(ttlV); return 0; } +int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb_writebatch_t* pBatch, void* key, + void* val, int32_t vlen, int64_t ttl, void* tmpBuf) { + char buf[128] = {0}; + int32_t klen = ginitDict[cfIdx].enFunc((void*)key, buf); + char* ttlV = tmpBuf; + int32_t ttlVLen = ginitDict[cfIdx].enValueFunc(val, vlen, ttl, &ttlV); + + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + + rocksdb_column_family_handle_t* pCf = wrapper->pHandle[ginitDict[cfIdx].idx]; + rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen); + + if (tmpBuf == NULL) { + taosMemoryFree(ttlV); + } + return 0; +} int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch) { - char* err = NULL; - rocksdb_write(pState->pTdbState->rocksdb, pState->pTdbState->writeOpts, (rocksdb_writebatch_t*)pBatch, &err); + char* err = NULL; + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + rocksdb_write(wrapper->rocksdb, wrapper->writeOpts, (rocksdb_writebatch_t*)pBatch, &err); if (err != NULL) { qError("streamState failed to write batch, err:%s", err); taosMemoryFree(err); @@ -2006,3 +2164,13 @@ int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch) { } return 0; } + +uint32_t nextPow2(uint32_t x) { + x = x - 1; + x = x | (x >> 1); + x = x | (x >> 2); + x = x | (x >> 4); + x = x | (x >> 8); + x = x | (x >> 16); + return x + 1; +} \ No newline at end of file diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 670cfbead1e180061fe0f972290351125eb9852c..722c557b8f1e2b6b44bf851454f60ed6ca14ad23 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -123,7 +123,7 @@ int32_t tDecodeSStreamCheckpointRsp(SDecoder* pDecoder, SStreamCheckpointRsp* pR static int32_t streamAlignCheckpoint(SStreamTask* pTask, int64_t checkpointId, int32_t childId) { if (pTask->checkpointingId == 0) { pTask->checkpointingId = checkpointId; - pTask->checkpointAlignCnt = taosArrayGetSize(pTask->childEpInfo); + pTask->checkpointAlignCnt = taosArrayGetSize(pTask->pUpstreamEpInfoList); } ASSERT(pTask->checkpointingId == checkpointId); @@ -165,7 +165,7 @@ int32_t streamProcessCheckpointReq(SStreamMeta* pMeta, SStreamTask* pTask, SStre int64_t checkpointId = pReq->checkpointId; int32_t childId = pReq->childId; - if (taosArrayGetSize(pTask->childEpInfo) > 0) { + if (taosArrayGetSize(pTask->pUpstreamEpInfoList) > 0) { code = streamAlignCheckpoint(pTask, checkpointId, childId); if (code > 0) { return 0; diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index e574cdbe8a7cf8e1166a47516a4ee0ef8021bc35..bb4b842787a640435f561d6e75074869da8885af 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -13,22 +13,31 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" + +SStreamDataBlock* createStreamDataFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg) { + SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, pReq->totalLen); + if (pData == NULL) { + return NULL; + } + + pData->type = blockType; + pData->srcVgId = srcVg; -int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData) { int32_t blockNum = pReq->blockNum; SArray* pArray = taosArrayInit_s(sizeof(SSDataBlock), blockNum); if (pArray == NULL) { - return -1; + taosFreeQitem(pData); + return NULL; } - ASSERT(pReq->blockNum == taosArrayGetSize(pReq->data)); - ASSERT(pReq->blockNum == taosArrayGetSize(pReq->dataLen)); + ASSERT((pReq->blockNum == taosArrayGetSize(pReq->data)) && (pReq->blockNum == taosArrayGetSize(pReq->dataLen))); for (int32_t i = 0; i < blockNum; i++) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*) taosArrayGetP(pReq->data, i); SSDataBlock* pDataBlock = taosArrayGet(pArray, i); blockDecode(pDataBlock, pRetrieve->data); + // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); @@ -39,8 +48,41 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock pDataBlock->info.type = pRetrieve->streamBlockType; pDataBlock->info.childId = pReq->upstreamChildId; } + pData->blocks = pArray; - return 0; + return pData; +} + +SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, SArray* pRes) { + SStreamDataBlock* pStreamBlocks = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, resultSize); + if (pStreamBlocks == NULL) { + taosArrayClearEx(pRes, (FDelete)blockDataFreeRes); + return NULL; + } + + pStreamBlocks->type = STREAM_INPUT__DATA_BLOCK; + pStreamBlocks->blocks = pRes; + + if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)pItem; + pStreamBlocks->childId = pTask->info.selfChildId; + pStreamBlocks->sourceVer = pSubmit->ver; + } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { + SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)pItem; + pStreamBlocks->childId = pTask->info.selfChildId; + pStreamBlocks->sourceVer = pMerged->ver; + } + + return pStreamBlocks; +} + +void destroyStreamDataBlock(SStreamDataBlock* pBlock) { + if (pBlock == NULL) { + return; + } + + taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); + taosFreeQitem(pBlock); } int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData) { @@ -67,8 +109,8 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock return 0; } -SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type) { - SStreamDataSubmit2* pDataSubmit = (SStreamDataSubmit2*)taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, submit.msgLen); +SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type) { + SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, pData->msgLen); if (pDataSubmit == NULL) { return NULL; } @@ -79,14 +121,14 @@ SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type) { return NULL; } - pDataSubmit->submit = submit; + pDataSubmit->submit = *pData; *pDataSubmit->dataRef = 1; // initialize the reference count to be 1 pDataSubmit->type = type; return pDataSubmit; } -void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit) { +void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit) { int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); ASSERT(ref >= 0 && pDataSubmit->type == STREAM_INPUT__DATA_SUBMIT); @@ -96,8 +138,8 @@ void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit) { } } -SStreamMergedSubmit2* streamMergedSubmitNew() { - SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)taosAllocateQitem(sizeof(SStreamMergedSubmit2), DEF_QITEM, 0); +SStreamMergedSubmit* streamMergedSubmitNew() { + SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)taosAllocateQitem(sizeof(SStreamMergedSubmit), DEF_QITEM, 0); if (pMerged == NULL) { return NULL; } @@ -116,34 +158,16 @@ SStreamMergedSubmit2* streamMergedSubmitNew() { return pMerged; } -int32_t streamMergeSubmit(SStreamMergedSubmit2* pMerged, SStreamDataSubmit2* pSubmit) { +int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubmit) { taosArrayPush(pMerged->dataRefs, &pSubmit->dataRef); taosArrayPush(pMerged->submits, &pSubmit->submit); pMerged->ver = pSubmit->ver; return 0; } -static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit2* pDataSubmit) { - atomic_add_fetch_32(pDataSubmit->dataRef, 1); -} - -SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit) { - int32_t len = 0; - if (pSubmit->type == STREAM_INPUT__DATA_SUBMIT) { - len = pSubmit->submit.msgLen; - } - - SStreamDataSubmit2* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, len); - if (pSubmitClone == NULL) { - return NULL; - } - - streamDataSubmitRefInc(pSubmit); - memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit2)); - return pSubmitClone; -} - SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem) { + terrno = 0; + if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) { SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)pElem; @@ -152,21 +176,25 @@ SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* taosFreeQitem(pElem); return dst; } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)dst; - SStreamDataSubmit2* pBlockSrc = (SStreamDataSubmit2*)pElem; + SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)dst; + SStreamDataSubmit* pBlockSrc = (SStreamDataSubmit*)pElem; streamMergeSubmit(pMerged, pBlockSrc); taosFreeQitem(pElem); return dst; } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamMergedSubmit2* pMerged = streamMergedSubmitNew(); - // todo handle error + SStreamMergedSubmit* pMerged = streamMergedSubmitNew(); + if (pMerged == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } - streamMergeSubmit(pMerged, (SStreamDataSubmit2*)dst); - streamMergeSubmit(pMerged, (SStreamDataSubmit2*)pElem); + streamMergeSubmit(pMerged, (SStreamDataSubmit*)dst); + streamMergeSubmit(pMerged, (SStreamDataSubmit*)pElem); taosFreeQitem(dst); taosFreeQitem(pElem); return (SStreamQueueItem*)pMerged; } else { + qDebug("block type:%d not merged with existed blocks list, type:%d", pElem->type, dst->type); return NULL; } } @@ -180,15 +208,17 @@ void streamFreeQitem(SStreamQueueItem* data) { taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(data); } else if (type == STREAM_INPUT__DATA_SUBMIT) { - streamDataSubmitDestroy((SStreamDataSubmit2*)data); + streamDataSubmitDestroy((SStreamDataSubmit*)data); taosFreeQitem(data); } else if (type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit2* pMerge = (SStreamMergedSubmit2*)data; - int32_t sz = taosArrayGetSize(pMerge->submits); + SStreamMergedSubmit* pMerge = (SStreamMergedSubmit*)data; + + int32_t sz = taosArrayGetSize(pMerge->submits); for (int32_t i = 0; i < sz; i++) { int32_t* pRef = taosArrayGetP(pMerge->dataRefs, i); int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); + if (ref == 0) { SPackedData* pSubmit = (SPackedData*)taosArrayGet(pMerge->submits, i); taosMemoryFree(pSubmit->msgStr); @@ -200,13 +230,7 @@ void streamFreeQitem(SStreamQueueItem* data) { taosFreeQitem(pMerge); } else if (type == STREAM_INPUT__REF_DATA_BLOCK) { SStreamRefDataBlock* pRefBlock = (SStreamRefDataBlock*)data; - - int32_t ref = atomic_sub_fetch_32(pRefBlock->dataRef, 1); - ASSERT(ref >= 0); - if (ref == 0) { - blockDataDestroy(pRefBlock->pBlock); - taosMemoryFree(pRefBlock->dataRef); - } + blockDataDestroy(pRefBlock->pBlock); taosFreeQitem(pRefBlock); } } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index a757d39d3f2be5bf5d2e528054e0faa18a3f457b..6771d0cc283c54b42c958cb8d3916fef31c4e821 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -13,9 +13,25 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" +#include "ttimer.h" + +#define MAX_BLOCK_NAME_NUM 1024 +#define DISPATCH_RETRY_INTERVAL_MS 300 +#define MAX_CONTINUE_RETRY_COUNT 5 + +typedef struct SBlockName { + uint32_t hashValue; + char parTbName[TSDB_TABLE_NAME_LEN]; +} SBlockName; + +static void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen) { + pMsg->msgType = msgType; + pMsg->pCont = pCont; + pMsg->contLen = contLen; +} -int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* pReq) { +static int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; @@ -24,6 +40,7 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p if (tEncodeI32(pEncoder, pReq->upstreamChildId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->blockNum) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->totalLen) < 0) return -1; ASSERT(taosArrayGetSize(pReq->data) == pReq->blockNum); ASSERT(taosArrayGetSize(pReq->dataLen) == pReq->blockNum); for (int32_t i = 0; i < pReq->blockNum; i++) { @@ -36,6 +53,37 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p return pEncoder->pos; } +static int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) { + int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + void* buf = taosMemoryCalloc(1, dataStrLen); + if (buf == NULL) return -1; + + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; + pRetrieve->useconds = 0; + pRetrieve->precision = TSDB_DEFAULT_PRECISION; + pRetrieve->compressed = 0; + pRetrieve->completed = 1; + pRetrieve->streamBlockType = pBlock->info.type; + pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); + pRetrieve->skey = htobe64(pBlock->info.window.skey); + pRetrieve->ekey = htobe64(pBlock->info.window.ekey); + pRetrieve->version = htobe64(pBlock->info.version); + pRetrieve->watermark = htobe64(pBlock->info.watermark); + memcpy(pRetrieve->parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); + + int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); + pRetrieve->numOfCols = htonl(numOfCols); + + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); + actualLen += sizeof(SRetrieveTableRsp); + ASSERT(actualLen <= dataStrLen); + taosArrayPush(pReq->dataLen, &actualLen); + taosArrayPush(pReq->data, &buf); + + pReq->totalLen += dataStrLen; + return 0; +} + int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; @@ -45,6 +93,8 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { if (tDecodeI32(pDecoder, &pReq->upstreamChildId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->blockNum) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->totalLen) < 0) return -1; + ASSERT(pReq->blockNum > 0); pReq->data = taosArrayInit(pReq->blockNum, sizeof(void*)); pReq->dataLen = taosArrayInit(pReq->blockNum, sizeof(int32_t)); @@ -62,6 +112,27 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { return 0; } +int32_t tInitStreamDispatchReq(SStreamDispatchReq* pReq, const SStreamTask* pTask, int32_t vgId, int32_t numOfBlocks, + int64_t dstTaskId) { + pReq->streamId = pTask->id.streamId; + pReq->dataSrcVgId = vgId; + pReq->upstreamTaskId = pTask->id.taskId; + pReq->upstreamChildId = pTask->info.selfChildId; + pReq->upstreamNodeId = pTask->info.nodeId; + pReq->blockNum = numOfBlocks; + pReq->taskId = dstTaskId; + + pReq->data = taosArrayInit(numOfBlocks, POINTER_BYTES); + pReq->dataLen = taosArrayInit(numOfBlocks, sizeof(int32_t)); + if (pReq->data == NULL || pReq->dataLen == NULL) { + taosArrayDestroyP(pReq->data, taosMemoryFree); + taosArrayDestroy(pReq->dataLen); + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + void tDeleteStreamDispatchReq(SStreamDispatchReq* pReq) { taosArrayDestroyP(pReq->data, taosMemoryFree); taosArrayDestroy(pReq->dataLen); @@ -122,20 +193,19 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) SStreamRetrieveReq req = { .streamId = pTask->id.streamId, - .srcNodeId = pTask->nodeId, + .srcNodeId = pTask->info.nodeId, .srcTaskId = pTask->id.taskId, .pRetrieve = pRetrieve, .retrieveLen = dataStrLen, }; - int32_t sz = taosArrayGetSize(pTask->childEpInfo); + int32_t sz = taosArrayGetSize(pTask->pUpstreamEpInfoList); ASSERT(sz > 0); for (int32_t i = 0; i < sz; i++) { req.reqId = tGenIdPI64(); - SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->childEpInfo, i); + SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->pUpstreamEpInfoList, i); req.dstNodeId = pEpInfo->nodeId; req.dstTaskId = pEpInfo->taskId; - int32_t code; int32_t len; tEncodeSize(tEncodeStreamRetrieveReq, &req, len, code); if (code < 0) { @@ -155,66 +225,31 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) tEncodeStreamRetrieveReq(&encoder, &req); tEncoderClear(&encoder); - SRpcMsg rpcMsg = { - .code = 0, - .msgType = TDMT_STREAM_RETRIEVE, - .pCont = buf, - .contLen = sizeof(SMsgHead) + len, - }; - + SRpcMsg rpcMsg = {.code = 0, .msgType = TDMT_STREAM_RETRIEVE, .pCont = buf, .contLen = sizeof(SMsgHead) + len}; if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) { ASSERT(0); goto CLEAR; } - buf = NULL; - qDebug("s-task:%s (child %d) send retrieve req to task %d at node %d, reqId %" PRId64, pTask->id.idStr, - pTask->selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId); + buf = NULL; + qDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr, + pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId); } code = 0; + CLEAR: taosMemoryFree(pRetrieve); rpcFreeCont(buf); return code; } -static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) { - int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); - void* buf = taosMemoryCalloc(1, dataStrLen); - if (buf == NULL) return -1; - - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; - pRetrieve->useconds = 0; - pRetrieve->precision = TSDB_DEFAULT_PRECISION; - pRetrieve->compressed = 0; - pRetrieve->completed = 1; - pRetrieve->streamBlockType = pBlock->info.type; - pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); - pRetrieve->skey = htobe64(pBlock->info.window.skey); - pRetrieve->ekey = htobe64(pBlock->info.window.ekey); - pRetrieve->version = htobe64(pBlock->info.version); - pRetrieve->watermark = htobe64(pBlock->info.watermark); - memcpy(pRetrieve->parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); - - int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); - pRetrieve->numOfCols = htonl(numOfCols); - - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); - actualLen += sizeof(SRetrieveTableRsp); - ASSERT(actualLen <= dataStrLen); - taosArrayPush(pReq->dataLen, &actualLen); - taosArrayPush(pReq->data, &buf); - - return 0; -} - int32_t streamDispatchCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet) { void* buf = NULL; int32_t code = -1; SRpcMsg msg = {0}; int32_t tlen; - tEncodeSize(tEncodeSStreamTaskCheckReq, pReq, tlen, code); + tEncodeSize(tEncodeStreamTaskCheckReq, pReq, tlen, code); if (code < 0) { return -1; } @@ -229,7 +264,7 @@ int32_t streamDispatchCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pR SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); - if ((code = tEncodeSStreamTaskCheckReq(&encoder, pReq)) < 0) { + if ((code = tEncodeStreamTaskCheckReq(&encoder, pReq)) < 0) { rpcFreeCont(buf); return code; } @@ -240,21 +275,21 @@ int32_t streamDispatchCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pR msg.pCont = buf; msg.msgType = TDMT_STREAM_TASK_CHECK; - qDebug("s-task:%s dispatch check msg to downstream s-task:%" PRIx64 ":%d node %d: check msg", pTask->id.idStr, - pReq->streamId, pReq->downstreamTaskId, nodeId); + qDebug("s-task:%s (level:%d) dispatch check msg to s-task:%" PRIx64 ":0x%x (vgId:%d)", pTask->id.idStr, + pTask->info.taskLevel, pReq->streamId, pReq->downstreamTaskId, nodeId); tmsgSendReq(pEpSet, &msg); return 0; } -int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, - SEpSet* pEpSet) { +int32_t streamDoDispatchScanHistoryFinishMsg(SStreamTask* pTask, const SStreamScanHistoryFinishReq* pReq, int32_t vgId, + SEpSet* pEpSet) { void* buf = NULL; int32_t code = -1; SRpcMsg msg = {0}; int32_t tlen; - tEncodeSize(tEncodeSStreamRecoverFinishReq, pReq, tlen, code); + tEncodeSize(tEncodeStreamScanHistoryFinishReq, pReq, tlen, code); if (code < 0) { return -1; } @@ -270,7 +305,7 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); - if ((code = tEncodeSStreamRecoverFinishReq(&encoder, pReq)) < 0) { + if ((code = tEncodeStreamScanHistoryFinishReq(&encoder, pReq)) < 0) { if (buf) { rpcFreeCont(buf); } @@ -281,17 +316,17 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov msg.contLen = tlen + sizeof(SMsgHead); msg.pCont = buf; - msg.msgType = TDMT_STREAM_RECOVER_FINISH; - msg.info.noResp = 1; + msg.msgType = TDMT_STREAM_SCAN_HISTORY_FINISH; tmsgSendReq(pEpSet, &msg); - qDebug("s-task:%s dispatch recover finish msg to taskId:%d node %d: recover finish msg", pTask->id.idStr, - pReq->taskId, vgId); + const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); + qDebug("s-task:%s status:%s dispatch scan-history finish msg to taskId:0x%x (vgId:%d)", pTask->id.idStr, pStatus, + pReq->downstreamTaskId, vgId); return 0; } -int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t vgId, SEpSet* pEpSet) { +static int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t vgId, SEpSet* pEpSet) { void* buf = NULL; int32_t code = -1; SRpcMsg msg = {0}; @@ -299,7 +334,10 @@ int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* p // serialize int32_t tlen; tEncodeSize(tEncodeStreamDispatchReq, pReq, tlen, code); - if (code < 0) goto FAIL; + if (code < 0) { + goto FAIL; + } + code = -1; buf = rpcMallocCont(sizeof(SMsgHead) + tlen); if (buf == NULL) { @@ -318,40 +356,60 @@ int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* p msg.contLen = tlen + sizeof(SMsgHead); msg.pCont = buf; - msg.msgType = pTask->dispatchMsgType; + msg.msgType = pTask->msgInfo.msgType; - qDebug("dispatch from s-task:%s to taskId:%d vgId:%d data msg", pTask->id.idStr, pReq->taskId, vgId); - tmsgSendReq(pEpSet, &msg); + qDebug("s-task:%s dispatch msg to taskId:0x%x vgId:%d data msg", pTask->id.idStr, pReq->taskId, vgId); + return tmsgSendReq(pEpSet, &msg); - code = 0; - return 0; FAIL: - if (buf) rpcFreeCont(buf); + if (buf) { + rpcFreeCont(buf); + } + return code; } int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int32_t vgSz, int64_t groupId) { - char* ctbName = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); - if (ctbName == NULL) { - return -1; + uint32_t hashValue = 0; + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + if (pTask->pNameMap == NULL) { + pTask->pNameMap = tSimpleHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); } - if (pDataBlock->info.parTbName[0]) { - snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->shuffleDispatcher.dbInfo.db, pDataBlock->info.parTbName); + void* pVal = tSimpleHashGet(pTask->pNameMap, &groupId, sizeof(int64_t)); + if (pVal) { + SBlockName* pBln = (SBlockName*)pVal; + hashValue = pBln->hashValue; + if (!pDataBlock->info.parTbName[0]) { + memset(pDataBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); + memcpy(pDataBlock->info.parTbName, pBln->parTbName, strlen(pBln->parTbName)); + } } else { - char* ctbShortName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, groupId); - snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->shuffleDispatcher.dbInfo.db, ctbShortName); - taosMemoryFree(ctbShortName); - } + char* ctbName = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); + if (ctbName == NULL) { + return -1; + } - SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + if (pDataBlock->info.parTbName[0]) { + snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->shuffleDispatcher.dbInfo.db, pDataBlock->info.parTbName); + } else { + buildCtbNameByGroupIdImpl(pTask->shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName); + snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->shuffleDispatcher.dbInfo.db, pDataBlock->info.parTbName); + } - /*uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));*/ - SUseDbRsp* pDbInfo = &pTask->shuffleDispatcher.dbInfo; - uint32_t hashValue = - taosGetTbHashVal(ctbName, strlen(ctbName), pDbInfo->hashMethod, pDbInfo->hashPrefix, pDbInfo->hashSuffix); - taosMemoryFree(ctbName); + /*uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));*/ + SUseDbRsp* pDbInfo = &pTask->shuffleDispatcher.dbInfo; + hashValue = + taosGetTbHashVal(ctbName, strlen(ctbName), pDbInfo->hashMethod, pDbInfo->hashPrefix, pDbInfo->hashSuffix); + taosMemoryFree(ctbName); + SBlockName bln = {0}; + bln.hashValue = hashValue; + memcpy(bln.parTbName, pDataBlock->info.parTbName, strlen(pDataBlock->info.parTbName)); + if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) { + tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); + } + } bool found = false; // TODO: optimize search @@ -359,13 +417,16 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S for (j = 0; j < vgSz; j++) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); ASSERT(pVgInfo->vgId > 0); + if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { - if (streamAddBlockToDispatchMsg(pDataBlock, &pReqs[j]) < 0) { + if (streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]) < 0) { return -1; } + if (pReqs[j].blockNum == 0) { atomic_add_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); } + pReqs[j].blockNum++; found = true; break; @@ -376,59 +437,48 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S } int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pData) { - int32_t code = -1; - int32_t blockNum = taosArrayGetSize(pData->blocks); - ASSERT(blockNum != 0); - - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - SStreamDispatchReq req = { - .streamId = pTask->id.streamId, - .dataSrcVgId = pData->srcVgId, - .upstreamTaskId = pTask->id.taskId, - .upstreamChildId = pTask->selfChildId, - .upstreamNodeId = pTask->nodeId, - .blockNum = blockNum, - }; - - req.data = taosArrayInit(blockNum, sizeof(void*)); - req.dataLen = taosArrayInit(blockNum, sizeof(int32_t)); - if (req.data == NULL || req.dataLen == NULL) { - goto FAIL_FIXED_DISPATCH; + int32_t code = 0; + + int32_t numOfBlocks = taosArrayGetSize(pData->blocks); + ASSERT(numOfBlocks != 0); + + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { + SStreamDispatchReq req = {0}; + + int32_t downstreamTaskId = pTask->fixedEpDispatcher.taskId; + code = tInitStreamDispatchReq(&req, pTask, pData->srcVgId, numOfBlocks, downstreamTaskId); + if (code != TSDB_CODE_SUCCESS) { + return code; } - for (int32_t i = 0; i < blockNum; i++) { + for (int32_t i = 0; i < numOfBlocks; i++) { SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i); - if (streamAddBlockToDispatchMsg(pDataBlock, &req) < 0) { - goto FAIL_FIXED_DISPATCH; + code = streamAddBlockIntoDispatchMsg(pDataBlock, &req); + + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroyP(req.data, taosMemoryFree); + taosArrayDestroy(req.dataLen); + return code; } } int32_t vgId = pTask->fixedEpDispatcher.nodeId; SEpSet* pEpSet = &pTask->fixedEpDispatcher.epSet; - int32_t downstreamTaskId = pTask->fixedEpDispatcher.taskId; - req.taskId = downstreamTaskId; + qDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to s-task:0x%x (vgId:%d)", pTask->id.idStr, + pTask->info.selfChildId, numOfBlocks, downstreamTaskId, vgId); - qDebug("s-task:%s (child taskId:%d) fix-dispatch blocks:%d to down stream s-task:%d in vgId:%d", pTask->id.idStr, - pTask->selfChildId, blockNum, downstreamTaskId, vgId); - - if (streamDispatchOneDataReq(pTask, &req, vgId, pEpSet) < 0) { - goto FAIL_FIXED_DISPATCH; - } - - code = 0; - - FAIL_FIXED_DISPATCH: + code = doSendDispatchMsg(pTask, &req, vgId, pEpSet); taosArrayDestroyP(req.data, taosMemoryFree); taosArrayDestroy(req.dataLen); return code; - - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t rspCnt = atomic_load_32(&pTask->shuffleDispatcher.waitingRspCnt); ASSERT(rspCnt == 0); - SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t vgSz = taosArrayGetSize(vgInfo); + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + int32_t vgSz = taosArrayGetSize(vgInfo); + SStreamDispatchReq* pReqs = taosMemoryCalloc(vgSz, sizeof(SStreamDispatchReq)); if (pReqs == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -436,36 +486,30 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat } for (int32_t i = 0; i < vgSz; i++) { - pReqs[i].streamId = pTask->id.streamId; - pReqs[i].dataSrcVgId = pData->srcVgId; - pReqs[i].upstreamTaskId = pTask->id.taskId; - pReqs[i].upstreamChildId = pTask->selfChildId; - pReqs[i].upstreamNodeId = pTask->nodeId; - pReqs[i].blockNum = 0; - pReqs[i].data = taosArrayInit(0, sizeof(void*)); - pReqs[i].dataLen = taosArrayInit(0, sizeof(int32_t)); - if (pReqs[i].data == NULL || pReqs[i].dataLen == NULL) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); + code = tInitStreamDispatchReq(&pReqs[i], pTask, pData->srcVgId, 0, pVgInfo->taskId); + if (code != TSDB_CODE_SUCCESS) { goto FAIL_SHUFFLE_DISPATCH; } - - SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - pReqs[i].taskId = pVgInfo->taskId; } - for (int32_t i = 0; i < blockNum; i++) { + for (int32_t i = 0; i < numOfBlocks; i++) { SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i); // TODO: do not use broadcast if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + for (int32_t j = 0; j < vgSz; j++) { - if (streamAddBlockToDispatchMsg(pDataBlock, &pReqs[j]) < 0) { + if (streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]) < 0) { goto FAIL_SHUFFLE_DISPATCH; } + if (pReqs[j].blockNum == 0) { atomic_add_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); } pReqs[j].blockNum++; } + continue; } @@ -474,16 +518,17 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat } } - qDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to %d vgroups", pTask->id.idStr, pTask->selfChildId, - blockNum, vgSz); + qDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to %d vgroups", pTask->id.idStr, pTask->info.selfChildId, + numOfBlocks, vgSz); for (int32_t i = 0; i < vgSz; i++) { if (pReqs[i].blockNum > 0) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - qDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to vgId:%d", pTask->id.idStr, pTask->selfChildId, + qDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to vgId:%d", pTask->id.idStr, pTask->info.selfChildId, pReqs[i].blockNum, pVgInfo->vgId); - if (streamDispatchOneDataReq(pTask, &pReqs[i], pVgInfo->vgId, &pVgInfo->epSet) < 0) { + code = doSendDispatchMsg(pTask, &pReqs[i], pVgInfo->vgId, &pVgInfo->epSet); + if (code < 0) { goto FAIL_SHUFFLE_DISPATCH; } } @@ -496,44 +541,191 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat taosArrayDestroyP(pReqs[i].data, taosMemoryFree); taosArrayDestroy(pReqs[i].dataLen); } + taosMemoryFree(pReqs); } + return code; } -int32_t streamDispatch(SStreamTask* pTask) { - ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH); +static void doRetryDispatchData(void* param, void* tmrId) { + SStreamTask* pTask = param; + + if (streamTaskShouldStop(&pTask->status)) { + atomic_sub_fetch_8(&pTask->status.timerActive, 1); + qDebug("s-task:%s should stop, abort from timer", pTask->id.idStr); + return; + } + + ASSERT(pTask->outputInfo.status == TASK_OUTPUT_STATUS__WAIT); + + int32_t code = streamDispatchAllBlocks(pTask, pTask->msgInfo.pData); + if (code != TSDB_CODE_SUCCESS) { + if (!streamTaskShouldStop(&pTask->status)) { + qDebug("s-task:%s reset the waitRspCnt to be 0 before launch retry dispatch", pTask->id.idStr); + atomic_store_32(&pTask->shuffleDispatcher.waitingRspCnt, 0); + streamRetryDispatchStreamBlock(pTask, DISPATCH_RETRY_INTERVAL_MS); + } else { + atomic_sub_fetch_8(&pTask->status.timerActive, 1); + qDebug("s-task:%s should stop, abort from timer", pTask->id.idStr); + } + } else { + atomic_sub_fetch_8(&pTask->status.timerActive, 1); + } +} + +void streamRetryDispatchStreamBlock(SStreamTask* pTask, int64_t waitDuration) { + qError("s-task:%s dispatch data in %"PRId64"ms", pTask->id.idStr, waitDuration); + taosTmrReset(doRetryDispatchData, waitDuration, pTask, streamEnv.timer, &pTask->launchTaskTimer); +} + +int32_t streamDispatchStreamBlock(SStreamTask* pTask) { + STaskOutputInfo* pInfo = &pTask->outputInfo; + ASSERT((pInfo->type == TASK_OUTPUT__FIXED_DISPATCH || pInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH)); - int32_t numOfElems = taosQueueItemSize(pTask->outputQueue->queue); + int32_t numOfElems = taosQueueItemSize(pInfo->queue->queue); if (numOfElems > 0) { qDebug("s-task:%s try to dispatch intermediate result block to downstream, elem in outputQ:%d", pTask->id.idStr, numOfElems); } - int8_t old = - atomic_val_compare_exchange_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL, TASK_OUTPUT_STATUS__WAIT); + // to make sure only one dispatch is running + int8_t old = atomic_val_compare_exchange_8(&pInfo->status, TASK_OUTPUT_STATUS__NORMAL, TASK_OUTPUT_STATUS__WAIT); if (old != TASK_OUTPUT_STATUS__NORMAL) { - qDebug("s-task:%s task wait for dispatch rsp, not dispatch now", pTask->id.idStr); + qDebug("s-task:%s wait for dispatch rsp, not dispatch now, output status:%d", pTask->id.idStr, old); return 0; } - SStreamDataBlock* pBlock = streamQueueNextItem(pTask->outputQueue); + ASSERT(pTask->msgInfo.pData == NULL); + qDebug("s-task:%s start to dispatch msg, set output status:%d", pTask->id.idStr, pInfo->status); + + SStreamDataBlock* pBlock = streamQueueNextItem(pInfo->queue); if (pBlock == NULL) { - qDebug("s-task:%s stop dispatching since no output in output queue", pTask->id.idStr); - atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); + atomic_store_8(&pInfo->status, TASK_OUTPUT_STATUS__NORMAL); + qDebug("s-task:%s not dispatch since no elems in outputQ, output status:%d", pTask->id.idStr, pInfo->status); return 0; } + pTask->msgInfo.pData = pBlock; ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - int32_t code = 0; - if (streamDispatchAllBlocks(pTask, pBlock) < 0) { - code = -1; - streamQueueProcessFail(pTask->outputQueue); - atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); + int32_t retryCount = 0; + + while (1) { + int32_t code = streamDispatchAllBlocks(pTask, pBlock); + if (code == TSDB_CODE_SUCCESS) { + break; + } + + qDebug("s-task:%s failed to dispatch msg to downstream, code:%s, output status:%d, retry cnt:%d", pTask->id.idStr, + tstrerror(terrno), pInfo->status, retryCount); + + // todo deal with only partially success dispatch case + atomic_store_32(&pTask->shuffleDispatcher.waitingRspCnt, 0); + if (terrno == TSDB_CODE_APP_IS_STOPPING) { // in case of this error, do not retry anymore + destroyStreamDataBlock(pTask->msgInfo.pData); + pTask->msgInfo.pData = NULL; + return code; + } + + if (++retryCount > MAX_CONTINUE_RETRY_COUNT) { // add to timer to retry + qDebug("s-task:%s failed to dispatch msg to downstream for %d times, code:%s, add timer to retry in %dms", + pTask->id.idStr, retryCount, tstrerror(terrno), DISPATCH_RETRY_INTERVAL_MS); + streamRetryDispatchStreamBlock(pTask, DISPATCH_RETRY_INTERVAL_MS); + break; + } } - taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); - taosFreeQitem(pBlock); - return code; + // this block can not be deleted until it has been sent to downstream task successfully. + return TSDB_CODE_SUCCESS; +} + +int32_t tEncodeCompleteHistoryDataMsg(SEncoder* pEncoder, const SStreamCompleteHistoryMsg* pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->downstreamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->downstreamNode) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeCompleteHistoryDataMsg(SDecoder* pDecoder, SStreamCompleteHistoryMsg* pRsp) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->downstreamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->downstreamNode) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->upstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->upstreamNodeId) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo, SStreamScanHistoryFinishReq* pReq) { + int32_t len = 0; + int32_t code = 0; + SEncoder encoder; + + SStreamCompleteHistoryMsg msg = { + .streamId = pReq->streamId, + .upstreamTaskId = pReq->upstreamTaskId, + .upstreamNodeId = pReq->upstreamNodeId, + .downstreamId = pReq->downstreamTaskId, + .downstreamNode = pTask->pMeta->vgId, + }; + + tEncodeSize(tEncodeCompleteHistoryDataMsg, &msg, len, code); + if (code < 0) { + return code; + } + + void* pBuf = rpcMallocCont(sizeof(SMsgHead) + len); + if (pBuf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + ((SMsgHead*)pBuf)->vgId = htonl(pReq->upstreamNodeId); + + void* abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead)); + + tEncoderInit(&encoder, (uint8_t*)abuf, len); + tEncodeCompleteHistoryDataMsg(&encoder, &msg); + tEncoderClear(&encoder); + + SStreamChildEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId); + + SStreamContinueExecInfo info = {.taskId = pReq->upstreamTaskId, .epset = pInfo->epSet}; + initRpcMsg(&info.msg, 0, pBuf, sizeof(SMsgHead) + len); + info.msg.info = *pRpcInfo; + + taosThreadMutexLock(&pTask->lock); + if (pTask->pRspMsgList == NULL) { + pTask->pRspMsgList = taosArrayInit(4, sizeof(SStreamContinueExecInfo)); + } + taosArrayPush(pTask->pRspMsgList, &info); + taosThreadMutexUnlock(&pTask->lock); + + int32_t num = taosArrayGetSize(pTask->pRspMsgList); + qDebug("s-task:%s add scan history finish rsp msg for task:0x%x, total:%d", pTask->id.idStr, pReq->upstreamTaskId, + num); + return TSDB_CODE_SUCCESS; +} + +int32_t streamNotifyUpstreamContinue(SStreamTask* pTask) { + ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG || pTask->info.taskLevel == TASK_LEVEL__SINK); + + int32_t num = taosArrayGetSize(pTask->pRspMsgList); + for (int32_t i = 0; i < num; ++i) { + SStreamContinueExecInfo* pInfo = taosArrayGet(pTask->pRspMsgList, i); + tmsgSendRsp(&pInfo->msg); + + qDebug("s-task:%s level:%d notify upstream:0x%x to continue process data from WAL", pTask->id.idStr, pTask->info.taskLevel, + pInfo->taskId); + } + + taosArrayClear(pTask->pRspMsgList); + qDebug("s-task:%s level:%d checkpoint ready msg sent to all %d upstreams", pTask->id.idStr, pTask->info.taskLevel, + num); + return 0; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index f4d8522f317ee4c0247edfcce134039d39674d8b..c7da80fdaf216896a6838d372cda4650990b7e90 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -13,11 +13,15 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" // maximum allowed processed block batches. One block may include several submit blocks -#define MAX_STREAM_EXEC_BATCH_NUM 128 -#define MIN_STREAM_EXEC_BATCH_NUM 16 +#define MAX_STREAM_EXEC_BATCH_NUM 32 +#define MIN_STREAM_EXEC_BATCH_NUM 4 +#define MAX_STREAM_RESULT_DUMP_THRESHOLD 100 + +static int32_t updateCheckPointInfo(SStreamTask* pTask); +static int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask); bool streamTaskShouldStop(const SStreamStatus* pStatus) { int32_t status = atomic_load_8((int8_t*)&pStatus->taskStatus); @@ -26,59 +30,63 @@ bool streamTaskShouldStop(const SStreamStatus* pStatus) { bool streamTaskShouldPause(const SStreamStatus* pStatus) { int32_t status = atomic_load_8((int8_t*)&pStatus->taskStatus); - return (status == TASK_STATUS__PAUSE); + return (status == TASK_STATUS__PAUSE || status == TASK_STATUS__HALT); } -static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) { - int32_t code = TSDB_CODE_SUCCESS; - void* pExecutor = pTask->exec.pExecutor; +static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* pRes, int32_t size, int64_t* totalSize, + int32_t* totalBlocks) { + int32_t code = updateCheckPointInfo(pTask); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + return code; + } - while (pTask->taskLevel == TASK_LEVEL__SOURCE) { - int8_t status = atomic_load_8(&pTask->status.taskStatus); - if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__PAUSE) { - qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr, - atomic_load_8(&pTask->status.taskStatus)); - taosMsleep(2); - } else { - break; + int32_t numOfBlocks = taosArrayGetSize(pRes); + if (numOfBlocks > 0) { + SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(pItem, pTask, size, pRes); + if (pStreamBlocks == NULL) { + qError("s-task:%s failed to create result stream data block, code:%s", pTask->id.idStr, tstrerror(terrno)); + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + return -1; + } + + qDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks, + size / 1048576.0); + + code = streamTaskOutputResultBlock(pTask, pStreamBlocks); + if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) { // back pressure and record position + destroyStreamDataBlock(pStreamBlocks); + return -1; } - } - // set input - const SStreamQueueItem* pItem = (const SStreamQueueItem*)data; - if (pItem->type == STREAM_INPUT__GET_RES) { - const SStreamTrigger* pTrigger = (const SStreamTrigger*)data; - qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); - } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { - ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - const SStreamDataSubmit2* pSubmit = (const SStreamDataSubmit2*)data; - qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); - qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, pSubmit, - pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); - } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { - const SStreamDataBlock* pBlock = (const SStreamDataBlock*)data; - - SArray* pBlockList = pBlock->blocks; - int32_t numOfBlocks = taosArrayGetSize(pBlockList); - qDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, pTask->id.idStr, numOfBlocks, pBlock->sourceVer); - qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK); - } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { - const SStreamMergedSubmit2* pMerged = (const SStreamMergedSubmit2*)data; - - SArray* pBlockList = pMerged->submits; - int32_t numOfBlocks = taosArrayGetSize(pBlockList); - qDebug("s-task:%s %p set submit input (merged), numOfblocks:%d", pTask->id.idStr, pTask, numOfBlocks); - qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); - } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { - const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)data; - qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK); + *totalSize += size; + *totalBlocks += numOfBlocks; } else { - ASSERT(0); + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); } - // pExecutor + return TSDB_CODE_SUCCESS; +} + +static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, + int32_t* totalBlocks) { + int32_t code = TSDB_CODE_SUCCESS; + void* pExecutor = pTask->exec.pExecutor; + + *totalBlocks = 0; + *totalSize = 0; + + int32_t size = 0; + int32_t numOfBlocks = 0; + SArray* pRes = NULL; + while (1) { + if (pRes == NULL) { + pRes = taosArrayInit(4, sizeof(SSDataBlock)); + } + if (streamTaskShouldStop(&pTask->status)) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); return 0; } @@ -95,19 +103,20 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { - SSDataBlock block = {0}; - - const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)data; + SSDataBlock block = {0}; + const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); block.info.type = STREAM_PULL_OVER; - block.info.childId = pTask->selfChildId; + block.info.childId = pTask->info.selfChildId; taosArrayPush(pRes, &block); + numOfBlocks += 1; - qDebug("task %d(child %d) processed retrieve, reqId %" PRId64, pTask->id.taskId, pTask->selfChildId, + qDebug("s-task:%s(child %d) retrieve process completed, reqId:0x%" PRIx64" dump results", pTask->id.idStr, pTask->info.selfChildId, pRetrieveBlock->reqId); } + break; } @@ -118,300 +127,524 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* continue; } - qDebug("s-task:%s (child %d) executed and get block", pTask->id.idStr, pTask->selfChildId); - SSDataBlock block = {0}; assignOneDataBlock(&block, output); - block.info.childId = pTask->selfChildId; + block.info.childId = pTask->info.selfChildId; + + size += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block); + numOfBlocks += 1; + taosArrayPush(pRes, &block); - } - return 0; -} + qDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr, + pTask->info.selfChildId, numOfBlocks, size / 1048576.0); -int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { - int32_t code = 0; + // current output should be dispatched to down stream nodes + if (numOfBlocks >= MAX_STREAM_RESULT_DUMP_THRESHOLD) { + ASSERT(numOfBlocks == taosArrayGetSize(pRes)); + code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); + pRes = NULL; + size = 0; + numOfBlocks = 0; + } + } + + if (numOfBlocks > 0) { + ASSERT(numOfBlocks == taosArrayGetSize(pRes)); + code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks); + } else { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + } - void* exec = pTask->exec.pExecutor; + return code; +} + +int32_t streamScanExec(SStreamTask* pTask, int32_t batchSize) { + ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); + int32_t code = TSDB_CODE_SUCCESS; + void* exec = pTask->exec.pExecutor; + bool finished = false; qSetStreamOpOpen(exec); - bool finished = false; - while (1) { + while (!finished) { + if (streamTaskShouldPause(&pTask->status)) { + double el = (taosGetTimestampMs() - pTask->tsInfo.step1Start) / 1000.0; + qDebug("s-task:%s paused from the scan-history task, elapsed time:%.2fsec", pTask->id.idStr, el); + break; + } + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - int32_t batchCnt = 0; + int32_t numOfBlocks = 0; while (1) { - if (streamTaskShouldStop(&pTask->status) || streamTaskShouldPause(&pTask->status)) { - taosArrayDestroy(pRes); + if (streamTaskShouldStop(&pTask->status)) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); return 0; } SSDataBlock* output = NULL; uint64_t ts = 0; - if (qExecTask(exec, &output, &ts) < 0) { + code = qExecTask(exec, &output, &ts); + if (code != TSDB_CODE_TSC_QUERY_KILLED && code != TSDB_CODE_SUCCESS) { + qError("%s scan-history data error occurred code:%s, continue scan", pTask->id.idStr, tstrerror(code)); continue; } + + // the generated results before fill-history task been paused, should be dispatched to sink node if (output == NULL) { - if (qStreamRecoverScanFinished(exec)) { - finished = true; - } else { - qSetStreamOpOpen(exec); - } + finished = qStreamRecoverScanFinished(exec); break; } SSDataBlock block = {0}; assignOneDataBlock(&block, output); - block.info.childId = pTask->selfChildId; + block.info.childId = pTask->info.selfChildId; taosArrayPush(pRes, &block); - batchCnt++; - - qDebug("s-task:%s scan exec numOfBlocks:%d, limit:%d", pTask->id.idStr, batchCnt, batchSz); - if (batchCnt >= batchSz) { + if ((++numOfBlocks) >= batchSize) { + qDebug("s-task:%s scan exec numOfBlocks:%d, output limit:%d reached", pTask->id.idStr, numOfBlocks, batchSize); break; } } - if (taosArrayGetSize(pRes) == 0) { - if (finished) { - taosArrayDestroy(pRes); - qDebug("s-task:%s finish recover exec task ", pTask->id.idStr); - break; - } else { - qDebug("s-task:%s continue recover exec task ", pTask->id.idStr); - continue; + if (taosArrayGetSize(pRes) > 0) { + SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); + if (qRes == NULL) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } - } - SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); - if (qRes == NULL) { - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + qRes->type = STREAM_INPUT__DATA_BLOCK; + qRes->blocks = pRes; - qRes->type = STREAM_INPUT__DATA_BLOCK; - qRes->blocks = pRes; - code = streamTaskOutput(pTask, qRes); - if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) { - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - taosFreeQitem(qRes); - return code; + code = streamTaskOutputResultBlock(pTask, qRes); + if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + taosFreeQitem(qRes); + return code; + } + } else { + taosArrayDestroy(pRes); } + } - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { - qDebug("s-task:%s scan exec dispatch blocks:%d", pTask->id.idStr, batchCnt); - streamDispatch(pTask); - } + return 0; +} - if (finished) { - break; +int32_t updateCheckPointInfo(SStreamTask* pTask) { + int64_t ckId = 0; + int64_t dataVer = 0; + qGetCheckpointVersion(pTask->exec.pExecutor, &dataVer, &ckId); + + SCheckpointInfo* pCkInfo = &pTask->chkInfo; + if (ckId > pCkInfo->id) { // save it since the checkpoint is updated + qDebug("s-task:%s exec end, start to update check point, ver from %" PRId64 " to %" PRId64 + ", checkPoint id:%" PRId64 " -> %" PRId64, + pTask->id.idStr, pCkInfo->version, dataVer, pCkInfo->id, ckId); + + pTask->chkInfo = (SCheckpointInfo){.version = dataVer, .id = ckId, .currentVer = pCkInfo->currentVer}; + + taosWLockLatch(&pTask->pMeta->lock); + + streamMetaSaveTask(pTask->pMeta, pTask); + if (streamMetaCommit(pTask->pMeta) < 0) { + taosWUnLockLatch(&pTask->pMeta->lock); + qError("s-task:%s failed to commit stream meta, since %s", pTask->id.idStr, terrstr()); + return -1; + } else { + taosWUnLockLatch(&pTask->pMeta->lock); + qDebug("s-task:%s update checkpoint ver succeed", pTask->id.idStr); } } - return 0; + + return TSDB_CODE_SUCCESS; } -#if 0 -int32_t streamBatchExec(SStreamTask* pTask, int32_t batchLimit) { - // fetch all queue item, merge according to batchLimit - int32_t numOfItems = taosReadAllQitems(pTask->inputQueue1, pTask->inputQall); - if (numOfItems == 0) { - qDebug("task: %d, stream task exec over, queue empty", pTask->id.taskId); - return 0; +static void waitForTaskIdle(SStreamTask* pTask, SStreamTask* pStreamTask) { + // wait for the stream task to be idle + int64_t st = taosGetTimestampMs(); + + while (!streamTaskIsIdle(pStreamTask)) { + qDebug("s-task:%s level:%d wait for stream task:%s to be idle, check again in 100ms", pTask->id.idStr, + pTask->info.taskLevel, pStreamTask->id.idStr); + taosMsleep(100); } - SStreamQueueItem* pMerged = NULL; - SStreamQueueItem* pItem = NULL; - taosGetQitem(pTask->inputQall, (void**)&pItem); - if (pItem == NULL) { - if (pMerged != NULL) { - // process merged item - } else { - return 0; - } + + double el = (taosGetTimestampMs() - st) / 1000.0; + if (el > 0) { + qDebug("s-task:%s wait for stream task:%s for %.2fs to be idle", pTask->id.idStr, + pStreamTask->id.idStr, el); } +} + +static int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) { + SStreamMeta* pMeta = pTask->pMeta; + + SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); + if (pStreamTask == NULL) { + // todo: destroy the fill-history task here + qError("s-task:%s failed to find related stream task:0x%x, it may have been destroyed or closed", pTask->id.idStr, + pTask->streamTaskId.taskId); + return TSDB_CODE_STREAM_TASK_NOT_EXIST; + } else { + qDebug("s-task:%s fill-history task end, update related stream task:%s info, transfer exec state", pTask->id.idStr, + pStreamTask->id.idStr); + } + + ASSERT(pStreamTask->historyTaskId.taskId == pTask->id.taskId && pTask->status.transferState == true); - // if drop - if (pItem->type == STREAM_INPUT__DESTROY) { - // set status drop - return -1; + STimeWindow* pTimeWindow = &pStreamTask->dataRange.window; + + // todo. the dropping status should be append to the status after the halt completed. + // It must be halted for a source stream task, since when the related scan-history-data task start scan the history + // for the step 2. + int8_t status = pStreamTask->status.taskStatus; + if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) { + ASSERT(status == TASK_STATUS__HALT || status == TASK_STATUS__DROPPING); + } else { + ASSERT(status == TASK_STATUS__SCAN_HISTORY); + pStreamTask->status.taskStatus = TASK_STATUS__HALT; + qDebug("s-task:%s halt by related fill-history task:%s", pStreamTask->id.idStr, pTask->id.idStr); } - if (pTask->taskLevel == TASK_LEVEL__SINK) { - ASSERT(((SStreamQueueItem*)pItem)->type == STREAM_INPUT__DATA_BLOCK); - streamTaskOutput(pTask, (SStreamDataBlock*)pItem); + // wait for the stream task to handle all in the inputQ, and to be idle + waitForTaskIdle(pTask, pStreamTask); + + // In case of sink tasks, no need to halt them. + // In case of source tasks and agg tasks, we should HALT them, and wait for them to be idle. And then, it's safe to + // start the task state transfer procedure. + // When a task is idle with halt status, all data in inputQ are consumed. + if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) { + // update the scan data range for source task. + qDebug("s-task:%s level:%d stream task window %" PRId64 " - %" PRId64 " update to %" PRId64 " - %" PRId64 + ", status:%s, sched-status:%d", + pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN, + pTimeWindow->ekey, streamGetTaskStatusStr(TASK_STATUS__NORMAL), pStreamTask->status.schedStatus); + } else { + qDebug("s-task:%s no need to update time window for non-source task", pStreamTask->id.idStr); } - // exec impl + // 1. expand the query time window for stream task of WAL scanner + pTimeWindow->skey = INT64_MIN; + qStreamInfoResetTimewindowFilter(pStreamTask->exec.pExecutor); - // output - // try dispatch - return 0; + // 2. transfer the ownership of executor state + streamTaskReleaseState(pTask); + streamTaskReloadState(pStreamTask); + + // 3. clear the link between fill-history task and stream task info + pStreamTask->historyTaskId.taskId = 0; + + // 4. resume the state of stream task, after this function, the stream task will run immidately. But it can not be + // pause, since the pause allowed attribute is not set yet. + streamTaskResumeFromHalt(pStreamTask); + + qDebug("s-task:%s fill-history task set status to be dropping, save the state into disk", pTask->id.idStr); + + // 5. free it and remove fill-history task from disk meta-store + streamMetaUnregisterTask(pMeta, pTask->id.streamId, pTask->id.taskId); + + // 6. save to disk + taosWLockLatch(&pMeta->lock); + streamMetaSaveTask(pMeta, pStreamTask); + if (streamMetaCommit(pMeta) < 0) { + // persist to disk + } + taosWUnLockLatch(&pMeta->lock); + + // 7. pause allowed. + streamTaskEnablePause(pStreamTask); + if (taosQueueEmpty(pStreamTask->inputQueue->queue)) { + SStreamRefDataBlock* pItem = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0);; + SSDataBlock* pDelBlock = createSpecialDataBlock(STREAM_DELETE_DATA); + pDelBlock->info.rows = 0; + pDelBlock->info.version = 0; + pItem->type = STREAM_INPUT__REF_DATA_BLOCK; + pItem->pBlock = pDelBlock; + int32_t code = tAppendDataToInputQueue(pStreamTask, (SStreamQueueItem*)pItem); + qDebug("s-task:%s append dummy delete block,res:%d", pStreamTask->id.idStr, code); + } + + streamSchedExec(pStreamTask); + streamMetaReleaseTask(pMeta, pStreamTask); + return TSDB_CODE_SUCCESS; } -#endif -int32_t streamExecForAll(SStreamTask* pTask) { - int32_t code = 0; - while (1) { - int32_t batchSize = 1; - int16_t times = 0; +static int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { + int32_t code = TSDB_CODE_SUCCESS; + if (!pTask->status.transferState) { + return code; + } - SStreamQueueItem* pInput = NULL; + int32_t level = pTask->info.taskLevel; + if (level == TASK_LEVEL__SOURCE) { + streamTaskFillHistoryFinished(pTask); + streamTaskEndScanWAL(pTask); + } else if (level == TASK_LEVEL__AGG) { // do transfer task operator states. + code = streamDoTransferStateToStreamTask(pTask); + if (code != TSDB_CODE_SUCCESS) { // todo handle this + return code; + } + } - // merge multiple input data if possible in the input queue. - qDebug("s-task:%s start to extract data block from inputQ", pTask->id.idStr); + return code; +} - while (1) { - SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); - if (qItem == NULL) { - if (pTask->taskLevel == TASK_LEVEL__SOURCE && batchSize < MIN_STREAM_EXEC_BATCH_NUM && times < 5) { - times++; - taosMsleep(1); - qDebug("===stream===try again batchSize:%d", batchSize); - continue; - } +static int32_t extractMsgFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInput, int32_t* numOfBlocks, + const char* id) { + int32_t retryTimes = 0; + int32_t MAX_RETRY_TIMES = 5; - qDebug("===stream===break batchSize:%d", batchSize); - break; + while (1) { + if (streamTaskShouldPause(&pTask->status)) { + qDebug("s-task:%s task should pause, input blocks:%d", pTask->id.idStr, *numOfBlocks); + return TSDB_CODE_SUCCESS; + } + + SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); + if (qItem == NULL) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE && (++retryTimes) < MAX_RETRY_TIMES) { + taosMsleep(10); + qDebug("===stream===try again batchSize:%d, retry:%d", *numOfBlocks, retryTimes); + continue; } - if (pInput == NULL) { - pInput = qItem; - streamQueueProcessSuccess(pTask->inputQueue); - if (pTask->taskLevel == TASK_LEVEL__SINK) { - break; - } - } else { - // todo we need to sort the data block, instead of just appending into the array list. - void* newRet = NULL; - if ((newRet = streamMergeQueueItem(pInput, qItem)) == NULL) { - streamQueueProcessFail(pTask->inputQueue); - break; + qDebug("===stream===break batchSize:%d", *numOfBlocks); + return TSDB_CODE_SUCCESS; + } + + // do not merge blocks for sink node + if (pTask->info.taskLevel == TASK_LEVEL__SINK) { + *numOfBlocks = 1; + *pInput = qItem; + return TSDB_CODE_SUCCESS; + } + + if (*pInput == NULL) { + ASSERT((*numOfBlocks) == 0); + *pInput = qItem; + } else { + // todo we need to sort the data block, instead of just appending into the array list. + void* newRet = streamMergeQueueItem(*pInput, qItem); + if (newRet == NULL) { + if (terrno == 0) { + qDebug("s-task:%s failed to merge blocks from inputQ, numOfBlocks:%d", id, *numOfBlocks); } else { - batchSize++; - pInput = newRet; - streamQueueProcessSuccess(pTask->inputQueue); - if (batchSize > MAX_STREAM_EXEC_BATCH_NUM) { - qDebug("maximum batch limit:%d reached, processing, %s", MAX_STREAM_EXEC_BATCH_NUM, pTask->id.idStr); - break; - } + qDebug("s-task:%s failed to merge blocks from inputQ, numOfBlocks:%d, code:%s", id, *numOfBlocks, + tstrerror(terrno)); } + streamQueueProcessFail(pTask->inputQueue); + return TSDB_CODE_SUCCESS; } + + *pInput = newRet; } - if (streamTaskShouldStop(&pTask->status)) { - if (pInput) { - streamFreeQitem(pInput); - } + *numOfBlocks += 1; + streamQueueProcessSuccess(pTask->inputQueue); - return 0; + if (*numOfBlocks >= MAX_STREAM_EXEC_BATCH_NUM) { + qDebug("s-task:%s batch size limit:%d reached, start to process blocks", id, MAX_STREAM_EXEC_BATCH_NUM); + return TSDB_CODE_SUCCESS; } + } +} + +/** + * todo: the batch of blocks should be tuned dynamic, according to the total elapsed time of each batch of blocks, the + * appropriate batch of blocks should be handled in 5 to 10 sec. + */ +int32_t streamExecForAll(SStreamTask* pTask) { + const char* id = pTask->id.idStr; + while (1) { + int32_t batchSize = 0; + SStreamQueueItem* pInput = NULL; + if (streamTaskShouldStop(&pTask->status)) { + qDebug("s-task:%s stream task stopped, abort", id); + break; + } + + // merge multiple input data if possible in the input queue. + qDebug("s-task:%s start to extract data block from inputQ", id); + + /*int32_t code = */extractMsgFromInputQ(pTask, &pInput, &batchSize, id); if (pInput == NULL) { + ASSERT(batchSize == 0); break; } - if (pTask->taskLevel == TASK_LEVEL__SINK) { + if (pTask->info.taskLevel == TASK_LEVEL__SINK) { ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK); - qDebug("s-task:%s sink node start to sink result. numOfBlocks:%d", pTask->id.idStr, batchSize); - streamTaskOutput(pTask, (SStreamDataBlock*)pInput); + qDebug("s-task:%s sink task start to sink %d blocks", id, batchSize); + streamTaskOutputResultBlock(pTask, (SStreamDataBlock*)pInput); continue; } - SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); - qDebug("s-task:%s start to execute, block batches:%d", pTask->id.idStr, batchSize); - - streamTaskExecImpl(pTask, pInput, pRes); - - int64_t ckId = 0; - int64_t dataVer = 0; - qGetCheckpointVersion(pTask->exec.pExecutor, &dataVer, &ckId); - if (ckId > pTask->chkInfo.id) { // save it since the checkpoint is updated - qDebug("s-task:%s exec end, start to update check point, ver from %" PRId64 " to %" PRId64 - ", checkPoint id:%" PRId64 " -> %" PRId64, - pTask->id.idStr, pTask->chkInfo.version, dataVer, pTask->chkInfo.id, ckId); - - pTask->chkInfo = (SCheckpointInfo){.version = dataVer, .id = ckId, .currentVer = pTask->chkInfo.currentVer}; - - taosWLockLatch(&pTask->pMeta->lock); - - streamMetaSaveTask(pTask->pMeta, pTask); - if (streamMetaCommit(pTask->pMeta) < 0) { - taosWUnLockLatch(&pTask->pMeta->lock); - qError("s-task:%s failed to commit stream meta, since %s", pTask->id.idStr, terrstr()); - return -1; + int64_t st = taosGetTimestampMs(); + qDebug("s-task:%s start to process batch of blocks, num:%d", id, batchSize); + + { + // set input + void* pExecutor = pTask->exec.pExecutor; + + const SStreamQueueItem* pItem = pInput; + if (pItem->type == STREAM_INPUT__GET_RES) { + const SStreamTrigger* pTrigger = (const SStreamTrigger*)pInput; + qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); + } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { + ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); + const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput; + qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); + qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit, + pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); + } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { + const SStreamDataBlock* pBlock = (const SStreamDataBlock*)pInput; + + SArray* pBlockList = pBlock->blocks; + int32_t numOfBlocks = taosArrayGetSize(pBlockList); + qDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, id, numOfBlocks, pBlock->sourceVer); + qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK); + } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { + const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)pInput; + + SArray* pBlockList = pMerged->submits; + int32_t numOfBlocks = taosArrayGetSize(pBlockList); + qDebug("s-task:%s %p set (merged) submit blocks as a batch, numOfBlocks:%d", id, pTask, numOfBlocks); + qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); + } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { + const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)pInput; + qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else { - taosWUnLockLatch(&pTask->pMeta->lock); - qDebug("s-task:%s update checkpoint ver succeed", pTask->id.idStr); + ASSERT(0); } - } else { - qDebug("s-task:%s exec end", pTask->id.idStr); } - if (taosArrayGetSize(pRes) != 0) { - SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); - if (qRes == NULL) { - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - streamFreeQitem(pInput); - return -1; - } - - qRes->type = STREAM_INPUT__DATA_BLOCK; - qRes->blocks = pRes; + int64_t resSize = 0; + int32_t totalBlocks = 0; + streamTaskExecImpl(pTask, pInput, &resSize, &totalBlocks); - if (((SStreamQueueItem*)pInput)->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit2* pSubmit = (SStreamDataSubmit2*)pInput; - qRes->childId = pTask->selfChildId; - qRes->sourceVer = pSubmit->ver; - } else if (((SStreamQueueItem*)pInput)->type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)pInput; - qRes->childId = pTask->selfChildId; - qRes->sourceVer = pMerged->ver; - } + double el = (taosGetTimestampMs() - st) / 1000.0; + qDebug("s-task:%s batch of input blocks exec end, elapsed time:%.2fs, result size:%.2fMiB, numOfBlocks:%d", + id, el, resSize / 1048576.0, totalBlocks); - code = streamTaskOutput(pTask, qRes); - if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) { - // backpressure and record position - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - streamFreeQitem(pInput); - taosFreeQitem(qRes); - return -1; - } - } else { - taosArrayDestroy(pRes); - } streamFreeQitem(pInput); } + return 0; } +// the task may be set dropping/stopping, while it is still in the task queue, therefore, the sched-status can not +// be updated by tryExec function, therefore, the schedStatus will always be the TASK_SCHED_STATUS__WAITING. +bool streamTaskIsIdle(const SStreamTask* pTask) { + return (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE || pTask->status.taskStatus == TASK_STATUS__STOP || + pTask->status.taskStatus == TASK_STATUS__DROPPING); +} + +int32_t streamTaskEndScanWAL(SStreamTask* pTask) { + const char* id = pTask->id.idStr; + double el = (taosGetTimestampMs() - pTask->tsInfo.step2Start) / 1000.0; + qDebug("s-task:%s scan-history from WAL stage(step 2) ended, elapsed time:%.2fs", id, el); + + // 1. notify all downstream tasks to transfer executor state after handle all history blocks. + int32_t code = streamDispatchTransferStateMsg(pTask); + if (code != TSDB_CODE_SUCCESS) { + // todo handle error + } + + // 2. do transfer stream task operator states. + pTask->status.transferState = true; + code = streamDoTransferStateToStreamTask(pTask); + if (code != TSDB_CODE_SUCCESS) { // todo handle error + return code; + } + + return TSDB_CODE_SUCCESS; +} + int32_t streamTryExec(SStreamTask* pTask) { // this function may be executed by multi-threads, so status check is required. int8_t schedStatus = atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE); + const char* id = pTask->id.idStr; + if (schedStatus == TASK_SCHED_STATUS__WAITING) { int32_t code = streamExecForAll(pTask); - if (code < 0) { + if (code < 0) { // todo this status shoudl be removed atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__FAILED); return -1; } // todo the task should be commit here - atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); - qDebug("s-task:%s exec completed", pTask->id.idStr); + if (taosQueueEmpty(pTask->inputQueue->queue)) { + // fill-history WAL scan has completed + if (pTask->status.transferState) { + code = streamTransferStateToStreamTask(pTask); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + streamSchedExec(pTask); + } else { + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + qDebug("s-task:%s exec completed, status:%s, sched-status:%d", id, streamGetTaskStatusStr(pTask->status.taskStatus), + pTask->status.schedStatus); + } + } else { + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + qDebug("s-task:%s exec completed, status:%s, sched-status:%d", id, streamGetTaskStatusStr(pTask->status.taskStatus), + pTask->status.schedStatus); - if (!taosQueueEmpty(pTask->inputQueue->queue) && (!streamTaskShouldStop(&pTask->status)) && (!streamTaskShouldPause(&pTask->status))) { - streamSchedExec(pTask); + if ((!streamTaskShouldStop(&pTask->status)) && (!streamTaskShouldPause(&pTask->status))) { + streamSchedExec(pTask); + } } + } else { + qDebug("s-task:%s already started to exec by other thread, status:%s, sched-status:%d", id, + streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus); } return 0; } + +int32_t streamTaskReleaseState(SStreamTask* pTask) { + qDebug("s-task:%s release exec state", pTask->id.idStr); + void* pExecutor = pTask->exec.pExecutor; + if (pExecutor != NULL) { + int32_t code = qStreamOperatorReleaseState(pExecutor); + return code; + } else { + return TSDB_CODE_SUCCESS; + } +} + +int32_t streamTaskReloadState(SStreamTask* pTask) { + qDebug("s-task:%s reload exec state", pTask->id.idStr); + void* pExecutor = pTask->exec.pExecutor; + if (pExecutor != NULL) { + int32_t code = qStreamOperatorReloadState(pExecutor); + return code; + } else { + return TSDB_CODE_SUCCESS; + } +} + +int32_t streamAlignTransferState(SStreamTask* pTask) { + int32_t numOfUpstream = taosArrayGetSize(pTask->pUpstreamEpInfoList); + int32_t old = atomic_val_compare_exchange_32(&pTask->transferStateAlignCnt, 0, numOfUpstream); + if (old == 0) { + qDebug("s-task:%s set the transfer state aligncnt %d", pTask->id.idStr, numOfUpstream); + } + + return atomic_sub_fetch_32(&pTask->transferStateAlignCnt, 1); +} diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 682ce08c7fcdd1d25279a8e9e8503c6e43dc2991..23534979168ed82a1dfda55a988da3ab54dc1e76 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -15,16 +15,24 @@ #include "executor.h" #include "streamBackendRocksdb.h" -#include "streamInc.h" +#include "streamInt.h" #include "tref.h" #include "ttimer.h" static TdThreadOnce streamMetaModuleInit = PTHREAD_ONCE_INIT; -static int32_t streamBackendId = 0; -static void streamMetaEnvInit() { streamBackendId = taosOpenRef(20, streamBackendCleanup); } +int32_t streamBackendId = 0; +int32_t streamBackendCfWrapperId = 0; + +static void streamMetaEnvInit() { + streamBackendId = taosOpenRef(64, streamBackendCleanup); + streamBackendCfWrapperId = taosOpenRef(64, streamBackendHandleCleanup); +} void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } -void streamMetaCleanup() { taosCloseRef(streamBackendId); } +void streamMetaCleanup() { + taosCloseRef(streamBackendId); + taosCloseRef(streamBackendCfWrapperId); +} SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId) { int32_t code = -1; @@ -58,14 +66,14 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } - _hash_fn_t fp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); + _hash_fn_t fp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR); pMeta->pTasks = taosHashInit(64, fp, true, HASH_NO_LOCK); if (pMeta->pTasks == NULL) { goto _err; } // task list - pMeta->pTaskList = taosArrayInit(4, sizeof(int32_t)); + pMeta->pTaskList = taosArrayInit(4, sizeof(SStreamId)); if (pMeta->pTaskList == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -79,7 +87,6 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->vgId = vgId; pMeta->ahandle = ahandle; pMeta->expandFunc = expandFunc; - pMeta->streamBackendId = streamBackendId; memset(streamPath, 0, len); sprintf(streamPath, "%s/%s", pMeta->path, "state"); @@ -90,11 +97,18 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF } pMeta->streamBackend = streamBackendInit(streamPath); + if (pMeta->streamBackend == NULL) { + goto _err; + } pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); + pMeta->pTaskBackendUnique = + taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); taosMemoryFree(streamPath); taosInitRWLatch(&pMeta->lock); + taosThreadMutexInit(&pMeta->backendMutex, NULL); + return pMeta; _err: @@ -123,64 +137,23 @@ void streamMetaClose(SStreamMeta* pMeta) { if (pIter == NULL) { break; } - - SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->timer) { - taosTmrStop(pTask->timer); - pTask->timer = NULL; - } - - tFreeStreamTask(pTask); + tFreeStreamTask(*(SStreamTask**)pIter); } taosHashCleanup(pMeta->pTasks); taosRemoveRef(streamBackendId, pMeta->streamBackendRid); pMeta->pTaskList = taosArrayDestroy(pMeta->pTaskList); taosMemoryFree(pMeta->path); + taosThreadMutexDestroy(&pMeta->backendMutex); + taosHashCleanup(pMeta->pTaskBackendUnique); taosMemoryFree(pMeta); } -#if 0 -int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg, int32_t msgLen) { - SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); - if (pTask == NULL) { - return -1; - } - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - if (tDecodeStreamTask(&decoder, pTask) < 0) { - tDecoderClear(&decoder); - goto FAIL; - } - tDecoderClear(&decoder); - - if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) { - ASSERT(0); - goto FAIL; - } - - if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { - goto FAIL; - } - - if (tdbTbUpsert(pMeta->pTaskDb, &pTask->id.taskId, sizeof(int32_t), msg, msgLen, pMeta->txn) < 0) { - taosHashRemove(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t)); - ASSERT(0); - goto FAIL; - } - - return 0; - -FAIL: - if (pTask) tFreeStreamTask(pTask); - return -1; -} -#endif - int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { void* buf = NULL; int32_t len; int32_t code; + pTask->ver = SSTREAM_TASK_VER; tEncodeSize(tEncodeStreamTask, pTask, len, code); if (code < 0) { return -1; @@ -196,6 +169,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { tEncoderClear(&encoder); if (tdbTbUpsert(pMeta->pTaskDb, &pTask->id.taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) { + qError("s-task:%s save to disk failed, code:%s", pTask->id.idStr, tstrerror(terrno)); return -1; } @@ -203,47 +177,65 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { return 0; } -// add to the ready tasks hash map, not the restored tasks hash map -int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { - if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) { - tFreeStreamTask(pTask); - return -1; +int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { + int32_t code = tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(taskId), pMeta->txn); + if (code != 0) { + qError("vgId:%d failed to remove task:0x%x from metastore, code:%s", pMeta->vgId, taskId, tstrerror(terrno)); + } else { + qDebug("vgId:%d remove task:0x%x from metastore", pMeta->vgId, taskId); } - if (streamMetaSaveTask(pMeta, pTask) < 0) { - tFreeStreamTask(pTask); - return -1; - } + return code; +} - if (streamMetaCommit(pMeta) < 0) { - tFreeStreamTask(pTask); - return -1; - } +// add to the ready tasks hash map, not the restored tasks hash map +int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded) { + *pAdded = false; - void* p = taosHashGet(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId)); + int64_t keys[2] = {pTask->id.streamId, pTask->id.taskId}; + void* p = taosHashGet(pMeta->pTasks, keys, sizeof(keys)); if (p == NULL) { - taosArrayPush(pMeta->pTaskList, &pTask->id.taskId); + if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) { + tFreeStreamTask(pTask); + return -1; + } + + taosArrayPush(pMeta->pTaskList, &pTask->id); + + if (streamMetaSaveTask(pMeta, pTask) < 0) { + tFreeStreamTask(pTask); + return -1; + } + + if (streamMetaCommit(pMeta) < 0) { + tFreeStreamTask(pTask); + return -1; + } + } else { + return 0; } - taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES); + taosHashPut(pMeta->pTasks, keys, sizeof(keys), &pTask, POINTER_BYTES); + *pAdded = true; return 0; } -int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta) { +int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta) { size_t size = taosHashGetSize(pMeta->pTasks); ASSERT(taosArrayGetSize(pMeta->pTaskList) == taosHashGetSize(pMeta->pTasks)); - return (int32_t)size; } -SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) { +SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { taosRLockLatch(&pMeta->lock); - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); + int64_t keys[2] = {streamId, taskId}; + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, keys, sizeof(keys)); if (ppTask != NULL) { if (!streamTaskShouldStop(&(*ppTask)->status)) { - atomic_add_fetch_32(&(*ppTask)->refCnt, 1); + int32_t ref = atomic_add_fetch_32(&(*ppTask)->refCnt, 1); taosRUnLockLatch(&pMeta->lock); + qTrace("s-task:%s acquire task, ref:%d", (*ppTask)->id.idStr, ref); return *ppTask; } } @@ -253,39 +245,86 @@ SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) { } void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) { - int32_t left = atomic_sub_fetch_32(&pTask->refCnt, 1); - if (left < 0) { - qError("task ref is invalid, ref:%d, %s", left, pTask->id.idStr); - } else if (left == 0) { + int32_t ref = atomic_sub_fetch_32(&pTask->refCnt, 1); + if (ref > 0) { + qTrace("s-task:%s release task, ref:%d", pTask->id.idStr, ref); + } else if (ref == 0) { ASSERT(streamTaskShouldStop(&pTask->status)); + qTrace("s-task:%s all refs are gone, free it", pTask->id.idStr); tFreeStreamTask(pTask); + } else if (ref < 0) { + qError("task ref is invalid, ref:%d, %s", ref, pTask->id.idStr); + } +} + +static void doRemoveIdFromList(SStreamMeta* pMeta, int32_t num, SStreamId* id) { + for (int32_t i = 0; i < num; ++i) { + SStreamId* pTaskId = taosArrayGet(pMeta->pTaskList, i); + if (pTaskId->streamId == id->streamId && pTaskId->taskId == id->taskId) { + taosArrayRemove(pMeta->pTaskList, i); + break; + } } } -void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { +int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { + SStreamTask* pTask = NULL; + + // pre-delete operation taosWLockLatch(&pMeta->lock); - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); + int64_t keys[2] = {streamId, taskId}; + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, keys, sizeof(keys)); if (ppTask) { - SStreamTask* pTask = *ppTask; - taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t)); - tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn); - + pTask = *ppTask; atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING); + } else { + qDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId); + taosWUnLockLatch(&pMeta->lock); + return 0; + } + taosWUnLockLatch(&pMeta->lock); - int32_t num = taosArrayGetSize(pMeta->pTaskList); - for (int32_t i = 0; i < num; ++i) { - int32_t* pTaskId = taosArrayGet(pMeta->pTaskList, i); - if (*pTaskId == taskId) { - taosArrayRemove(pMeta->pTaskList, i); + qDebug("s-task:0x%x set task status:%s and start to unregister it", taskId, + streamGetTaskStatusStr(TASK_STATUS__DROPPING)); + + while (1) { + taosRLockLatch(&pMeta->lock); + ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, keys, sizeof(keys)); + + if (ppTask) { + if ((*ppTask)->status.timerActive == 0) { + taosRUnLockLatch(&pMeta->lock); break; } + + taosMsleep(10); + qDebug("s-task:%s wait for quit from timer", (*ppTask)->id.idStr); + taosRUnLockLatch(&pMeta->lock); + } else { + taosRUnLockLatch(&pMeta->lock); + break; } + } + // let's do delete of stream task + taosWLockLatch(&pMeta->lock); + ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, keys, sizeof(keys)); + if (ppTask) { + taosHashRemove(pMeta->pTasks, keys, sizeof(keys)); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING); + + ASSERT(pTask->status.timerActive == 0); + doRemoveIdFromList(pMeta, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id); + + streamMetaRemoveTask(pMeta, taskId); streamMetaReleaseTask(pMeta, pTask); + } else { + qDebug("vgId:%d failed to find the task:0x%x, it may have been dropped already", pMeta->vgId, taskId); } taosWUnLockLatch(&pMeta->lock); + return 0; } int32_t streamMetaBegin(SStreamMeta* pMeta) { @@ -296,6 +335,7 @@ int32_t streamMetaBegin(SStreamMeta* pMeta) { return 0; } +// todo add error log int32_t streamMetaCommit(SStreamMeta* pMeta) { if (tdbCommit(pMeta->db, pMeta->txn) < 0) { qError("failed to commit stream meta"); @@ -311,6 +351,7 @@ int32_t streamMetaCommit(SStreamMeta* pMeta) { TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } + return 0; } @@ -328,7 +369,9 @@ int32_t streamMetaAbort(SStreamMeta* pMeta) { int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { TBC* pCur = NULL; + if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { + qError("vgId:%d failed to open stream meta, code:%s", pMeta->vgId, tstrerror(terrno)); return -1; } @@ -337,6 +380,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { void* pVal = NULL; int32_t vLen = 0; SDecoder decoder; + SArray* pRecycleList = taosArrayInit(4, sizeof(int32_t)); tdbTbcMoveToFirst(pCur); @@ -346,43 +390,85 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); + taosArrayDestroy(pRecycleList); return -1; } - tDecoderInit(&decoder, (uint8_t*)pVal, vLen); - tDecodeStreamTask(&decoder, pTask); - tDecoderClear(&decoder); - - if (pMeta->expandFunc(pMeta->ahandle, pTask, pTask->chkInfo.version) < 0) { + if (tDecodeStreamTask(&decoder, pTask) < 0) { + tDecoderClear(&decoder); tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); + taosArrayDestroy(pRecycleList); + tFreeStreamTask(pTask); + qError( + "stream read incompatible data, rm %s/vnode/vnode*/tq/stream if taosd cannot start, and rebuild stream " + "manually", + tsDataDir); return -1; } + tDecoderClear(&decoder); + + if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { + int32_t taskId = pTask->id.taskId; + tFreeStreamTask(pTask); + + taosArrayPush(pRecycleList, &taskId); + + int32_t total = taosArrayGetSize(pRecycleList); + qDebug("s-task:0x%x is already dropped, add into recycle list, total:%d", taskId, total); + continue; + } - void* p = taosHashGet(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId)); + // do duplicate task check. + int64_t keys[2] = {pTask->id.streamId, pTask->id.taskId}; + void* p = taosHashGet(pMeta->pTasks, keys, sizeof(keys)); if (p == NULL) { - taosArrayPush(pMeta->pTaskList, &pTask->id.taskId); + if (pMeta->expandFunc(pMeta->ahandle, pTask, pTask->chkInfo.version) < 0) { + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCur); + tFreeStreamTask(pTask); + taosArrayDestroy(pRecycleList); + return -1; + } + + taosArrayPush(pMeta->pTaskList, &pTask->id); + } else { + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCur); + taosMemoryFree(pTask); + continue; } - if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, sizeof(void*)) < 0) { + if (taosHashPut(pMeta->pTasks, keys, sizeof(keys), &pTask, sizeof(void*)) < 0) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); + tFreeStreamTask(pTask); + taosArrayDestroy(pRecycleList); return -1; } - if (pTask->fillHistory) { - pTask->status.taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; - streamTaskCheckDownstream(pTask, ver); - } + ASSERT(pTask->status.downstreamReady == 0); } tdbFree(pKey); tdbFree(pVal); if (tdbTbcClose(pCur) < 0) { + taosArrayDestroy(pRecycleList); return -1; } + if (taosArrayGetSize(pRecycleList) > 0) { + for (int32_t i = 0; i < taosArrayGetSize(pRecycleList); ++i) { + int32_t taskId = *(int32_t*)taosArrayGet(pRecycleList, i); + streamMetaRemoveTask(pMeta, taskId); + } + } + + qDebug("vgId:%d load %d task from disk", pMeta->vgId, (int32_t)taosArrayGetSize(pMeta->pTaskList)); + taosArrayDestroy(pRecycleList); return 0; } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 20abcca1976b66897341bffac74a9b8c73a78c3c..aaf9fdec724cf6765232cb8a77adfb36754ce6d9 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" SStreamQueue* streamQueueOpen(int64_t cap) { SStreamQueue* pQueue = taosMemoryCalloc(1, sizeof(SStreamQueue)); @@ -27,6 +27,7 @@ SStreamQueue* streamQueueOpen(int64_t cap) { taosSetQueueCapacity(pQueue->queue, cap); taosSetQueueMemoryCapacity(pQueue->queue, cap * 1024); return pQueue; + FAIL: if (pQueue->queue) taosCloseQueue(pQueue->queue); if (pQueue->qall) taosFreeQall(pQueue->qall); @@ -105,3 +106,61 @@ SStreamQueueRes streamQueueGetRes(SStreamQueue1* pQueue) { return (SStreamQueueRes){0}; } #endif + +#define MAX_STREAM_EXEC_BATCH_NUM 128 +#define MIN_STREAM_EXEC_BATCH_NUM 16 + +// todo refactor: +// read data from input queue +typedef struct SQueueReader { + SStreamQueue* pQueue; + int32_t taskLevel; + int32_t maxBlocks; // maximum block in one batch + int32_t waitDuration; // maximum wait time to format several block into a batch to process, unit: ms +} SQueueReader; + +SStreamQueueItem* doReadMultiBlocksFromQueue(SQueueReader* pReader, const char* idstr) { + int32_t numOfBlocks = 0; + int32_t tryCount = 0; + SStreamQueueItem* pRet = NULL; + + while (1) { + SStreamQueueItem* qItem = streamQueueNextItem(pReader->pQueue); + if (qItem == NULL) { + if (pReader->taskLevel == TASK_LEVEL__SOURCE && numOfBlocks < MIN_STREAM_EXEC_BATCH_NUM && tryCount < pReader->waitDuration) { + tryCount++; + taosMsleep(1); + qDebug("===stream===try again batchSize:%d", numOfBlocks); + continue; + } + + qDebug("===stream===break batchSize:%d", numOfBlocks); + break; + } + + if (pRet == NULL) { + pRet = qItem; + streamQueueProcessSuccess(pReader->pQueue); + if (pReader->taskLevel == TASK_LEVEL__SINK) { + break; + } + } else { + // todo we need to sort the data block, instead of just appending into the array list. + void* newRet = NULL; + if ((newRet = streamMergeQueueItem(pRet, qItem)) == NULL) { + streamQueueProcessFail(pReader->pQueue); + break; + } else { + numOfBlocks++; + pRet = newRet; + streamQueueProcessSuccess(pReader->pQueue); + if (numOfBlocks > pReader->maxBlocks) { + qDebug("maximum blocks limit:%d reached, processing, %s", pReader->maxBlocks, idstr); + break; + } + } + } + } + + return pRet; +} diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 1fd7f77c5593d154eb0d6a345e4bbee70e751484..1e17ef7ef1b42ab76027402bfcf804ac7946c618 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -13,93 +13,154 @@ * along with this program. If not, see . */ -#include "streamInc.h" - -int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) { - qDebug("s-task:%s at node %d launch recover", pTask->id.idStr, pTask->nodeId); +#include "streamInt.h" +#include "ttimer.h" +#include "wal.h" + +typedef struct SStreamTaskRetryInfo { + SStreamMeta* pMeta; + int32_t taskId; + int64_t streamId; +} SStreamTaskRetryInfo; + +static int32_t streamSetParamForScanHistory(SStreamTask* pTask); +static void launchFillHistoryTask(SStreamTask* pTask); +static void streamTaskSetRangeStreamCalc(SStreamTask* pTask); +static int32_t initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated); + +static void streamTaskSetReady(SStreamTask* pTask, int32_t numOfReqs) { + ASSERT(pTask->status.downstreamReady == 0); + pTask->status.downstreamReady = 1; + + int64_t el = (taosGetTimestampMs() - pTask->tsInfo.init); + qDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%dms, task status:%s", + pTask->id.idStr, numOfReqs, (int32_t) el, streamGetTaskStatusStr(pTask->status.taskStatus)); +} - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__RECOVER_PREPARE); - qDebug("s-task:%s set task status:%d and start recover", pTask->id.idStr, pTask->status.taskStatus); +int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated) { + SStreamScanHistoryReq req; + initScanHistoryReq(pTask, &req, igUntreated); - streamSetParamForRecover(pTask); - streamSourceRecoverPrepareStep1(pTask, version); + int32_t len = sizeof(SStreamScanHistoryReq); + void* serializedReq = rpcMallocCont(len); + if (serializedReq == NULL) { + return -1; + } - SStreamRecoverStep1Req req; - streamBuildSourceRecover1Req(pTask, &req); - int32_t len = sizeof(SStreamRecoverStep1Req); + memcpy(serializedReq, &req, len); - void* serializedReq = rpcMallocCont(len); - if (serializedReq == NULL) { - return -1; - } + SRpcMsg rpcMsg = {.contLen = len, .pCont = serializedReq, .msgType = TDMT_VND_STREAM_SCAN_HISTORY}; + if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) { + /*ASSERT(0);*/ + } - memcpy(serializedReq, &req, len); + return 0; +} - SRpcMsg rpcMsg = { .contLen = len, .pCont = serializedReq, .msgType = TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE }; - if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) { - /*ASSERT(0);*/ - } +const char* streamGetTaskStatusStr(int32_t status) { + switch(status) { + case TASK_STATUS__NORMAL: return "normal"; + case TASK_STATUS__SCAN_HISTORY: return "scan-history"; + case TASK_STATUS__HALT: return "halt"; + case TASK_STATUS__PAUSE: return "paused"; + case TASK_STATUS__DROPPING: return "dropping"; + default:return ""; + } +} - } else if (pTask->taskLevel == TASK_LEVEL__AGG) { - atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); - streamSetParamForRecover(pTask); - streamAggRecoverPrepare(pTask); - } else if (pTask->taskLevel == TASK_LEVEL__SINK) { - atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); +static int32_t doLaunchScanHistoryTask(SStreamTask* pTask) { + SVersionRange* pRange = &pTask->dataRange.range; + if (pTask->info.fillHistory) { + streamSetParamForScanHistory(pTask); } + streamSetParamForStreamScannerStep1(pTask, pRange, &pTask->dataRange.window); + int32_t code = streamStartScanHistoryAsync(pTask, 0); + return code; +} + +int32_t streamTaskLaunchScanHistory(SStreamTask* pTask) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + if (pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) { + return doLaunchScanHistoryTask(pTask); + } else { + ASSERT(pTask->status.taskStatus == TASK_STATUS__NORMAL); + qDebug("s-task:%s no need to scan-history-data, status:%s, sched-status:%d, ver:%" PRId64, pTask->id.idStr, + streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus, + walReaderGetCurrentVer(pTask->exec.pWalReader)); + } + } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) { + if (pTask->info.fillHistory) { + streamSetParamForScanHistory(pTask); + } + streamTaskEnablePause(pTask); + streamTaskScanHistoryPrepare(pTask); + } else if (pTask->info.taskLevel == TASK_LEVEL__SINK) { + qDebug("s-task:%s sink task do nothing to handle scan-history", pTask->id.idStr); + streamTaskScanHistoryPrepare(pTask); + } return 0; } -// checkstatus -int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { - qDebug("s-taks:%s in fill history stage, ver:%"PRId64, pTask->id.idStr, version); +// check status +int32_t streamTaskDoCheckDownstreamTasks(SStreamTask* pTask) { + SHistDataRange* pRange = &pTask->dataRange; + STimeWindow* pWindow = &pRange->window; SStreamTaskCheckReq req = { .streamId = pTask->id.streamId, .upstreamTaskId = pTask->id.taskId, - .upstreamNodeId = pTask->nodeId, - .childId = pTask->selfChildId, + .upstreamNodeId = pTask->info.nodeId, + .childId = pTask->info.selfChildId, }; // serialize - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { req.reqId = tGenIdPI64(); req.downstreamNodeId = pTask->fixedEpDispatcher.nodeId; req.downstreamTaskId = pTask->fixedEpDispatcher.taskId; pTask->checkReqId = req.reqId; - qDebug("s-task:%s at node %d check downstream task %d at node %d", pTask->id.idStr, pTask->nodeId, req.downstreamTaskId, - req.downstreamNodeId); + qDebug("s-task:%s check single downstream task:0x%x(vgId:%d) ver:%" PRId64 "-%" PRId64 " window:%" PRId64 + "-%" PRId64 ", req:0x%" PRIx64, + pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId, pRange->range.minVer, pRange->range.maxVer, + pWindow->skey, pWindow->ekey, req.reqId); + streamDispatchCheckMsg(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t vgSz = taosArrayGetSize(vgInfo); - pTask->recoverTryingDownstream = vgSz; - pTask->checkReqIds = taosArrayInit(vgSz, sizeof(int64_t)); + int32_t numOfVgs = taosArrayGetSize(vgInfo); + pTask->notReadyTasks = numOfVgs; + pTask->checkReqIds = taosArrayInit(numOfVgs, sizeof(int64_t)); + + qDebug("s-task:%s check %d downstream tasks, ver:%" PRId64 "-%" PRId64 " window:%" PRId64 "-%" PRId64, + pTask->id.idStr, numOfVgs, pRange->range.minVer, pRange->range.maxVer, pWindow->skey, pWindow->ekey); - for (int32_t i = 0; i < vgSz; i++) { + for (int32_t i = 0; i < numOfVgs; i++) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); req.reqId = tGenIdPI64(); taosArrayPush(pTask->checkReqIds, &req.reqId); req.downstreamNodeId = pVgInfo->vgId; req.downstreamTaskId = pVgInfo->taskId; - qDebug("s-task:%s at node %d check downstream task %d at node %d (shuffle)", pTask->id.idStr, pTask->nodeId, - req.downstreamTaskId, req.downstreamNodeId); + qDebug("s-task:%s (vgId:%d) check downstream task:0x%x (vgId:%d) (shuffle), idx:%d", pTask->id.idStr, pTask->info.nodeId, + req.downstreamTaskId, req.downstreamNodeId, i); streamDispatchCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); } } else { - qDebug("s-task:%s at node %d direct launch recover since no downstream", pTask->id.idStr, pTask->nodeId); - streamTaskLaunchRecover(pTask, version); + qDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", pTask->id.idStr, pTask->info.nodeId); + + streamTaskSetReady(pTask, 0); + streamTaskSetRangeStreamCalc(pTask); + streamTaskLaunchScanHistory(pTask); + + launchFillHistoryTask(pTask); } return 0; } -int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) { +int32_t streamRecheckDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) { SStreamTaskCheckReq req = { .reqId = pRsp->reqId, .streamId = pRsp->streamId, @@ -110,15 +171,16 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp .childId = pRsp->childId, }; - qDebug("s-task:%s at node %d check downstream task %d at node %d (recheck)", pTask->id.idStr, pTask->nodeId, + qDebug("s-task:%s (vgId:%d) check downstream task:0x%x (vgId:%d) (recheck)", pTask->id.idStr, pTask->info.nodeId, req.downstreamTaskId, req.downstreamNodeId); - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { streamDispatchCheckMsg(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet); - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t vgSz = taosArrayGetSize(vgInfo); - for (int32_t i = 0; i < vgSz; i++) { + + int32_t numOfVgs = taosArrayGetSize(vgInfo); + for (int32_t i = 0; i < numOfVgs; i++) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); if (pVgInfo->taskId == req.downstreamTaskId) { streamDispatchCheckMsg(pTask, &req, pRsp->downstreamNodeId, &pVgInfo->epSet); @@ -130,15 +192,36 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp } int32_t streamTaskCheckStatus(SStreamTask* pTask) { - return atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL? 1:0; + return (pTask->status.downstreamReady == 1)? 1:0; } -int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version) { - qDebug("task %d at node %d recv check rsp from task %d at node %d: status %d", pRsp->upstreamTaskId, - pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status); +static void doProcessDownstreamReadyRsp(SStreamTask* pTask, int32_t numOfReqs) { + streamTaskSetReady(pTask, numOfReqs); + const char* id = pTask->id.idStr; + + int8_t status = pTask->status.taskStatus; + const char* str = streamGetTaskStatusStr(status); + + ASSERT(status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__NORMAL); + streamTaskSetRangeStreamCalc(pTask); + + if (status == TASK_STATUS__SCAN_HISTORY) { + qDebug("s-task:%s enter into scan-history data stage, status:%s", id, str); + streamTaskLaunchScanHistory(pTask); + } else { + qDebug("s-task:%s downstream tasks are ready, now ready for data from wal, status:%s", id, str); + } + + // when current stream task is ready, check the related fill history task. + launchFillHistoryTask(pTask); +} + +int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) { + ASSERT(pTask->id.taskId == pRsp->upstreamTaskId); + const char* id = pTask->id.idStr; if (pRsp->status == 1) { - if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { bool found = false; int32_t numOfReqs = taosArrayGetSize(pTask->checkReqIds); @@ -154,138 +237,453 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* return -1; } - int32_t left = atomic_sub_fetch_32(&pTask->recoverTryingDownstream, 1); + int32_t left = atomic_sub_fetch_32(&pTask->notReadyTasks, 1); ASSERT(left >= 0); if (left == 0) { taosArrayDestroy(pTask->checkReqIds); pTask->checkReqIds = NULL; - qDebug("s-task:%s all downstream tasks:%d are ready, now enter into recover stage", pTask->id.idStr, numOfReqs); - streamTaskLaunchRecover(pTask, version); + doProcessDownstreamReadyRsp(pTask, numOfReqs); + } else { + int32_t total = taosArrayGetSize(pTask->shuffleDispatcher.dbInfo.pVgroupInfos); + qDebug("s-task:%s (vgId:%d) recv check rsp from task:0x%x (vgId:%d) status:%d, total:%d not ready:%d", id, + pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status, total, left); } - } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + } else { + ASSERT(pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH); if (pRsp->reqId != pTask->checkReqId) { return -1; } - streamTaskLaunchRecover(pTask, version); - } else { - ASSERT(0); + doProcessDownstreamReadyRsp(pTask, 1); } - } else { // not ready, wait for 100ms and retry - qDebug("s-task:%s downstream taskId:%d (vgId:%d) not ready, wait for 100ms and retry", pTask->id.idStr, - pRsp->downstreamTaskId, pRsp->downstreamNodeId); + } else { // not ready, wait for 100ms and retry + qDebug("s-task:%s downstream taskId:0x%x (vgId:%d) not ready, wait for 100ms and retry", id, pRsp->downstreamTaskId, + pRsp->downstreamNodeId); taosMsleep(100); - streamRecheckOneDownstream(pTask, pRsp); + + streamRecheckDownstream(pTask, pRsp); } return 0; } +int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp, + SRpcHandleInfo *pRpcInfo, int32_t taskId) { + SEncoder encoder; + int32_t code; + int32_t len; + + tEncodeSize(tEncodeStreamTaskCheckRsp, pRsp, len, code); + if (code < 0) { + qError("vgId:%d failed to encode task check rsp, s-task:0x%x", pMeta->vgId, taskId); + return -1; + } + + void* buf = rpcMallocCont(sizeof(SMsgHead) + len); + ((SMsgHead*)buf)->vgId = htonl(pReq->upstreamNodeId); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncoderInit(&encoder, (uint8_t*)abuf, len); + tEncodeStreamTaskCheckRsp(&encoder, pRsp); + tEncoderClear(&encoder); + + SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo}; + + tmsgSendRsp(&rspMsg); + return 0; +} + // common -int32_t streamSetParamForRecover(SStreamTask* pTask) { - void* exec = pTask->exec.pExecutor; - return qStreamSetParamForRecover(exec); +int32_t streamSetParamForScanHistory(SStreamTask* pTask) { + qDebug("s-task:%s set operator option for scan-history data", pTask->id.idStr); + return qSetStreamOperatorOptionForScanHistory(pTask->exec.pExecutor); } + int32_t streamRestoreParam(SStreamTask* pTask) { - void* exec = pTask->exec.pExecutor; - return qStreamRestoreParam(exec); + qDebug("s-task:%s restore operator param after scan-history", pTask->id.idStr); + return qRestoreStreamOperatorOption(pTask->exec.pExecutor); } int32_t streamSetStatusNormal(SStreamTask* pTask) { - atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); - return 0; + int32_t status = atomic_load_8(&pTask->status.taskStatus); + if (status == TASK_STATUS__DROPPING) { + qError("s-task:%s cannot be set normal, since in dropping state", pTask->id.idStr); + return -1; + } else { + qDebug("s-task:%s set task status to be normal, prev:%s", pTask->id.idStr, streamGetTaskStatusStr(status)); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); + return 0; + } } // source -int32_t streamSourceRecoverPrepareStep1(SStreamTask* pTask, int64_t ver) { - void* exec = pTask->exec.pExecutor; - return qStreamSourceRecoverStep1(exec, ver); +int32_t streamSetParamForStreamScannerStep1(SStreamTask* pTask, SVersionRange *pVerRange, STimeWindow* pWindow) { + return qStreamSourceScanParamForHistoryScanStep1(pTask->exec.pExecutor, pVerRange, pWindow); } -int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamRecoverStep1Req* pReq) { - pReq->msgHead.vgId = pTask->nodeId; +int32_t streamSetParamForStreamScannerStep2(SStreamTask* pTask, SVersionRange *pVerRange, STimeWindow* pWindow) { + return qStreamSourceScanParamForHistoryScanStep2(pTask->exec.pExecutor, pVerRange, pWindow); +} + +int32_t initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated) { + pReq->msgHead.vgId = pTask->info.nodeId; pReq->streamId = pTask->id.streamId; pReq->taskId = pTask->id.taskId; + pReq->igUntreated = igUntreated; return 0; } -int32_t streamSourceRecoverScanStep1(SStreamTask* pTask) { +int32_t streamSourceScanHistoryData(SStreamTask* pTask) { return streamScanExec(pTask, 100); } -int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req* pReq) { - pReq->msgHead.vgId = pTask->nodeId; - pReq->streamId = pTask->id.streamId; - pReq->taskId = pTask->id.taskId; +int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask) { + SStreamScanHistoryFinishReq req = { + .streamId = pTask->id.streamId, + .childId = pTask->info.selfChildId, + .upstreamTaskId = pTask->id.taskId, + .upstreamNodeId = pTask->pMeta->vgId, + }; + + // serialize + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { + req.downstreamTaskId = pTask->fixedEpDispatcher.taskId; + pTask->notReadyTasks = 1; + streamDoDispatchScanHistoryFinishMsg(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + int32_t numOfVgs = taosArrayGetSize(vgInfo); + pTask->notReadyTasks = numOfVgs; + + qDebug("s-task:%s send scan-history data complete msg to downstream (shuffle-dispatch) %d tasks, status:%s", pTask->id.idStr, + numOfVgs, streamGetTaskStatusStr(pTask->status.taskStatus)); + for (int32_t i = 0; i < numOfVgs; i++) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); + req.downstreamTaskId = pVgInfo->taskId; + streamDoDispatchScanHistoryFinishMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); + } + } else { + qDebug("s-task:%s no downstream tasks, invoke scan-history finish rsp directly", pTask->id.idStr); + streamProcessScanHistoryFinishRsp(pTask); + } + return 0; } -int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) { - void* exec = pTask->exec.pExecutor; +static int32_t doDispatchTransferMsg(SStreamTask* pTask, const SStreamTransferReq* pReq, int32_t vgId, SEpSet* pEpSet) { + void* buf = NULL; + int32_t code = -1; + SRpcMsg msg = {0}; - qDebug("s-task:%s recover step2 (blocking stage) started", pTask->id.idStr); - if (qStreamSourceRecoverStep2(exec, ver) < 0) { + int32_t tlen; + tEncodeSize(tEncodeStreamScanHistoryFinishReq, pReq, tlen, code); + if (code < 0) { + return -1; } - return streamScanExec(pTask, 100); + buf = rpcMallocCont(sizeof(SMsgHead) + tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + ((SMsgHead*)buf)->vgId = htonl(vgId); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + SEncoder encoder; + tEncoderInit(&encoder, abuf, tlen); + if ((code = tEncodeStreamScanHistoryFinishReq(&encoder, pReq)) < 0) { + if (buf) { + rpcFreeCont(buf); + } + return code; + } + + tEncoderClear(&encoder); + + msg.contLen = tlen + sizeof(SMsgHead); + msg.pCont = buf; + msg.msgType = TDMT_STREAM_TRANSFER_STATE; + msg.info.noResp = 1; + + tmsgSendReq(pEpSet, &msg); + qDebug("s-task:%s level:%d, status:%s dispatch transfer state msg to taskId:0x%x (vgId:%d)", pTask->id.idStr, + pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus), pReq->downstreamTaskId, vgId); + + return 0; } -int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) { - SStreamRecoverFinishReq req = { .streamId = pTask->id.streamId, .childId = pTask->selfChildId }; +int32_t streamDispatchTransferStateMsg(SStreamTask* pTask) { + SStreamTransferReq req = { .streamId = pTask->id.streamId, .childId = pTask->info.selfChildId }; // serialize - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - qDebug("s-task:%s send recover finish msg to downstream (fix-dispatch) to taskId:%d, status:%d", pTask->id.idStr, - pTask->fixedEpDispatcher.taskId, pTask->status.taskStatus); - - req.taskId = pTask->fixedEpDispatcher.taskId; - streamDispatchOneRecoverFinishReq(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { + req.downstreamTaskId = pTask->fixedEpDispatcher.taskId; + doDispatchTransferMsg(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t vgSz = taosArrayGetSize(vgInfo); - for (int32_t i = 0; i < vgSz; i++) { + + int32_t numOfVgs = taosArrayGetSize(vgInfo); + for (int32_t i = 0; i < numOfVgs; i++) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - req.taskId = pVgInfo->taskId; - streamDispatchOneRecoverFinishReq(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); + req.downstreamTaskId = pVgInfo->taskId; + doDispatchTransferMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); } } + return 0; } // agg -int32_t streamAggRecoverPrepare(SStreamTask* pTask) { - pTask->recoverWaitingUpstream = taosArrayGetSize(pTask->childEpInfo); - qDebug("s-task:%s wait for %d upstreams", pTask->id.idStr, pTask->recoverWaitingUpstream); +int32_t streamTaskScanHistoryPrepare(SStreamTask* pTask) { + pTask->numOfWaitingUpstream = taosArrayGetSize(pTask->pUpstreamEpInfoList); + qDebug("s-task:%s level:%d task wait for %d upstream tasks complete scan-history procedure, status:%s", + pTask->id.idStr, pTask->info.taskLevel, pTask->numOfWaitingUpstream, + streamGetTaskStatusStr(pTask->status.taskStatus)); return 0; } -int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) { +int32_t streamAggUpstreamScanHistoryFinish(SStreamTask* pTask) { void* exec = pTask->exec.pExecutor; - if (qStreamRestoreParam(exec) < 0) { + if (pTask->info.fillHistory && qRestoreStreamOperatorOption(exec) < 0) { return -1; } + if (qStreamRecoverFinish(exec) < 0) { return -1; } - streamSetStatusNormal(pTask); return 0; } -int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId) { - if (pTask->taskLevel == TASK_LEVEL__AGG) { - int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingUpstream, 1); - qDebug("s-task:%s remain unfinished child tasks:%d", pTask->id.idStr, left); - ASSERT(left >= 0); - if (left == 0) { - streamAggChildrenRecoverFinish(pTask); +int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, + SRpcHandleInfo* pRpcInfo) { + int32_t taskLevel = pTask->info.taskLevel; + ASSERT(taskLevel == TASK_LEVEL__AGG || taskLevel == TASK_LEVEL__SINK); + + // sink node do not send end of scan history msg to its upstream, which is agg task. + streamAddEndScanHistoryMsg(pTask, pRpcInfo, pReq); + + int32_t left = atomic_sub_fetch_32(&pTask->numOfWaitingUpstream, 1); + ASSERT(left >= 0); + + if (left == 0) { + int32_t numOfTasks = taosArrayGetSize(pTask->pUpstreamEpInfoList); + qDebug( + "s-task:%s all %d upstream tasks finish scan-history data, set param for agg task for stream data and send " + "rsp to all upstream tasks", + pTask->id.idStr, numOfTasks); + + if (pTask->info.taskLevel == TASK_LEVEL__AGG) { + streamAggUpstreamScanHistoryFinish(pTask); } + + streamNotifyUpstreamContinue(pTask); + + // sink node does not receive the pause msg from mnode, so does not need enable it + if (pTask->info.taskLevel == TASK_LEVEL__AGG) { + streamTaskEnablePause(pTask); + } + } else { + qDebug("s-task:%s receive scan-history data finish msg from upstream:0x%x(index:%d), unfinished:%d", + pTask->id.idStr, pReq->upstreamTaskId, pReq->childId, left); } + return 0; } -int32_t tEncodeSStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) { +int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask) { + ASSERT(pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY); + SStreamMeta* pMeta = pTask->pMeta; + + // execute in the scan history complete call back msg, ready to process data from inputQ + streamSetStatusNormal(pTask); + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + + taosWLockLatch(&pMeta->lock); + streamMetaSaveTask(pMeta, pTask); + streamMetaCommit(pMeta); + taosWUnLockLatch(&pMeta->lock); + + // history data scan in the stream time window finished, now let's enable the pause + streamTaskEnablePause(pTask); + + // for source tasks, let's continue execute. + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + streamSchedExec(pTask); + } + + return TSDB_CODE_SUCCESS; +} + +static void doCheckDownstreamStatus(SStreamTask* pTask, SStreamTask* pHTask) { + pHTask->dataRange.range.minVer = 0; + pHTask->dataRange.range.maxVer = pTask->chkInfo.currentVer; + + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + qDebug("s-task:%s set the launch condition for fill-history s-task:%s, window:%" PRId64 " - %" PRId64 + " ver range:%" PRId64 " - %" PRId64, + pTask->id.idStr, pHTask->id.idStr, pHTask->dataRange.window.skey, pHTask->dataRange.window.ekey, + pHTask->dataRange.range.minVer, pHTask->dataRange.range.maxVer); + } else { + qDebug("s-task:%s no fill history condition for non-source task:%s", pTask->id.idStr, pHTask->id.idStr); + } + + // check if downstream tasks have been ready + streamTaskDoCheckDownstreamTasks(pHTask); +} + +static void tryLaunchHistoryTask(void* param, void* tmrId) { + SStreamTaskRetryInfo* pInfo = param; + SStreamMeta* pMeta = pInfo->pMeta; + + qDebug("s-task:0x%x in timer to launch related history task", pInfo->taskId); + + taosWLockLatch(&pMeta->lock); + int64_t keys[2] = {pInfo->streamId, pInfo->taskId}; + + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, keys, sizeof(keys)); + if (ppTask) { + ASSERT((*ppTask)->status.timerActive >= 1); + + if (streamTaskShouldStop(&(*ppTask)->status)) { + const char* pStatus = streamGetTaskStatusStr((*ppTask)->status.taskStatus); + qDebug("s-task:%s status:%s quit timer task", (*ppTask)->id.idStr, pStatus); + + taosMemoryFree(pInfo); + atomic_sub_fetch_8(&(*ppTask)->status.timerActive, 1); + taosWUnLockLatch(&pMeta->lock); + return; + } + } + taosWUnLockLatch(&pMeta->lock); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pInfo->streamId, pInfo->taskId); + if (pTask != NULL) { + ASSERT(pTask->status.timerActive >= 1); + + // abort the timer if intend to stop task + SStreamTask* pHTask = streamMetaAcquireTask(pMeta, pTask->historyTaskId.streamId, pTask->historyTaskId.taskId); + if (pHTask == NULL && (!streamTaskShouldStop(&pTask->status))) { + const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); + qWarn( + "s-task:%s vgId:%d status:%s failed to launch history task:0x%x, since it may not be built, or may have been " + "destroyed, or should stop", + pTask->id.idStr, pMeta->vgId, pStatus, pTask->historyTaskId.taskId); + + taosTmrReset(tryLaunchHistoryTask, 100, pInfo, streamEnv.timer, &pTask->launchTaskTimer); + streamMetaReleaseTask(pMeta, pTask); + return; + } + + if (pHTask != NULL) { + doCheckDownstreamStatus(pTask, pHTask); + streamMetaReleaseTask(pMeta, pHTask); + } + + // not in timer anymore + atomic_sub_fetch_8(&pTask->status.timerActive, 1); + streamMetaReleaseTask(pMeta, pTask); + } else { + qError("s-task:0x%x failed to load task, it may have been destroyed", pInfo->taskId); + } + + taosMemoryFree(pInfo); +} + +// todo fix the bug: 2. race condition +// an fill history task needs to be started. +int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { + SStreamMeta* pMeta = pTask->pMeta; + int32_t hTaskId = pTask->historyTaskId.taskId; + + int64_t keys[2] = {pTask->historyTaskId.streamId, pTask->historyTaskId.taskId}; + // Set the execute conditions, including the query time window and the version range + SStreamTask** pHTask = taosHashGet(pMeta->pTasks, keys, sizeof(keys)); + if (pHTask == NULL) { + qWarn("s-task:%s vgId:%d failed to launch history task:0x%x, since it is not built yet", pTask->id.idStr, + pMeta->vgId, hTaskId); + + SStreamTaskRetryInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamTaskRetryInfo)); + pInfo->taskId = pTask->id.taskId; + pInfo->streamId = pTask->id.streamId; + pInfo->pMeta = pTask->pMeta; + + if (pTask->launchTaskTimer == NULL) { + pTask->launchTaskTimer = taosTmrStart(tryLaunchHistoryTask, 100, pInfo, streamEnv.timer); + if (pTask->launchTaskTimer == NULL) { + // todo failed to create timer + taosMemoryFree(pInfo); + } else { + atomic_add_fetch_8(&pTask->status.timerActive, 1);// timer is active + qDebug("s-task:%s set timer active flag", pTask->id.idStr); + } + } else { // timer exists + ASSERT(pTask->status.timerActive > 0); + qDebug("s-task:%s set timer active flag, task timer not null", pTask->id.idStr); + taosTmrReset(tryLaunchHistoryTask, 100, pInfo, streamEnv.timer, &pTask->launchTaskTimer); + } + + // try again in 500ms + return TSDB_CODE_SUCCESS; + } + + doCheckDownstreamStatus(pTask, *pHTask); + return TSDB_CODE_SUCCESS; +} + +int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { + return 0; + } + + // restore param + int32_t code = 0; + if (pTask->info.fillHistory) { + code = streamRestoreParam(pTask); + if (code < 0) { + return -1; + } + } + + // dispatch scan-history finish req to all related downstream task + code = streamDispatchScanHistoryFinishMsg(pTask); + if (code < 0) { + return -1; + } + + return 0; +} + +int32_t streamTaskFillHistoryFinished(SStreamTask* pTask) { + void* exec = pTask->exec.pExecutor; + return qStreamInfoResetTimewindowFilter(exec); +} + +bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer) { + SVersionRange* pRange = &pTask->dataRange.range; + ASSERT(latestVer >= pRange->maxVer); + + int64_t nextStartVer = pRange->maxVer + 1; + if (nextStartVer > latestVer - 1) { + // no input data yet. no need to execute the secondardy scan while stream task halt + streamTaskFillHistoryFinished(pTask); + qDebug( + "s-task:%s no need to perform secondary scan-history data(step 2), since no data ingest during step1 scan, " + "related stream task currentVer:%" PRId64, + pTask->id.idStr, latestVer); + return true; + } else { + // 2. do secondary scan of the history data, the time window remain, and the version range is updated to + // [pTask->dataRange.range.maxVer, ver1] + pRange->minVer = nextStartVer; + pRange->maxVer = latestVer - 1; + return false; + } +} + +int32_t tEncodeStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->reqId) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; @@ -298,7 +696,7 @@ int32_t tEncodeSStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq return pEncoder->pos; } -int32_t tDecodeSStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq) { +int32_t tDecodeStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->reqId) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; @@ -311,7 +709,7 @@ int32_t tDecodeSStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq return 0; } -int32_t tEncodeSStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp) { +int32_t tEncodeStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pRsp->reqId) < 0) return -1; if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; @@ -325,7 +723,7 @@ int32_t tEncodeSStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp return pEncoder->pos; } -int32_t tDecodeSStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) { +int32_t tDecodeStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pRsp->reqId) < 0) return -1; if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1; @@ -339,19 +737,210 @@ int32_t tDecodeSStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp return 0; } -int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq) { +int32_t tEncodeStreamScanHistoryFinishReq(SEncoder* pEncoder, const SStreamScanHistoryFinishReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->downstreamTaskId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->childId) < 0) return -1; tEndEncode(pEncoder); return pEncoder->pos; } -int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishReq* pReq) { + +int32_t tDecodeStreamScanHistoryFinishReq(SDecoder* pDecoder, SStreamScanHistoryFinishReq* pReq) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->downstreamTaskId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->childId) < 0) return -1; tEndDecode(pDecoder); return 0; } + +void streamTaskSetRangeStreamCalc(SStreamTask* pTask) { + if (pTask->historyTaskId.taskId == 0) { + SHistDataRange* pRange = &pTask->dataRange; + if (pTask->info.fillHistory == 1) { + qDebug("s-task:%s fill-history task, time window:%" PRId64 "-%" PRId64 ", verRange:%" PRId64 + "-%" PRId64, + pTask->id.idStr, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, pRange->range.maxVer); + } else { + qDebug("s-task:%s no related fill-history task, stream time window:%" PRId64 "-%" PRId64 ", verRange:%" PRId64 + "-%" PRId64, + pTask->id.idStr, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, pRange->range.maxVer); + } + } else { + SHistDataRange* pRange = &pTask->dataRange; + + int64_t ekey = 0; + if (pRange->window.ekey < INT64_MAX) { + ekey = pRange->window.ekey + 1; + } else { + ekey = pRange->window.ekey; + } + + int64_t ver = pRange->range.minVer; + + pRange->window.skey = ekey; + pRange->window.ekey = INT64_MAX; + pRange->range.minVer = 0; + pRange->range.maxVer = ver; + + qDebug("s-task:%s level:%d related fill-history task exists, update stream calc time window:%" PRId64 " - %" PRId64 + ", verRang:%" PRId64 " - %" PRId64, + pTask->id.idStr, pTask->info.taskLevel, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, + pRange->range.maxVer); + } +} + +void launchFillHistoryTask(SStreamTask* pTask) { + int32_t tId = pTask->historyTaskId.taskId; + if (tId == 0) { + return; + } + + ASSERT(pTask->status.downstreamReady == 1); + qDebug("s-task:%s start to launch related fill-history task:0x%" PRIx64 "-0x%x", pTask->id.idStr, + pTask->historyTaskId.streamId, tId); + + // launch associated fill history task + streamLaunchFillHistoryTask(pTask); +} + +void streamTaskCheckDownstreamTasks(SStreamTask* pTask) { + if (pTask->info.fillHistory) { + qDebug("s-task:%s fill history task, wait for being launched", pTask->id.idStr); + return; + } + + ASSERT(pTask->status.downstreamReady == 0); + + // check downstream tasks for itself + streamTaskDoCheckDownstreamTasks(pTask); +} + +// normal -> pause, pause/stop/dropping -> pause, halt -> pause, scan-history -> pause +void streamTaskPause(SStreamTask* pTask) { + SStreamMeta* pMeta = pTask->pMeta; + + int64_t st = taosGetTimestampMs(); + + int8_t status = pTask->status.taskStatus; + if (status == TASK_STATUS__DROPPING) { + qDebug("vgId:%d s-task:%s task already dropped, do nothing", pMeta->vgId, pTask->id.idStr); + return; + } + + const char* str = streamGetTaskStatusStr(status); + if (status == TASK_STATUS__STOP || status == TASK_STATUS__PAUSE) { + qDebug("vgId:%d s-task:%s task already stopped/paused, status:%s, do nothing", pMeta->vgId, pTask->id.idStr, str); + return; + } + + while (!pTask->status.pauseAllowed || (pTask->status.taskStatus == TASK_STATUS__HALT)) { + status = pTask->status.taskStatus; + if (status == TASK_STATUS__DROPPING) { + qDebug("vgId:%d s-task:%s task already dropped, do nothing", pMeta->vgId, pTask->id.idStr); + return; + } + + if (status == TASK_STATUS__STOP || status == TASK_STATUS__PAUSE) { + qDebug("vgId:%d s-task:%s task already stopped/paused, status:%s, do nothing", pMeta->vgId, pTask->id.idStr, str); + return; + } + + const char* pStatus = streamGetTaskStatusStr(status); + qDebug("s-task:%s wait for the task can be paused, status:%s, vgId:%d", pTask->id.idStr, pStatus, pMeta->vgId); + taosMsleep(100); + } + + // todo: use the task lock, stead of meta lock + taosWLockLatch(&pMeta->lock); + + status = pTask->status.taskStatus; + if (status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP) { + taosWUnLockLatch(&pMeta->lock); + qDebug("vgId:%d s-task:%s task already dropped/stopped/paused, do nothing", pMeta->vgId, pTask->id.idStr); + return; + } + + atomic_store_8(&pTask->status.keepTaskStatus, pTask->status.taskStatus); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__PAUSE); + taosWUnLockLatch(&pMeta->lock); + + // in case of fill-history task, stop the tsdb file scan operation. + if (pTask->info.fillHistory == 1) { + void* pExecutor = pTask->exec.pExecutor; + qKillTask(pExecutor, TSDB_CODE_SUCCESS); + } + + int64_t el = taosGetTimestampMs() - st; + qDebug("vgId:%d s-task:%s set pause flag, prev:%s, elapsed time:%dms", pMeta->vgId, pTask->id.idStr, + streamGetTaskStatusStr(pTask->status.keepTaskStatus), (int32_t)el); +} + +void streamTaskResume(SStreamTask* pTask) { + int8_t status = pTask->status.taskStatus; + if (status == TASK_STATUS__PAUSE) { + pTask->status.taskStatus = pTask->status.keepTaskStatus; + pTask->status.keepTaskStatus = TASK_STATUS__NORMAL; + qDebug("s-task:%s resume from pause", pTask->id.idStr); + } else { + qError("s-task:%s not in pause, failed to resume, status:%s", pTask->id.idStr, streamGetTaskStatusStr(status)); + } +} + +// todo fix race condition +void streamTaskDisablePause(SStreamTask* pTask) { + // pre-condition check + const char* id = pTask->id.idStr; + while (pTask->status.taskStatus == TASK_STATUS__PAUSE) { + qDebug("s-task:%s already in pause, wait for pause being cancelled, and set pause disabled, recheck in 100ms", id); + taosMsleep(100); + } + + qDebug("s-task:%s disable task pause", id); + pTask->status.pauseAllowed = 0; +} + +void streamTaskEnablePause(SStreamTask* pTask) { + qDebug("s-task:%s enable task pause", pTask->id.idStr); + pTask->status.pauseAllowed = 1; +} + +void streamTaskHalt(SStreamTask* pTask) { + int8_t status = pTask->status.taskStatus; + if (status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP) { + return; + } + + if (status == TASK_STATUS__HALT) { + return; + } + + // upgrade to halt status + if (status == TASK_STATUS__PAUSE) { + qDebug("s-task:%s upgrade status to %s from %s", pTask->id.idStr, streamGetTaskStatusStr(TASK_STATUS__HALT), + streamGetTaskStatusStr(TASK_STATUS__PAUSE)); + } else { + qDebug("s-task:%s halt task", pTask->id.idStr); + } + + pTask->status.keepTaskStatus = status; + pTask->status.taskStatus = TASK_STATUS__HALT; +} + +void streamTaskResumeFromHalt(SStreamTask* pTask) { + const char* id = pTask->id.idStr; + int8_t status = pTask->status.taskStatus; + if (status != TASK_STATUS__HALT) { + qError("s-task:%s not in halt status, status:%s", id, streamGetTaskStatusStr(status)); + return; + } + + pTask->status.taskStatus = pTask->status.keepTaskStatus; + pTask->status.keepTaskStatus = TASK_STATUS__NORMAL; + qDebug("s-task:%s resume from halt, current status:%s", id, streamGetTaskStatusStr(pTask->status.taskStatus)); +} diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 373cb27941f56abf55f8bcfb0486a67225703d3b..5b42be182c0a0abc24d587e0a986b9129fc6c416 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -18,14 +18,13 @@ #include "osMemory.h" #include "rocksdb/c.h" #include "streamBackendRocksdb.h" -#include "streamInc.h" +#include "streamInt.h" #include "tcoding.h" #include "tcommon.h" #include "tcompare.h" #include "tref.h" -#include "ttimer.h" -#define MAX_TABLE_NAME_NUM 2000000 +#define MAX_TABLE_NAME_NUM 200000 int sessionRangeKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2) { if (pWin1->groupId > pWin2->groupId) { @@ -91,13 +90,14 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { return winKeyCmprImpl(&pWin1->key, &pWin2->key); } -SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages) { - qWarn("open stream state, %s", path); +SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages) { + qDebug("open stream state, %s", path); SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState)); if (pState == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pState->pTdbState = taosMemoryCalloc(1, sizeof(STdbState)); if (pState->pTdbState == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -105,29 +105,50 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int return NULL; } - char statePath[1024]; + SStreamTask* pStreamTask = pTask; + char statePath[1024]; if (!specPath) { - sprintf(statePath, "%s/%d", path, pTask->id.taskId); + sprintf(statePath, "%s/%d", path, pStreamTask->id.taskId); } else { memset(statePath, 0, 1024); tstrncpy(statePath, path, 1024); } - pState->taskId = pTask->id.taskId; - pState->streamId = pTask->id.streamId; + + pState->taskId = pStreamTask->id.taskId; + pState->streamId = pStreamTask->id.streamId; + sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-%d", pState->streamId, pState->taskId); + #ifdef USE_ROCKSDB - // qWarn("open stream state1"); - taosAcquireRef(pTask->pMeta->streamBackendId, pTask->pMeta->streamBackendRid); - int code = streamStateOpenBackend(pTask->pMeta->streamBackend, pState); - if (code == -1) { - taosReleaseRef(pTask->pMeta->streamBackendId, pTask->pMeta->streamBackendRid); - taosMemoryFree(pState); - pState = NULL; + SStreamMeta* pMeta = pStreamTask->pMeta; + pState->streamBackendRid = pMeta->streamBackendRid; + // taosWLockLatch(&pMeta->lock); + taosThreadMutexLock(&pMeta->backendMutex); + void* uniqueId = + taosHashGet(pMeta->pTaskBackendUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); + if (uniqueId == NULL) { + int code = streamStateOpenBackend(pMeta->streamBackend, pState); + if (code == -1) { + taosReleaseRef(streamBackendId, pState->streamBackendRid); + taosThreadMutexUnlock(&pMeta->backendMutex); + taosMemoryFree(pState); + return NULL; + } + taosHashPut(pMeta->pTaskBackendUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1, + &pState->pTdbState->backendCfWrapperId, sizeof(pState->pTdbState->backendCfWrapperId)); + } else { + int64_t id = *(int64_t*)uniqueId; + pState->pTdbState->backendCfWrapperId = id; + pState->pTdbState->pBackendCfWrapper = taosAcquireRef(streamBackendCfWrapperId, id); + + taosAcquireRef(streamBackendId, pState->streamBackendRid); } + taosThreadMutexUnlock(&pMeta->backendMutex); + pState->pTdbState->pOwner = pTask; pState->pFileState = NULL; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT); - pState->parNameMap = tSimpleHashInit(1024, hashFn); + pState->parNameMap = tSimpleHashInit(1024, hashFn); return pState; #else @@ -218,9 +239,7 @@ _err: void streamStateClose(SStreamState* pState, bool remove) { SStreamTask* pTask = pState->pTdbState->pOwner; #ifdef USE_ROCKSDB - // streamStateCloseBackend(pState); streamStateDestroy(pState, remove); - taosReleaseRef(pTask->pMeta->streamBackendId, pTask->pMeta->streamBackendRid); #else tdbCommit(pState->pTdbState->db, pState->pTdbState->txn); tdbPostCommit(pState->pTdbState->db, pState->pTdbState->txn); @@ -272,26 +291,30 @@ int32_t streamStateCommit(SStreamState* pState) { #endif } -int32_t streamStateFuncPut(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen) { +int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { #ifdef USE_ROCKSDB - return streamStateFuncPut_rocksdb(pState, key, value, vLen); + void* pVal = NULL; + int32_t len = 0; + int32_t code = getRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), &pVal, &len); + char* buf = ((SRowBuffPos*)pVal)->pRowBuff; + uint32_t rowSize = streamFileStateGeSelectRowSize(pState->pFileState); + memcpy(buf + len - rowSize, value, vLen); + return code; #else return tdbTbUpsert(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), value, vLen, pState->pTdbState->txn); #endif } -int32_t streamStateFuncGet(SStreamState* pState, const STupleKey* key, void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB - return streamStateFuncGet_rocksdb(pState, key, pVal, pVLen); -#else - return tdbTbGet(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), pVal, pVLen); -#endif -} - -int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key) { +int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen) { #ifdef USE_ROCKSDB - return streamStateFuncDel_rocksdb(pState, key); + void* pVal = NULL; + int32_t len = 0; + int32_t code = getRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), (void**)(&pVal), &len); + char* buf = ((SRowBuffPos*)pVal)->pRowBuff; + uint32_t rowSize = streamFileStateGeSelectRowSize(pState->pFileState); + *ppVal = buf + len - rowSize; + return code; #else - return tdbTbDelete(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), pState->pTdbState->txn); + return tdbTbGet(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), ppVal, pVLen); #endif } @@ -379,8 +402,8 @@ int32_t streamStateClear(SStreamState* pState) { streamStatePut(pState, &key, NULL, 0); while (1) { SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &key); - SWinKey delKey = {0}; - int32_t code = streamStateGetKVByCur(pCur, &delKey, NULL, 0); + SWinKey delKey = {0}; + int32_t code = streamStateGetKVByCur(pCur, &delKey, NULL, 0); streamStateFreeCur(pCur); if (code == 0) { streamStateDel(pState, &delKey); @@ -445,7 +468,7 @@ int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pV #ifdef USE_ROCKSDB taosMemoryFree(pVal); #else - streamFreeVal(pVal); + streamStateFreeVal(pVal); #endif return 0; } @@ -492,7 +515,7 @@ int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** return -1; } const SStateKey* pKTmp = NULL; - int32_t kLen; + int32_t kLen; if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { return -1; } @@ -512,7 +535,7 @@ int32_t streamStateFillGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const vo return -1; } const SWinKey* pKTmp = NULL; - int32_t kLen; + int32_t kLen; if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { return -1; } @@ -529,7 +552,7 @@ int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const v return -1; } uint64_t groupId = pKey->groupId; - int32_t code = streamStateFillGetKVByCur(pCur, pKey, pVal, pVLen); + int32_t code = streamStateFillGetKVByCur(pCur, pKey, pVal, pVLen); if (code == 0) { if (pKey->groupId == groupId) { return 0; @@ -547,7 +570,7 @@ int32_t streamStateGetFirst(SStreamState* pState, SWinKey* key) { SWinKey tmp = {.ts = 0, .groupId = 0}; streamStatePut(pState, &tmp, NULL, 0); SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &tmp); - int32_t code = streamStateGetKVByCur(pCur, key, NULL, 0); + int32_t code = streamStateGetKVByCur(pCur, key, NULL, 0); streamStateFreeCur(pCur); streamStateDel(pState, &tmp); return code; @@ -587,7 +610,7 @@ SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key } SStateKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateKey), &c) < 0) { streamStateFreeCur(pCur); return NULL; @@ -696,7 +719,7 @@ void streamStateFreeCur(SStreamStateCur* pCur) { taosMemoryFree(pCur); } -void streamFreeVal(void* val) { +void streamStateFreeVal(void* val) { #ifdef USE_ROCKSDB taosMemoryFree(val); #else @@ -706,6 +729,7 @@ void streamFreeVal(void* val) { int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) { #ifdef USE_ROCKSDB + qDebug("===stream===save skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64, key->win.skey,key->win.ekey, key->groupId); return streamStateSessionPut_rocksdb(pState, key, value, vLen); #else SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; @@ -720,9 +744,9 @@ int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVa #else SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pState, key); - SSessionKey resKey = *key; - void* tmp = NULL; - int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, &tmp, pVLen); + SSessionKey resKey = *key; + void* tmp = NULL; + int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, &tmp, pVLen); if (code == 0) { if (key->win.skey != resKey.win.skey) { code = -1; @@ -739,6 +763,7 @@ int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVa int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { #ifdef USE_ROCKSDB + qDebug("===stream===delete skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64, key->win.skey,key->win.ekey, key->groupId); return streamStateSessionDel_rocksdb(pState, key); #else SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; @@ -761,7 +786,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, cons } SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { streamStateFreeCur(pCur); return NULL; @@ -792,7 +817,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, cons } SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { streamStateFreeCur(pCur); return NULL; @@ -824,7 +849,7 @@ SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSess } SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { streamStateFreeCur(pCur); return NULL; @@ -848,7 +873,7 @@ int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, v return -1; } SStateSessionKey* pKTmp = NULL; - int32_t kLen; + int32_t kLen; if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, (const void**)pVal, pVLen) < 0) { return -1; } @@ -867,13 +892,13 @@ int32_t streamStateSessionClear(SStreamState* pState) { #ifdef USE_ROCKSDB return streamStateSessionClear_rocksdb(pState); #else - SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; + SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pState, &key); while (1) { SSessionKey delKey = {0}; - void* buf = NULL; - int32_t size = 0; - int32_t code = streamStateSessionGetKVByCur(pCur, &delKey, &buf, &size); + void* buf = NULL; + int32_t size = 0; + int32_t code = streamStateSessionGetKVByCur(pCur, &delKey, &buf, &size); if (code == 0 && size > 0) { memset(buf, 0, size); streamStateSessionPut(pState, &delKey, buf, size); @@ -902,14 +927,14 @@ int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* } SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { streamStateFreeCur(pCur); return -1; } SSessionKey resKey = *key; - int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); + int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { *curKey = resKey; streamStateFreeCur(pCur); @@ -945,19 +970,19 @@ int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, return streamStateSessionAddIfNotExist_rocksdb(pState, key, gap, pVal, pVLen); #else // todo refactor - int32_t res = 0; + int32_t res = 0; SSessionKey originKey = *key; SSessionKey searchKey = *key; searchKey.win.skey = key->win.skey - gap; searchKey.win.ekey = key->win.ekey + gap; int32_t valSize = *pVLen; - void* tmp = tdbRealloc(NULL, valSize); + void* tmp = tdbRealloc(NULL, valSize); if (!tmp) { return -1; } SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pState, key); - int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); + int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); if (code == 0) { if (sessionRangeKeyCmpr(&searchKey, key) == 0) { memcpy(tmp, *pVal, valSize); @@ -1000,16 +1025,16 @@ int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, ch #ifdef USE_ROCKSDB return streamStateStateAddIfNotExist_rocksdb(pState, key, pKeyData, keyDataLen, fn, pVal, pVLen); #else - int32_t res = 0; + int32_t res = 0; SSessionKey tmpKey = *key; - int32_t valSize = *pVLen; - void* tmp = tdbRealloc(NULL, valSize); + int32_t valSize = *pVLen; + void* tmp = tdbRealloc(NULL, valSize); if (!tmp) { return -1; } SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pState, key); - int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); + int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); if (code == 0) { if (key->win.skey <= tmpKey.win.skey && tmpKey.win.ekey <= key->win.ekey) { memcpy(tmp, *pVal, valSize); @@ -1054,7 +1079,7 @@ _end: } int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) { - qWarn("try to write to cf parname"); + qDebug("try to write to cf parname"); #ifdef USE_ROCKSDB if (tSimpleHashGetSize(pState->parNameMap) > MAX_TABLE_NAME_NUM) { if (tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t)) == NULL) { @@ -1107,6 +1132,8 @@ int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark) { #endif } +void streamStateReloadInfo(SStreamState* pState, TSKEY ts) { streamFileStateReloadInfo(pState->pFileState, ts); } + #if 0 char* streamStateSessionDump(SStreamState* pState) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 8a038969785b9a4f2a64436cca1953d95805d128..122d18e9f080a9bae822c6d2936714f5a8414fc0 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -13,27 +13,43 @@ * along with this program. If not, see . */ +#include "streamInt.h" #include "executor.h" #include "tstream.h" #include "wal.h" +#include "ttimer.h" -SStreamTask* tNewStreamTask(int64_t streamId) { +static int32_t addToTaskset(SArray* pArray, SStreamTask* pTask) { + int32_t childId = taosArrayGetSize(pArray); + pTask->info.selfChildId = childId; + taosArrayPush(pArray, &pTask); + return 0; +} + +SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory, int64_t triggerParam, + SArray* pTaskList) { SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - + pTask->ver = SSTREAM_TASK_VER; pTask->id.taskId = tGenIdPI32(); pTask->id.streamId = streamId; + pTask->info.taskLevel = taskLevel; + pTask->info.fillHistory = fillHistory; + pTask->triggerParam = triggerParam; char buf[128] = {0}; sprintf(buf, "0x%" PRIx64 "-%d", pTask->id.streamId, pTask->id.taskId); pTask->id.idStr = taosStrdup(buf); pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; + pTask->status.taskStatus = TASK_STATUS__SCAN_HISTORY; pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; - pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; + pTask->outputInfo.status = TASK_OUTPUT_STATUS__NORMAL; + addToTaskset(pTaskList, pTask); return pTask; } @@ -57,48 +73,59 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) { int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pTask->ver) < 0) return -1; if (tEncodeI64(pEncoder, pTask->id.streamId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->id.taskId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->totalLevel) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->taskLevel) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->outputType) < 0) return -1; - if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->info.totalLevel) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->info.taskLevel) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->outputInfo.type) < 0) return -1; + if (tEncodeI16(pEncoder, pTask->msgInfo.msgType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->status.taskStatus) < 0) return -1; if (tEncodeI8(pEncoder, pTask->status.schedStatus) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; - if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->info.selfChildId) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->info.nodeId) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pTask->info.epSet) < 0) return -1; if (tEncodeI64(pEncoder, pTask->chkInfo.id) < 0) return -1; if (tEncodeI64(pEncoder, pTask->chkInfo.version) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->fillHistory) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->info.fillHistory) < 0) return -1; - int32_t epSz = taosArrayGetSize(pTask->childEpInfo); + if (tEncodeI64(pEncoder, pTask->historyTaskId.streamId)) return -1; + if (tEncodeI32(pEncoder, pTask->historyTaskId.taskId)) return -1; + if (tEncodeI64(pEncoder, pTask->streamTaskId.streamId)) return -1; + if (tEncodeI32(pEncoder, pTask->streamTaskId.taskId)) return -1; + + if (tEncodeU64(pEncoder, pTask->dataRange.range.minVer)) return -1; + if (tEncodeU64(pEncoder, pTask->dataRange.range.maxVer)) return -1; + if (tEncodeI64(pEncoder, pTask->dataRange.window.skey)) return -1; + if (tEncodeI64(pEncoder, pTask->dataRange.window.ekey)) return -1; + + int32_t epSz = taosArrayGetSize(pTask->pUpstreamEpInfoList); if (tEncodeI32(pEncoder, epSz) < 0) return -1; for (int32_t i = 0; i < epSz; i++) { - SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->childEpInfo, i); + SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->pUpstreamEpInfoList, i); if (tEncodeStreamEpInfo(pEncoder, pInfo) < 0) return -1; } - if (pTask->taskLevel != TASK_LEVEL__SINK) { + if (pTask->info.taskLevel != TASK_LEVEL__SINK) { if (tEncodeCStr(pEncoder, pTask->exec.qmsg) < 0) return -1; } - if (pTask->outputType == TASK_OUTPUT__TABLE) { + if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { if (tEncodeI64(pEncoder, pTask->tbSink.stbUid) < 0) return -1; if (tEncodeCStr(pEncoder, pTask->tbSink.stbFullName) < 0) return -1; if (tEncodeSSchemaWrapper(pEncoder, pTask->tbSink.pSchemaWrapper) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__SMA) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { if (tEncodeI64(pEncoder, pTask->smaSink.smaId) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__FETCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) { if (tEncodeI8(pEncoder, pTask->fetchSink.reserved) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { if (tSerializeSUseDbRspImp(pEncoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1; if (tEncodeCStr(pEncoder, pTask->shuffleDispatcher.stbFullName) < 0) return -1; } @@ -110,27 +137,41 @@ int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pTask->ver) < 0) return -1; + if (pTask->ver != SSTREAM_TASK_VER) return -1; + if (tDecodeI64(pDecoder, &pTask->id.streamId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->id.taskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->totalLevel) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->taskLevel) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->outputType) < 0) return -1; - if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->info.totalLevel) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->info.taskLevel) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->outputInfo.type) < 0) return -1; + if (tDecodeI16(pDecoder, &pTask->msgInfo.msgType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->status.taskStatus) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->status.schedStatus) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; - if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->info.selfChildId) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->info.nodeId) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &pTask->info.epSet) < 0) return -1; if (tDecodeI64(pDecoder, &pTask->chkInfo.id) < 0) return -1; if (tDecodeI64(pDecoder, &pTask->chkInfo.version) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->fillHistory) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->info.fillHistory) < 0) return -1; - int32_t epSz; + if (tDecodeI64(pDecoder, &pTask->historyTaskId.streamId)) return -1; + if (tDecodeI32(pDecoder, &pTask->historyTaskId.taskId)) return -1; + if (tDecodeI64(pDecoder, &pTask->streamTaskId.streamId)) return -1; + if (tDecodeI32(pDecoder, &pTask->streamTaskId.taskId)) return -1; + + if (tDecodeU64(pDecoder, &pTask->dataRange.range.minVer)) return -1; + if (tDecodeU64(pDecoder, &pTask->dataRange.range.maxVer)) return -1; + if (tDecodeI64(pDecoder, &pTask->dataRange.window.skey)) return -1; + if (tDecodeI64(pDecoder, &pTask->dataRange.window.ekey)) return -1; + + int32_t epSz = -1; if (tDecodeI32(pDecoder, &epSz) < 0) return -1; - pTask->childEpInfo = taosArrayInit(epSz, sizeof(void*)); + + pTask->pUpstreamEpInfoList = taosArrayInit(epSz, POINTER_BYTES); for (int32_t i = 0; i < epSz; i++) { SStreamChildEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamChildEpInfo)); if (pInfo == NULL) return -1; @@ -138,28 +179,28 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { taosMemoryFreeClear(pInfo); return -1; } - taosArrayPush(pTask->childEpInfo, &pInfo); + taosArrayPush(pTask->pUpstreamEpInfoList, &pInfo); } - if (pTask->taskLevel != TASK_LEVEL__SINK) { + if (pTask->info.taskLevel != TASK_LEVEL__SINK) { if (tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg) < 0) return -1; } - if (pTask->outputType == TASK_OUTPUT__TABLE) { + if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { if (tDecodeI64(pDecoder, &pTask->tbSink.stbUid) < 0) return -1; if (tDecodeCStrTo(pDecoder, pTask->tbSink.stbFullName) < 0) return -1; pTask->tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); if (pTask->tbSink.pSchemaWrapper == NULL) return -1; if (tDecodeSSchemaWrapper(pDecoder, pTask->tbSink.pSchemaWrapper) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__SMA) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { if (tDecodeI64(pDecoder, &pTask->smaSink.smaId) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__FETCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) { if (tDecodeI8(pDecoder, &pTask->fetchSink.reserved) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { if (tDeserializeSUseDbRspImp(pDecoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1; if (tDecodeCStrTo(pDecoder, pTask->shuffleDispatcher.stbFullName) < 0) return -1; } @@ -169,15 +210,39 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { return 0; } +static void freeItem(void* p) { + SStreamContinueExecInfo* pInfo = p; + rpcFreeCont(pInfo->msg.pCont); +} + void tFreeStreamTask(SStreamTask* pTask) { - qDebug("free s-task:%s", pTask->id.idStr); + qDebug("free s-task:0x%x, %p", pTask->id.taskId, pTask); + + // remove the ref by timer + while(pTask->status.timerActive > 0) { + qDebug("s-task:%s wait for task stop timer activities", pTask->id.idStr); + taosMsleep(10); + } + + if (pTask->schedTimer != NULL) { + taosTmrStop(pTask->schedTimer); + pTask->schedTimer = NULL; + } + + if (pTask->launchTaskTimer != NULL) { + taosTmrStop(pTask->launchTaskTimer); + pTask->launchTaskTimer = NULL; + } + int32_t status = atomic_load_8((int8_t*)&(pTask->status.taskStatus)); if (pTask->inputQueue) { streamQueueClose(pTask->inputQueue); } - if (pTask->outputQueue) { - streamQueueClose(pTask->outputQueue); + + if (pTask->outputInfo.queue) { + streamQueueClose(pTask->outputInfo.queue); } + if (pTask->exec.qmsg) { taosMemoryFree(pTask->exec.qmsg); } @@ -191,14 +256,12 @@ void tFreeStreamTask(SStreamTask* pTask) { walCloseReader(pTask->exec.pWalReader); } - taosArrayDestroyP(pTask->childEpInfo, taosMemoryFree); - if (pTask->outputType == TASK_OUTPUT__TABLE) { + taosArrayDestroyP(pTask->pUpstreamEpInfoList, taosMemoryFree); + if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { tDeleteSchemaWrapper(pTask->tbSink.pSchemaWrapper); taosMemoryFree(pTask->tbSink.pTSchema); tSimpleHashCleanup(pTask->tbSink.pTblInfo); - } - - if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { taosArrayDestroy(pTask->shuffleDispatcher.dbInfo.pVgroupInfos); taosArrayDestroy(pTask->checkReqIds); pTask->checkReqIds = NULL; @@ -212,5 +275,15 @@ void tFreeStreamTask(SStreamTask* pTask) { taosMemoryFree((void*)pTask->id.idStr); } + if (pTask->pNameMap) { + tSimpleHashCleanup(pTask->pNameMap); + } + + if (pTask->pRspMsgList != NULL) { + taosArrayDestroyEx(pTask->pRspMsgList, freeItem); + pTask->pRspMsgList = NULL; + } + + taosThreadMutexDestroy(&pTask->lock); taosMemoryFree(pTask); } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index fff666ec9f3f90bd60d921a011fbd8925c52696f..7a8de91d7735fb7b43a8fb65e747ff5aa7737723 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -20,9 +20,9 @@ #include "ttime.h" #define DEFAULT_FALSE_POSITIVE 0.01 -#define DEFAULT_BUCKET_SIZE 1310720 -#define DEFAULT_MAP_CAPACITY 1310720 -#define DEFAULT_MAP_SIZE (DEFAULT_MAP_CAPACITY * 10) +#define DEFAULT_BUCKET_SIZE 131072 +#define DEFAULT_MAP_CAPACITY 131072 +#define DEFAULT_MAP_SIZE (DEFAULT_MAP_CAPACITY * 100) #define ROWS_PER_MILLISECOND 1 #define MAX_NUM_SCALABLE_BF 100000 #define MIN_NUM_SCALABLE_BF 10 @@ -33,7 +33,7 @@ static int64_t adjustExpEntries(int64_t entries) { return TMIN(DEFAULT_EXPECTED_ENTRIES, entries); } -static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { +void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { if (pInfo->numSBFs < count) { count = pInfo->numSBFs; } @@ -44,12 +44,12 @@ static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { } } -static void clearItemHelper(void* p) { - SScalableBf** pBf = p; +static void clearItemHelper(void *p) { + SScalableBf **pBf = p; tScalableBfDestroy(*pBf); } -static void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) { +void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) { if (count < pInfo->numSBFs) { for (uint64_t i = 0; i < count; ++i) { SScalableBf *pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0); @@ -274,7 +274,7 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { } int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) { - if(!pInfo) { + if (!pInfo) { return 0; } diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 67835e77b8847d45d1015923bdc6bc3b035fdf5d..dd857141c1288da621b8ed2b58af8373d28fbe31 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -16,6 +16,7 @@ #include "tstreamFileState.h" #include "query.h" +#include "storageapi.h" #include "streamBackendRocksdb.h" #include "taos.h" #include "tcommon.h" @@ -32,6 +33,7 @@ struct SStreamFileState { SSHashObj* rowBuffMap; void* pFileStore; int32_t rowSize; + int32_t selectivityRowSize; int32_t keyLen; uint64_t preCheckPointVersion; uint64_t checkPointVersion; @@ -41,12 +43,13 @@ struct SStreamFileState { uint64_t maxRowCount; uint64_t curRowCount; GetTsFun getTs; + char* id; }; typedef SRowBuffPos SRowBuffInfo; -SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, GetTsFun fp, void* pFile, - TSKEY delMark) { +SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, + GetTsFun fp, void* pFile, TSKEY delMark, const char* idstr) { if (memSize <= 0) { memSize = DEFAULT_MAX_STREAM_BUFFER_SIZE; } @@ -58,6 +61,7 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ if (!pFileState) { goto _error; } + rowSize += selectRowSize; pFileState->maxRowCount = TMAX((uint64_t)memSize / rowSize, FLUSH_NUM * 2); pFileState->usedBuffs = tdListNew(POINTER_BYTES); pFileState->freeBuffs = tdListNew(POINTER_BYTES); @@ -67,17 +71,20 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ if (!pFileState->usedBuffs || !pFileState->freeBuffs || !pFileState->rowBuffMap) { goto _error; } + pFileState->keyLen = keySize; pFileState->rowSize = rowSize; + pFileState->selectivityRowSize = selectRowSize; pFileState->preCheckPointVersion = 0; pFileState->checkPointVersion = 1; pFileState->pFileStore = pFile; pFileState->getTs = fp; - pFileState->maxRowCount = TMAX((uint64_t)memSize / rowSize, FLUSH_NUM * 2); pFileState->curRowCount = 0; pFileState->deleteMark = delMark; pFileState->flushMark = INT64_MIN; pFileState->maxTs = INT64_MIN; + pFileState->id = taosStrdup(idstr); + recoverSnapshot(pFileState); return pFileState; @@ -121,6 +128,8 @@ void streamFileStateDestroy(SStreamFileState* pFileState) { if (!pFileState) { return; } + + taosMemoryFree(pFileState->id); tdListFreeP(pFileState->usedBuffs, destroyRowBuffAllPosPtr); tdListFreeP(pFileState->freeBuffs, destroyRowBuff); tSimpleHashCleanup(pFileState->rowBuffMap); @@ -134,7 +143,7 @@ void clearExpiredRowBuff(SStreamFileState* pFileState, TSKEY ts, bool all) { SListNode* pNode = NULL; while ((pNode = tdListNext(&iter)) != NULL) { SRowBuffPos* pPos = *(SRowBuffPos**)(pNode->data); - if (all || (pFileState->getTs(pPos->pKey) < ts)) { + if (all || (pFileState->getTs(pPos->pKey) < ts && !pPos->beUsed)) { ASSERT(pPos->pRowBuff != NULL); tdListAppend(pFileState->freeBuffs, &(pPos->pRowBuff)); pPos->pRowBuff = NULL; @@ -174,7 +183,8 @@ void popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uin i++; } } - qInfo("do stream state flush %d rows to disck. is used: %d", listNEles(pFlushList), used); + + qInfo("stream state flush %d rows to disk. is used:%d", listNEles(pFlushList), used); } int32_t flushRowBuff(SStreamFileState* pFileState) { @@ -182,13 +192,17 @@ int32_t flushRowBuff(SStreamFileState* pFileState) { if (!pFlushList) { return TSDB_CODE_OUT_OF_MEMORY; } + uint64_t num = (uint64_t)(pFileState->curRowCount * FLUSH_RATIO); num = TMAX(num, FLUSH_NUM); popUsedBuffs(pFileState, pFlushList, num, false); + if (isListEmpty(pFlushList)) { popUsedBuffs(pFileState, pFlushList, num, true); } + flushSnapshot(pFileState, pFlushList, false); + SListIter fIter = {0}; tdListInitIter(pFlushList, &fIter, TD_LIST_FORWARD); SListNode* pNode = NULL; @@ -198,6 +212,7 @@ int32_t flushRowBuff(SStreamFileState* pFileState) { tdListAppend(pFileState->freeBuffs, &pPos->pRowBuff); pPos->pRowBuff = NULL; } + tdListFreeP(pFlushList, destroyRowBuffPosPtr); return TSDB_CODE_SUCCESS; } @@ -266,13 +281,13 @@ int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, voi TSKEY ts = pFileState->getTs(pKey); if (ts > pFileState->maxTs - pFileState->deleteMark && ts < pFileState->flushMark) { int32_t len = 0; - void* pVal = NULL; - int32_t code = streamStateGet_rocksdb(pFileState->pFileStore, pKey, &pVal, &len); + void* p = NULL; + int32_t code = streamStateGet_rocksdb(pFileState->pFileStore, pKey, &p, &len); qDebug("===stream===get %" PRId64 " from disc, res %d", ts, code); if (code == TSDB_CODE_SUCCESS) { - memcpy(pNewPos->pRowBuff, pVal, len); + memcpy(pNewPos->pRowBuff, p, len); } - taosMemoryFree(pVal); + taosMemoryFree(p); } tSimpleHashPut(pFileState->rowBuffMap, pKey, keyLen, &pNewPos, POINTER_BYTES); @@ -345,26 +360,45 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, tdListInitIter(pSnapshot, &iter, TD_LIST_FORWARD); const int32_t BATCH_LIMIT = 256; - SListNode* pNode = NULL; + + int64_t st = taosGetTimestampMs(); + int32_t numOfElems = listNEles(pSnapshot); + SListNode* pNode = NULL; + + int idx = streamStateGetCfIdx(pFileState->pFileStore, "state"); + + int32_t len = pFileState->rowSize + sizeof(uint64_t) + sizeof(int32_t) + 1; + char* buf = taosMemoryCalloc(1, len); void* batch = streamStateCreateBatch(); while ((pNode = tdListNext(&iter)) != NULL && code == TSDB_CODE_SUCCESS) { SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data; ASSERT(pPos->pRowBuff && pFileState->rowSize > 0); + if (streamStateGetBatchSize(batch) >= BATCH_LIMIT) { - code = streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); + streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); streamStateClearBatch(batch); } SStateKey sKey = {.key = *((SWinKey*)pPos->pKey), .opNum = ((SStreamState*)pFileState->pFileStore)->number}; - code = streamStatePutBatch(pFileState->pFileStore, "state", batch, &sKey, pPos->pRowBuff, pFileState->rowSize, 0); - qDebug("===stream===put %" PRId64 " to disc, res %d", sKey.key.ts, code); + code = streamStatePutBatchOptimize(pFileState->pFileStore, idx, batch, &sKey, pPos->pRowBuff, pFileState->rowSize, + 0, buf); + // todo handle failure + memset(buf, 0, len); +// qDebug("===stream===put %" PRId64 " to disc, res %d", sKey.key.ts, code); } + taosMemoryFree(buf); + if (streamStateGetBatchSize(batch) > 0) { - code = streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); + streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); } + streamStateClearBatch(batch); + int64_t elapsed = taosGetTimestampMs() - st; + qDebug("%s flush to disk in batch model completed, rows:%d, batch size:%d, elapsed time:%"PRId64"ms", pFileState->id, numOfElems, + BATCH_LIMIT, elapsed); + if (flushState) { const char* taskKey = "streamFileState"; { @@ -373,7 +407,7 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, int32_t len = 0; sprintf(keyBuf, "%s:%" PRId64 "", taskKey, ((SStreamState*)pFileState->pFileStore)->checkPointId); streamFileStateEncode(&pFileState->flushMark, &valBuf, &len); - code = streamStatePutBatch(pFileState->pFileStore, "default", batch, keyBuf, valBuf, len, 0); + streamStatePutBatch(pFileState->pFileStore, "default", batch, keyBuf, valBuf, len, 0); taosMemoryFree(valBuf); } { @@ -386,8 +420,8 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, } streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); } - streamStateDestroyBatch(batch); + streamStateDestroyBatch(batch); return code; } @@ -413,10 +447,13 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { int32_t len = 0; memcpy(buf, taskKey, strlen(taskKey)); code = streamDefaultGet_rocksdb(pFileState->pFileStore, buf, &val, &len); - if (code != 0) { + if (code != 0 || len == 0 || val == NULL) { return TSDB_CODE_FAILED; } - sscanf(val, "%" PRId64 "", &maxCheckPointId); + memcpy(buf, val, len); + buf[len] = 0; + maxCheckPointId = atol((char*)buf); + taosMemoryFree(val); } for (int64_t i = maxCheckPointId; i > 0; i--) { char buf[128] = {0}; @@ -427,13 +464,16 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { if (code != 0) { return TSDB_CODE_FAILED; } + memcpy(buf, val, len); + buf[len] = 0; + taosMemoryFree(val); + TSKEY ts; - sscanf(val, "%" PRId64 "", &ts); + ts = atol((char*)buf); if (ts < mark) { // statekey winkey.ts < mark forceRemoveCheckpoint(pFileState, i); break; - } else { } } return code; @@ -441,9 +481,12 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { int32_t recoverSnapshot(SStreamFileState* pFileState) { int32_t code = TSDB_CODE_SUCCESS; - int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) ? INT64_MIN - : pFileState->maxTs - pFileState->deleteMark; - deleteExpiredCheckPoint(pFileState, mark); + if (pFileState->maxTs != INT64_MIN) { + int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) + ? INT64_MIN + : pFileState->maxTs - pFileState->deleteMark; + deleteExpiredCheckPoint(pFileState, mark); + } void* pStVal = NULL; int32_t len = 0; @@ -468,7 +511,7 @@ int32_t recoverSnapshot(SStreamFileState* pFileState) { break; } memcpy(pNewPos->pRowBuff, pVal, pVLen); - code = tSimpleHashPut(pFileState->rowBuffMap, pNewPos->pKey, pFileState->rowSize, &pNewPos, POINTER_BYTES); + code = tSimpleHashPut(pFileState->rowBuffMap, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { destroyRowBuffPos(pNewPos); break; @@ -478,4 +521,11 @@ int32_t recoverSnapshot(SStreamFileState* pFileState) { streamStateFreeCur(pCur); return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} + +int32_t streamFileStateGeSelectRowSize(SStreamFileState* pFileState) { return pFileState->selectivityRowSize; } + +void streamFileStateReloadInfo(SStreamFileState* pFileState, TSKEY ts) { + pFileState->flushMark = TMAX(pFileState->flushMark, ts); + pFileState->maxTs = TMAX(pFileState->maxTs, ts); +} diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt index a0c171769025f62b0a70a3677d9faeab2a4f6693..629b04ae51435c83b6556a87636929ca732bbec3 100644 --- a/source/libs/stream/test/CMakeLists.txt +++ b/source/libs/stream/test/CMakeLists.txt @@ -8,10 +8,9 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) # bloomFilterTest ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp") -TARGET_LINK_LIBRARIES( - streamUpdateTest - PUBLIC os util common gtest gtest_main stream -) +TARGET_LINK_LIBRARIES(streamUpdateTest + PUBLIC os util common gtest gtest_main stream executor index + ) TARGET_INCLUDE_DIRECTORIES( streamUpdateTest diff --git a/source/libs/stream/test/tstreamUpdateTest.cpp b/source/libs/stream/test/tstreamUpdateTest.cpp index 18c60aff284414e5ba5044d50000a9bd45718965..0e84d6b8bdad0e40e7257a0d60880439d1f3ba37 100644 --- a/source/libs/stream/test/tstreamUpdateTest.cpp +++ b/source/libs/stream/test/tstreamUpdateTest.cpp @@ -158,7 +158,7 @@ TEST(TD_STREAM_UPDATE_TEST, update) { // void *buf = taosMemoryCalloc(1, bufLen); // int32_t resSize = updateInfoSerialize(buf, bufLen, pSU7); - // SUpdateInfo *pSU6 = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); + // SUpdateInfo *pSU6 = taosMemoryCalloc(1, sizeof(SUpdateInfo)); // int32_t desSize = updateInfoDeserialize(buf, bufLen, pSU6); // GTEST_ASSERT_EQ(desSize, 0); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 7d336c83135a20449f09431c2d9427a6dd6cc0c5..f74e43f47f4b3d81ec13a7e99578f7224b05bf5d 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -213,7 +213,7 @@ typedef struct SSyncNode { int64_t minMatchIndex; int64_t startTime; - int64_t leaderTime; + int64_t roleTimeMs; int64_t lastReplicateTime; int32_t electNum; diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index 02790732a23e02e21a5708169780e222711754c2..65e2cc22a07134893f171552b55acf22d8050725 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -79,17 +79,16 @@ static FORCE_INLINE int32_t syncLogReplGetNextRetryBackoff(SSyncLogReplMgr* pMgr SyncTerm syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); -int32_t syncLogReplDoOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplStart(SSyncLogReplMgr* pMgr, SSyncNode* pNode); int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode); int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); - int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode); int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, SRaftId* pDestId, bool* pBarrier); int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplContinue(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); int32_t syncLogReplProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 499df4a98b6f6bc8fac65a916e2163313e020e5e..f1e3c35a49228b9805b12711e8ba6d41f5a44dd9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -37,7 +37,6 @@ #include "syncVoteMgr.h" #include "tglobal.h" #include "tref.h" -#include "syncUtil.h" static void syncNodeEqPingTimer(void* param, void* tmrId); static void syncNodeEqElectTimer(void* param, void* tmrId); @@ -141,10 +140,10 @@ int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) return -1; - if(pSyncNode->raftCfg.lastConfigIndex >= pNewCfg->lastIndex){ + if (pSyncNode->raftCfg.lastConfigIndex >= pNewCfg->lastIndex) { syncNodeRelease(pSyncNode); sInfo("vgId:%d, no need Reconfig, current index:%" PRId64 ", new index:%" PRId64, pSyncNode->vgId, - pSyncNode->raftCfg.lastConfigIndex, pNewCfg->lastIndex); + pSyncNode->raftCfg.lastConfigIndex, pNewCfg->lastIndex); return 0; } @@ -323,8 +322,8 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { } if (pSyncNode->totalReplicaNum > 1) { - if (pSyncNode->state != TAOS_SYNC_STATE_LEADER && pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER - && pSyncNode->state != TAOS_SYNC_STATE_LEARNER) { + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER && pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER && + pSyncNode->state != TAOS_SYNC_STATE_LEARNER) { sNTrace(pSyncNode, "new-snapshot-index:%" PRId64 " candidate or unknown state, do not delete wal", lastApplyIndex); syncNodeRelease(pSyncNode); @@ -509,12 +508,14 @@ SSyncState syncGetState(int64_t rid) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode != NULL) { state.state = pSyncNode->state; + state.roleTimeMs = pSyncNode->roleTimeMs; state.restored = pSyncNode->restoreFinish; if (pSyncNode->vgId != 1) { state.canRead = syncNodeIsReadyForRead(pSyncNode); } else { state.canRead = state.restored; } + state.term = raftStoreGetTerm(pSyncNode); syncNodeRelease(pSyncNode); } @@ -544,7 +545,7 @@ void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) { if (pSyncNode == NULL) return; for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { - if(pSyncNode->raftCfg.cfg.nodeInfo[i].nodeRole == TAOS_SYNC_ROLE_LEARNER) continue; + if (pSyncNode->raftCfg.cfg.nodeInfo[i].nodeRole == TAOS_SYNC_ROLE_LEARNER) continue; SEp* pEp = &pEpSet->eps[i]; tstrncpy(pEp->fqdn, pSyncNode->raftCfg.cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN); pEp->port = (pSyncNode->raftCfg.cfg.nodeInfo)[i].nodePort; @@ -579,21 +580,19 @@ int32_t syncIsCatchUp(int64_t rid) { } int32_t isCatchUp = 0; - if(pSyncNode->pLogBuf->totalIndex < 0 || pSyncNode->pLogBuf->commitIndex < 0 || + if (pSyncNode->pLogBuf->totalIndex < 0 || pSyncNode->pLogBuf->commitIndex < 0 || pSyncNode->pLogBuf->totalIndex < pSyncNode->pLogBuf->commitIndex || - pSyncNode->pLogBuf->totalIndex - pSyncNode->pLogBuf->commitIndex > SYNC_LEARNER_CATCHUP){ - sInfo("vgId:%d, Not catch up, wait one second, totalIndex:%" PRId64 " commitIndex:%" PRId64 " matchIndex:%" PRId64, - pSyncNode->vgId, pSyncNode->pLogBuf->totalIndex, pSyncNode->pLogBuf->commitIndex, - pSyncNode->pLogBuf->matchIndex); + pSyncNode->pLogBuf->totalIndex - pSyncNode->pLogBuf->commitIndex > SYNC_LEARNER_CATCHUP) { + sInfo("vgId:%d, Not catch up, wait one second, totalIndex:%" PRId64 " commitIndex:%" PRId64 " matchIndex:%" PRId64, + pSyncNode->vgId, pSyncNode->pLogBuf->totalIndex, pSyncNode->pLogBuf->commitIndex, + pSyncNode->pLogBuf->matchIndex); isCatchUp = 0; - } - else{ - sInfo("vgId:%d, Catch up, totalIndex:%" PRId64 " commitIndex:%" PRId64 " matchIndex:%" PRId64, - pSyncNode->vgId, pSyncNode->pLogBuf->totalIndex, pSyncNode->pLogBuf->commitIndex, - pSyncNode->pLogBuf->matchIndex); + } else { + sInfo("vgId:%d, Catch up, totalIndex:%" PRId64 " commitIndex:%" PRId64 " matchIndex:%" PRId64, pSyncNode->vgId, + pSyncNode->pLogBuf->totalIndex, pSyncNode->pLogBuf->commitIndex, pSyncNode->pLogBuf->matchIndex); isCatchUp = 1; } - + syncNodeRelease(pSyncNode); return isCatchUp; } @@ -606,7 +605,7 @@ ESyncRole syncGetRole(int64_t rid) { } ESyncRole role = pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex].nodeRole; - + syncNodeRelease(pSyncNode); return role; } @@ -618,8 +617,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_ return -1; } - // not restored, vnode enable - if (!pSyncNode->restoreFinish && pSyncNode->vgId != 1) { + if (!pSyncNode->restoreFinish) { terrno = TSDB_CODE_SYN_PROPOSE_NOT_READY; sNError(pSyncNode, "failed to sync propose since not ready, type:%s, last:%" PRId64 ", cmt:%" PRId64, TMSG_INFO(pMsg->msgType), syncNodeGetLastIndex(pSyncNode), pSyncNode->commitIndex); @@ -802,8 +800,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->vgId = pSyncInfo->vgId; SSyncCfg* pCfg = &pSyncNode->raftCfg.cfg; bool updated = false; - sInfo("vgId:%d, start to open sync node, totalReplicaNum:%d replicaNum:%d selfIndex:%d", - pSyncNode->vgId, pCfg->totalReplicaNum, pCfg->replicaNum, pCfg->myIndex); + sInfo("vgId:%d, start to open sync node, totalReplicaNum:%d replicaNum:%d selfIndex:%d", pSyncNode->vgId, + pCfg->totalReplicaNum, pCfg->replicaNum, pCfg->myIndex); for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SNodeInfo* pNode = &pCfg->nodeInfo[i]; if (tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort)) { @@ -902,6 +900,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init TLA+ server vars pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; + pSyncNode->roleTimeMs = taosGetTimestampMs(); if (raftStoreOpen(pSyncNode) != 0) { sError("vgId:%d, failed to open raft store at path %s", pSyncNode->vgId, pSyncNode->raftStorePath); goto _error; @@ -1039,7 +1038,6 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { int64_t timeNow = taosGetTimestampMs(); pSyncNode->startTime = timeNow; - pSyncNode->leaderTime = timeNow; pSyncNode->lastReplicateTime = timeNow; // snapshotting @@ -1110,10 +1108,9 @@ int32_t syncNodeRestore(SSyncNode* pSyncNode) { int32_t syncNodeStart(SSyncNode* pSyncNode) { // start raft - if(pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex].nodeRole == TAOS_SYNC_ROLE_LEARNER){ + if (pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex].nodeRole == TAOS_SYNC_ROLE_LEARNER) { syncNodeBecomeLearner(pSyncNode, "first start"); - } - else{ + } else { if (pSyncNode->replicaNum == 1) { raftStoreNextTerm(pSyncNode); syncNodeBecomeLeader(pSyncNode, "one replica start"); @@ -1122,7 +1119,7 @@ int32_t syncNodeStart(SSyncNode* pSyncNode) { syncNodeAppendNoop(pSyncNode); } else { syncNodeBecomeFollower(pSyncNode, "first start"); - } + } } int32_t ret = 0; @@ -1136,6 +1133,7 @@ int32_t syncNodeStart(SSyncNode* pSyncNode) { int32_t syncNodeStartStandBy(SSyncNode* pSyncNode) { // state change pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; + pSyncNode->roleTimeMs = taosGetTimestampMs(); syncNodeStopHeartbeatTimer(pSyncNode); // reset elect timer, long enough @@ -1438,7 +1436,7 @@ static bool syncIsConfigChanged(const SSyncCfg* pOldCfg, const SSyncCfg* pNewCfg const SNodeInfo* pNewInfo = &pNewCfg->nodeInfo[i]; if (strcmp(pOldInfo->nodeFqdn, pNewInfo->nodeFqdn) != 0) return true; if (pOldInfo->nodePort != pNewInfo->nodePort) return true; - if(pOldInfo->nodeRole != pNewInfo->nodeRole) return true; + if (pOldInfo->nodeRole != pNewInfo->nodeRole) return true; } return false; @@ -1475,10 +1473,9 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } // log begin config change - sNInfo(pSyncNode, "begin do config change, from %d to %d, from %" PRId64 " to %" PRId64 ", replicas:%d", - pSyncNode->vgId, - oldConfig.totalReplicaNum, pNewConfig->totalReplicaNum, - oldConfig.lastIndex, pNewConfig->lastIndex); + sNInfo(pSyncNode, "begin do config change, from %d to %d, from %" PRId64 " to %" PRId64 ", replicas:%d", + pSyncNode->vgId, oldConfig.totalReplicaNum, pNewConfig->totalReplicaNum, oldConfig.lastIndex, + pNewConfig->lastIndex); if (IamInNew) { pSyncNode->raftCfg.isStandBy = 0; // change isStandBy to normal @@ -1595,6 +1592,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // persist cfg syncWriteCfgFile(pSyncNode); +#if 0 // change isStandBy to normal (election timeout) if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { syncNodeBecomeLeader(pSyncNode, ""); @@ -1606,6 +1604,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } else { syncNodeBecomeFollower(pSyncNode, ""); } +#endif } else { // persist cfg syncWriteCfgFile(pSyncNode); @@ -1671,6 +1670,7 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // state change pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; + pSyncNode->roleTimeMs = taosGetTimestampMs(); syncNodeStopHeartbeatTimer(pSyncNode); // trace log @@ -1699,6 +1699,7 @@ void syncNodeBecomeLearner(SSyncNode* pSyncNode, const char* debugStr) { // state change pSyncNode->state = TAOS_SYNC_STATE_LEARNER; + pSyncNode->roleTimeMs = taosGetTimestampMs(); // trace log sNTrace(pSyncNode, "become learner %s", debugStr); @@ -1734,8 +1735,6 @@ void syncNodeBecomeLearner(SSyncNode* pSyncNode, const char* debugStr) { // /\ UNCHANGED <> // void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { - pSyncNode->leaderTime = taosGetTimestampMs(); - pSyncNode->becomeLeaderNum++; pSyncNode->hbrSlowNum = 0; @@ -1744,6 +1743,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { // state change pSyncNode->state = TAOS_SYNC_STATE_LEADER; + pSyncNode->roleTimeMs = taosGetTimestampMs(); // set leader cache pSyncNode->leaderCache = pSyncNode->myRaftId; @@ -1843,6 +1843,7 @@ int32_t syncNodePeerStateInit(SSyncNode* pSyncNode) { void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { ASSERT(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER); pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE; + pSyncNode->roleTimeMs = taosGetTimestampMs(); SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); sInfo("vgId:%d, become candidate from follower. term:%" PRId64 ", commit index:%" PRId64 ", last index:%" PRId64, pSyncNode->vgId, raftStoreGetTerm(pSyncNode), pSyncNode->commitIndex, lastIndex); @@ -2234,7 +2235,10 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { syncNodeRelease(pSyncNode); } -static void deleteCacheEntry(const void* key, size_t keyLen, void* value) { taosMemoryFree(value); } +static void deleteCacheEntry(const void* key, size_t keyLen, void* value, void* ud) { + (void)ud; + taosMemoryFree(value); +} int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHandle** h) { SSyncLogStoreData* pData = pLogStore->data; @@ -2243,7 +2247,7 @@ int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHand int32_t code = 0; int32_t entryLen = sizeof(*pEntry) + pEntry->dataLen; LRUStatus status = taosLRUCacheInsert(pLogStore->pCache, &pEntry->index, sizeof(pEntry->index), pEntry, entryLen, - deleteCacheEntry, h, TAOS_LRU_PRIORITY_LOW); + deleteCacheEntry, h, TAOS_LRU_PRIORITY_LOW, NULL); if (status != TAOS_LRU_STATUS_OK) { code = -1; } @@ -2409,9 +2413,13 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pMsgReply->startTime = ths->startTime; pMsgReply->timeStamp = tsMs; - sTrace( - "vgId:%d, heartbeat msg from dnode:%d, cluster:%d, Msgterm:%" PRId64 " currentTerm:%" PRId64, - ths->vgId, DID(&(pMsg->srcId)), CID(&(pMsg->srcId)), pMsg->term, currentTerm); + sTrace("vgId:%d, heartbeat msg from dnode:%d, cluster:%d, Msgterm:%" PRId64 " currentTerm:%" PRId64, ths->vgId, + DID(&(pMsg->srcId)), CID(&(pMsg->srcId)), pMsg->term, currentTerm); + + if (pMsg->term > currentTerm && ths->state == TAOS_SYNC_STATE_LEARNER) { + raftStoreSetTerm(ths, pMsg->term); + currentTerm = pMsg->term; + } if (pMsg->term == currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) { syncIndexMgrSetRecvTime(ths->pNextIndex, &(pMsg->srcId), tsMs); diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 8bb72de518cd7ae4f91d0a6db631f7bc34aeeffc..92f34db16d2d84b0edbf284ecb5c0b1c0a5abc60 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -482,6 +482,7 @@ int64_t syncLogBufferProceed(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncTerm* p if (syncLogStorePersist(pLogStore, pNode, pEntry) < 0) { sError("vgId:%d, failed to persist sync log entry from buffer since %s. index:%" PRId64, pNode->vgId, terrstr(), pEntry->index); + taosMsleep(1); goto _out; } ASSERT(pEntry->index == pBuf->matchIndex); @@ -717,7 +718,7 @@ _out: return ret; } -int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; SRaftId destId = pMsg->srcId; ASSERT(pMgr->restored == false); @@ -820,15 +821,15 @@ int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncApp } if (pMgr->restored) { - (void)syncLogReplProcessReplyAsNormal(pMgr, pNode, pMsg); + (void)syncLogReplContinue(pMgr, pNode, pMsg); } else { - (void)syncLogReplProcessReplyAsRecovery(pMgr, pNode, pMsg); + (void)syncLogReplRecover(pMgr, pNode, pMsg); } taosThreadMutexUnlock(&pBuf->mutex); return 0; } -int32_t syncLogReplDoOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplStart(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->restored) { (void)syncLogReplAttempt(pMgr, pNode); } else { @@ -931,7 +932,7 @@ int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { return 0; } -int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplContinue(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { ASSERT(pMgr->restored == true); if (pMgr->startIndex <= pMsg->lastSendIndex && pMsg->lastSendIndex < pMgr->endIndex) { if (pMgr->startIndex < pMgr->matchIndex && pMgr->retryBackoff > 0) { diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 5a3a3bbb709e63d3fb512a6e3bead861df16a5e5..2c7fc70ae9f303b3b7d57e863aa1a453828d93b0 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -72,7 +72,7 @@ int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) { continue; } SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; - (void)syncLogReplDoOnce(pMgr, pNode); + (void)syncLogReplStart(pMgr, pNode); } return 0; } diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 5ee67da9ab8d2cef07b7d85ef275199e8cf2ef0e..37166805cee2746313b8511776928ccb74255878 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -87,22 +87,6 @@ static int32_t syncNodeTimerRoutine(SSyncNode* ths) { } } - if (atomic_load_64(&ths->snapshottingIndex) != SYNC_INDEX_INVALID) { - // end timeout wal snapshot - if (timeNow - ths->snapshottingTime > SYNC_DEL_WAL_MS && - atomic_load_64(&ths->snapshottingIndex) != SYNC_INDEX_INVALID) { - SSyncLogStoreData* pData = ths->pLogStore->data; - int32_t code = walEndSnapshot(pData->pWal); - if (code != 0) { - sNError(ths, "timer wal snapshot end error since:%s", terrstr()); - return -1; - } else { - sNTrace(ths, "wal snapshot end, index:%" PRId64, atomic_load_64(&ths->snapshottingIndex)); - atomic_store_64(&ths->snapshottingIndex, SYNC_INDEX_INVALID); - } - } - } - if (!syncNodeIsMnode(ths)) { syncRespClean(ths->pSyncRespMgr); } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index cf796c386297271d1138189ea812fe4b1fbe984c..ae1c775a18f6c47291f3065f83db0c2ff8cee94e 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -364,10 +364,10 @@ void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, int64 if (timeDiff > SYNC_HEARTBEAT_SLOW_MS) { pSyncNode->hbSlowNum++; - sNInfo(pSyncNode, - "recv sync-heartbeat from dnode:%d slow {term:%" PRId64 ", commit-index:%" PRId64 ", min-match:%" PRId64 - ", ts:%" PRId64 "}, %s, net elapsed:%" PRId64, - DID(&pMsg->srcId), pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s, timeDiff); + sNTrace(pSyncNode, + "recv sync-heartbeat from dnode:%d slow {term:%" PRId64 ", commit-index:%" PRId64 ", min-match:%" PRId64 + ", ts:%" PRId64 "}, %s, net elapsed:%" PRId64, + DID(&pMsg->srcId), pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s, timeDiff); } sNTrace(pSyncNode, diff --git a/source/libs/sync/test/sync_test_lib/src/syncIO.c b/source/libs/sync/test/sync_test_lib/src/syncIO.c index 2e0078558662f5a8aaee5af792f2c28a0cbec5b2..4f8ae5934880c589d13978a68db92f3303aa4d33 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncIO.c +++ b/source/libs/sync/test/sync_test_lib/src/syncIO.c @@ -21,6 +21,7 @@ #include "tglobal.h" #include "ttimer.h" #include "tutil.h" +#include "tversion.h" bool gRaftDetailLog = false; SSyncIO *gSyncIO = NULL; @@ -188,7 +189,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) { rpcInit.idleTime = 100; rpcInit.user = "sync-io"; rpcInit.connType = TAOS_CONN_CLIENT; - + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); io->clientRpc = rpcOpen(&rpcInit); if (io->clientRpc == NULL) { sError("failed to initialize RPC"); @@ -209,7 +210,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) { rpcInit.idleTime = 2 * 1500; rpcInit.parent = io; rpcInit.connType = TAOS_CONN_SERVER; - + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { sError("failed to start RPC server"); @@ -470,11 +471,10 @@ static void syncIOTickPing(void *param, void *tmrId) { taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer); } -void syncEntryDestory(SSyncRaftEntry* pEntry) {} - +void syncEntryDestory(SSyncRaftEntry *pEntry) {} -void syncUtilMsgNtoH(void* msg) { - SMsgHead* pHead = msg; +void syncUtilMsgNtoH(void *msg) { + SMsgHead *pHead = msg; pHead->contLen = ntohl(pHead->contLen); pHead->vgId = ntohl(pHead->vgId); } @@ -487,9 +487,9 @@ static inline bool syncUtilCanPrint(char c) { } } -char* syncUtilPrintBin(char* ptr, uint32_t len) { +char *syncUtilPrintBin(char *ptr, uint32_t len) { int64_t memLen = (int64_t)(len + 1); - char* s = taosMemoryMalloc(memLen); + char *s = taosMemoryMalloc(memLen); ASSERT(s != NULL); memset(s, 0, len + 1); memcpy(s, ptr, len); @@ -502,13 +502,13 @@ char* syncUtilPrintBin(char* ptr, uint32_t len) { return s; } -char* syncUtilPrintBin2(char* ptr, uint32_t len) { +char *syncUtilPrintBin2(char *ptr, uint32_t len) { uint32_t len2 = len * 4 + 1; - char* s = taosMemoryMalloc(len2); + char *s = taosMemoryMalloc(len2); ASSERT(s != NULL); memset(s, 0, len2); - char* p = s; + char *p = s; for (int32_t i = 0; i < len; ++i) { int32_t n = sprintf(p, "%d,", ptr[i]); p += n; @@ -516,7 +516,7 @@ char* syncUtilPrintBin2(char* ptr, uint32_t len) { return s; } -void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port) { +void syncUtilU642Addr(uint64_t u64, char *host, int64_t len, uint16_t *port) { uint32_t hostU32 = (uint32_t)((u64 >> 32) & 0x00000000FFFFFFFF); struct in_addr addr = {.s_addr = hostU32}; @@ -524,7 +524,7 @@ void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port) { *port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16); } -uint64_t syncUtilAddr2U64(const char* host, uint16_t port) { +uint64_t syncUtilAddr2U64(const char *host, uint16_t port) { uint32_t hostU32 = taosGetIpv4FromFqdn(host); if (hostU32 == (uint32_t)-1) { sError("failed to resolve ipv4 addr, host:%s", host); diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 0e20941b3ae472986d5116616eda05af93a83dc1..4dd47e97ac94bdba2b80bd6f5815a38e987bc9d8 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -46,12 +46,16 @@ int32_t tdbAlter(TDB *pDb, int pages); int32_t tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb, int8_t rollback); int32_t tdbTbClose(TTB *pTb); +bool tdbTbExist(const char *tbname, TDB *pEnv); +int tdbTbDropByName(const char *tbname, TDB *pEnv, TXN* pTxn); int32_t tdbTbDrop(TTB *pTb); int32_t tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); int32_t tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn); int32_t tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); int32_t tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen); int32_t tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); +int32_t tdbTbTraversal(TTB *pTb, void *data, + int32_t (*func)(const void *pKey, int keyLen, const void *pVal, int valLen, void *data)); // TBC int32_t tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c49b5726b647b2f9cb7741331b1dd6b6ede3cd87..612179b2053ded041941a01e9f2aca905da3f72b 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -233,7 +233,11 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { int ret; tdbBtcOpen(&btc, pBt, pTxn); - + /* + btc.coder.ofps = taosArrayInit(8, sizeof(SPage *)); + // btc.coder.ofps = taosArrayInit(8, sizeof(SPgno)); + //pBtc->coder.ofps = taosArrayInit(8, sizeof(SPage *)); + */ tdbTrace("tdb delete, btc: %p, pTxn: %p", &btc, pTxn); // move the cursor @@ -254,7 +258,18 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { tdbBtcClose(&btc); return -1; } + /* + SArray *ofps = btc.coder.ofps; + if (ofps) { + for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { + SPage *ofp = *(SPage **)taosArrayGet(ofps, i); + tdbPagerInsertFreePage(btc.pBt->pPager, ofp, btc.pTxn); + } + taosArrayDestroy(ofps); + btc.coder.ofps = NULL; + } + */ tdbBtcClose(&btc); return 0; } @@ -345,7 +360,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL } *ppKey = pTKey; *pkLen = cd.kLen; - memcpy(*ppKey, cd.pKey, cd.kLen); + memcpy(*ppKey, cd.pKey, (size_t)cd.kLen); } if (ppVal) { @@ -357,7 +372,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL } *ppVal = pTVal; *vLen = cd.vLen; - memcpy(*ppVal, cd.pVal, cd.vLen); + memcpy(*ppVal, cd.pVal, (size_t)cd.vLen); } if (TDB_CELLDECODER_FREE_KEY(&cd)) { @@ -563,6 +578,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } } // copy the parent key out if child pages are not leaf page + // childNotLeaf = !(TDB_BTREE_PAGE_IS_LEAF(pOlds[0]) || TDB_BTREE_PAGE_IS_OVFL(pOlds[0])); childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(pOlds[0]); if (childNotLeaf) { for (int i = 0; i < nOlds; i++) { @@ -592,7 +608,30 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx for (int i = 0; i < nOlds; i++) { nCells = TDB_PAGE_TOTAL_CELLS(pParent); if (sIdx < nCells) { + bool destroyOfps = false; + if (!childNotLeaf) { + if (!pParent->pPager->ofps) { + pParent->pPager->ofps = taosArrayInit(8, sizeof(SPage *)); + destroyOfps = true; + } + } + tdbPageDropCell(pParent, sIdx, pTxn, pBt); + + if (!childNotLeaf) { + SArray *ofps = pParent->pPager->ofps; + if (ofps) { + for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { + SPage *ofp = *(SPage **)taosArrayGet(ofps, i); + tdbPagerInsertFreePage(pParent->pPager, ofp, pTxn); + } + + if (destroyOfps) { + taosArrayDestroy(ofps); + pParent->pPager->ofps = NULL; + } + } + } } else { ((SIntHdr *)pParent->pData)->pgno = 0; } @@ -861,6 +900,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx if (!TDB_BTREE_PAGE_IS_LEAF(pNews[0])) { ((SIntHdr *)(pParent->pData))->pgno = ((SIntHdr *)(pNews[0]->pData))->pgno; } + + tdbPagerInsertFreePage(pBt->pPager, pNews[0], pTxn); } for (int i = 0; i < 3; i++) { @@ -870,6 +911,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } for (pageIdx = 0; pageIdx < nOlds; ++pageIdx) { + if (pageIdx >= nNews) { + tdbPagerInsertFreePage(pBt->pPager, pOlds[pageIdx], pTxn); + } tdbPagerReturnPage(pBt->pPager, pOlds[pageIdx], pTxn); } for (; pageIdx < nNews; ++pageIdx) { @@ -1311,7 +1355,11 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, if (ret < 0) { return -1; } - + /* + if (pDecoder->ofps) { + taosArrayPush(pDecoder->ofps, &ofp); + } + */ ofpCell = tdbPageGetCell(ofp, 0); if (nLeft <= ofp->maxLocal - sizeof(SPgno)) { @@ -1346,11 +1394,17 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, int lastKeyPageSpace = 0; // load left key & val to ovpages while (pgno != 0) { + tdbTrace("tdb decode-ofp, pTxn: %p, pgno:%u by cell:%p", pTxn, pgno, pCell); + // printf("tdb decode-ofp, pTxn: %p, pgno:%u by cell:%p\n", pTxn, pgno, pCell); ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt); if (ret < 0) { return -1; } - + /* + if (pDecoder->ofps) { + taosArrayPush(pDecoder->ofps, &ofp); + } + */ ofpCell = tdbPageGetCell(ofp, 0); int lastKeyPage = 0; @@ -1518,8 +1572,8 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * if (pPage->vLen == TDB_VARIANT_LEN) { if (!leaf) { - tdbError("tdb/btree-cell-size: not a leaf page."); - return -1; + tdbError("tdb/btree-cell-size: not a leaf page:%p, pgno:%" PRIu32 ".", pPage, TDB_PAGE_PGNO(pPage)); + // return -1; } nHeader += tdbGetVarInt(pCell + nHeader, &vLen); } else if (leaf) { @@ -1559,8 +1613,27 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * bytes = ofp->maxLocal - sizeof(SPgno); } + // SPgno origPgno = pgno; memcpy(&pgno, ofpCell + bytes, sizeof(pgno)); + ret = tdbPagerWrite(pBt->pPager, ofp); + if (ret < 0) { + tdbError("failed to write page since %s", terrstr()); + return -1; + } + /* + tdbPageDropCell(ofp, 0, pTxn, pBt); + */ + // SIntHdr *pIntHdr = (SIntHdr *)(ofp->pData); + // pIntHdr->flags = TDB_FLAG_ADD(0, TDB_BTREE_OVFL); + // pIntHdr->pgno = 0; + // ofp->pPager = NULL; + + SArray *ofps = pPage->pPager->ofps; + if (ofps) { + taosArrayPush(ofps, &ofp); + } + tdbPagerReturnPage(pPage->pPager, ofp, pTxn); nLeft -= bytes; @@ -1793,7 +1866,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { *ppKey = pKey; *kLen = cd.kLen; - memcpy(pKey, cd.pKey, cd.kLen); + memcpy(pKey, cd.pKey, (size_t)cd.kLen); if (ppVal) { if (cd.vLen > 0) { @@ -1852,7 +1925,7 @@ int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { *ppKey = pKey; *kLen = cd.kLen; - memcpy(pKey, cd.pKey, cd.kLen); + memcpy(pKey, cd.pKey, (size_t)cd.kLen); if (ppVal) { // TODO: vLen may be zero @@ -1864,7 +1937,7 @@ int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { *ppVal = pVal; *vLen = cd.vLen; - memcpy(pVal, cd.pVal, cd.vLen); + memcpy(pVal, cd.pVal, (size_t)cd.vLen); } ret = tdbBtcMoveToPrev(pBtc); @@ -1980,6 +2053,11 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { return -1; } + if (TDB_BTREE_PAGE_IS_OVFL(pBtc->pPage)) { + tdbError("tdb/btc-move-downward: should not be a ovfl page here."); + return -1; + } + if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); pgno = ((SPgno *)pCell)[0]; @@ -2068,8 +2146,27 @@ int tdbBtcDelete(SBTC *pBtc) { return -1; } + bool destroyOfps = false; + if (!pBtc->pPage->pPager->ofps) { + pBtc->pPage->pPager->ofps = taosArrayInit(8, sizeof(SPage *)); + destroyOfps = true; + } + tdbPageDropCell(pBtc->pPage, idx, pBtc->pTxn, pBtc->pBt); + SArray *ofps = pBtc->pPage->pPager->ofps; + if (ofps) { + for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { + SPage *ofp = *(SPage **)taosArrayGet(ofps, i); + tdbPagerInsertFreePage(pBtc->pPage->pPager, ofp, pBtc->pTxn); + } + + if (destroyOfps) { + taosArrayDestroy(ofps); + pBtc->pPage->pPager->ofps = NULL; + } + } + // update interior page or do balance if (idx == nCells - 1) { if (idx) { @@ -2113,6 +2210,8 @@ int tdbBtcDelete(SBTC *pBtc) { return -1; } + // printf("tdb/btc-delete: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pBtc->pPage)); + ret = tdbBtreeBalance(pBtc); if (ret < 0) { tdbError("tdb/btc-delete: btree balance failed with ret: %d.", ret); @@ -2181,7 +2280,13 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int tdbError("tdb/btc-upsert: page insert/update cell failed with ret: %d.", ret); return -1; } - + /* + bool destroyOfps = false; + if (!pBtc->pPage->pPager->ofps) { + pBtc->pPage->pPager->ofps = taosArrayInit(8, sizeof(SPage *)); + destroyOfps = true; + } + */ // check balance if (pBtc->pPage->nOverflow > 0) { ret = tdbBtreeBalance(pBtc); @@ -2190,7 +2295,20 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int return -1; } } + /* + SArray *ofps = pBtc->pPage->pPager->ofps; + if (ofps) { + for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { + SPage *ofp = *(SPage **)taosArrayGet(ofps, i); + tdbPagerInsertFreePage(pBtc->pPage->pPager, ofp, pBtc->pTxn); + } + if (destroyOfps) { + taosArrayDestroy(ofps); + pBtc->pPage->pPager->ofps = NULL; + } + } + */ return 0; } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 952c49db73a2c8986d67c8e327175ad00e778ae6..4f595d8d4ae83f4003082bd1b1b3ffb6816d473e 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -62,7 +62,10 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i } memset(pDb->pgrHash, 0, tsize); - taosMulModeMkDir(dbname, 0755); + ret = taosMulModeMkDir(dbname, 0755); + if (ret < 0) { + return -1; + } #ifdef USE_MAINDB // open main db @@ -70,6 +73,11 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i if (ret < 0) { return -1; } + + ret = tdbTbOpen(TDB_FREEDB_NAME, sizeof(SPgno), 0, NULL, pDb, &pDb->pFreeDb, rollback); + if (ret < 0) { + return -1; + } #endif *ppDb = pDb; @@ -82,6 +90,7 @@ int tdbClose(TDB *pDb) { if (pDb) { #ifdef USE_MAINDB if (pDb->pMainDb) tdbTbClose(pDb->pMainDb); + if (pDb->pFreeDb) tdbTbClose(pDb->pFreeDb); #endif for (pPager = pDb->pgrList; pPager; pPager = pDb->pgrList) { diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 5ea9be63dbf7bb9415668d232324ca1272b49db6..474e5d227071de65cd91811ac27c018131da818d 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -292,7 +292,23 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { */ return 0; } +/* +int tdbPagerCancelDirty(SPager *pPager, SPage *pPage, TXN *pTxn) { + SRBTreeNode *pNode = tRBTreeGet(&pPager->rbt, (SRBTreeNode *)pPage); + if (pNode) { + pPage->isDirty = 0; + + tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); + if (pTxn->jPageSet) { + hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); + } + + tdbPCacheRelease(pPager->pCache, pPage, pTxn); + } + return 0; +} +*/ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { SPage *pPage; int ret; @@ -338,10 +354,13 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { if (pTxn->jPageSet) { hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); } + + tdbTrace("tdb/pager-commit: remove page: %p %d from dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt); + tdbPCacheRelease(pPager->pCache, pPage, pTxn); } - tdbTrace("pager/commit reset dirty tree: %p", &pPager->rbt); + tdbTrace("tdb/pager-commit reset dirty tree: %p", &pPager->rbt); tRBTreeCreate(&pPager->rbt, pageCmpFn); // sync the db file @@ -629,6 +648,8 @@ int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) { return 0; } +static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno, TXN *pTxn); + int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, TXN *pTxn) { SPage *pPage; @@ -643,7 +664,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa // alloc new page if (pgno == 0) { loadPage = 0; - ret = tdbPagerAllocPage(pPager, &pgno); + ret = tdbPagerAllocPage(pPager, &pgno, pTxn); if (ret < 0) { tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); return -1; @@ -695,23 +716,114 @@ void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn) { // TDB_PAGE_PGNO(pPage), pPage); } -static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) { - // TODO: Allocate a page from the free list +int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) { + int code = 0; + SPgno pgno = TDB_PAGE_PGNO(pPage); + + if (pPager->frps) { + taosArrayPush(pPager->frps, &pgno); + pPage->pPager = NULL; + return code; + } + + pPager->frps = taosArrayInit(8, sizeof(SPgno)); + // memset(pPage->pData, 0, pPage->pageSize); + tdbTrace("tdb/insert-free-page: tbc recycle page: %d.", pgno); + // printf("tdb/insert-free-page: tbc recycle page: %d.\n", pgno); + code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn); + if (code < 0) { + tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); + taosArrayDestroy(pPager->frps); + pPager->frps = NULL; + return -1; + } + + while (TARRAY_SIZE(pPager->frps) > 0) { + pgno = *(SPgno *)taosArrayPop(pPager->frps); + + code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn); + if (code < 0) { + tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); + taosArrayDestroy(pPager->frps); + pPager->frps = NULL; + return -1; + } + } + + taosArrayDestroy(pPager->frps); + pPager->frps = NULL; + + pPage->pPager = NULL; + + return code; +} + +static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) { + int code = 0; + TBC *pCur; + + if (!pPager->pEnv->pFreeDb) { + return code; + } + + if (pPager->frps) { + return code; + } + + code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, pTxn); + if (code < 0) { + return 0; + } + + code = tdbTbcMoveToFirst(pCur); + if (code) { + tdbError("tdb/remove-free-page: moveto first failed with ret: %d.", code); + tdbTbcClose(pCur); + return 0; + } + + void *pKey = NULL; + int nKey = 0; + + code = tdbTbcGet(pCur, (const void **)&pKey, &nKey, NULL, NULL); + if (code < 0) { + // tdbError("tdb/remove-free-page: tbc get failed with ret: %d.", code); + tdbTbcClose(pCur); + return 0; + } + + *pPgno = *(SPgno *)pKey; + tdbTrace("tdb/remove-free-page: tbc get page: %d.", *pPgno); + // printf("tdb/remove-free-page: tbc get page: %d.\n", *pPgno); + + code = tdbTbcDelete(pCur); + if (code < 0) { + tdbError("tdb/remove-free-page: tbc delete failed with ret: %d.", code); + tdbTbcClose(pCur); + return 0; + } + tdbTbcClose(pCur); return 0; } +static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno, TXN *pTxn) { + // Allocate a page from the free list + return tdbPagerRemoveFreePage(pPager, ppgno, pTxn); +} + static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) { *ppgno = ++pPager->dbFileSize; + // tdbError("tdb/alloc-new-page: %d.", *ppgno); return 0; } -int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) { +static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno, TXN *pTxn) { int ret; *ppgno = 0; // Try to allocate from the free list of the pager - ret = tdbPagerAllocFreePage(pPager, ppgno); + ret = tdbPagerAllocFreePage(pPager, ppgno, pTxn); if (ret < 0) { return -1; } @@ -980,6 +1092,7 @@ int tdbPagerRestoreJournals(SPager *pPager) { jname[dirLen] = '/'; sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); if (tdbPagerRestore(pPager, jname) < 0) { + taosArrayDestroy(pTxnList); tdbCloseDir(&pDir); tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), jname); diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 18d14fa474f51104442cf689bd52649598b9c396..446a21f312ba4e3118744ae2f8837126c9ff3fcf 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -134,11 +134,67 @@ int tdbTbClose(TTB *pTb) { return 0; } +bool tdbTbExist(const char *tbname, TDB *pEnv) { + bool exist = false; + SPager *pPager; + char fFullName[TDB_FILENAME_LEN]; + +#ifdef USE_MAINDB + + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, TDB_MAINDB_NAME); + + if (strcmp(TDB_MAINDB_NAME, tbname)) { + pPager = tdbEnvGetPager(pEnv, fFullName); + + exist = tdbTbGet(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, NULL, NULL) == 0; + } else { + exist = taosCheckExistFile(fFullName); + } + +#else + + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, tbname); + + exist = taosCheckExistFile(fFullName); + +#endif + + return exist; +} + int tdbTbDrop(TTB *pTb) { // TODO return 0; } +int tdbTbDropByName(const char *tbname, TDB *pEnv, TXN *pTxn) { + int ret; + SPager *pPager; + char fFullName[TDB_FILENAME_LEN]; + +#ifdef USE_MAINDB + + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, TDB_MAINDB_NAME); + + if (strcmp(TDB_MAINDB_NAME, tbname)) { + pPager = tdbEnvGetPager(pEnv, fFullName); + + ret = tdbTbDelete(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, pTxn); + } else { + ret = taosRemoveFile(fFullName); + } + +#else + + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, tbname); + + ret = taosRemoveFile(fFullName); + +#endif + + return ret; +} + int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { return tdbBtreeInsert(pTb->pBt, pKey, keyLen, pVal, valLen, pTxn); } @@ -173,6 +229,38 @@ int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn) { return 0; } +int32_t tdbTbTraversal(TTB *pTb, void *data, + int32_t (*func)(const void *pKey, int keyLen, const void *pVal, int valLen, void *data)) { + TBC *pCur; + int ret = tdbTbcOpen(pTb, &pCur, NULL); + if (ret < 0) { + return ret; + } + + tdbTbcMoveToFirst(pCur); + + void *pKey = NULL; + int kLen = 0; + void *pValue = NULL; + int vLen = 0; + + while (1) { + ret = tdbTbcNext(pCur, &pKey, &kLen, &pValue, &vLen); + if (ret < 0) { + ret = 0; + break; + } + + ret = func(pKey, kLen, pValue, vLen, data); + if (ret < 0) break; + } + tdbFree(pKey); + tdbFree(pValue); + tdbTbcClose(pCur); + + return 0; +} + int tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c) { return tdbBtcMoveTo(&pTbc->btc, pKey, kLen, c); } int tdbTbcMoveToFirst(TBC *pTbc) { return tdbBtcMoveToFirst(&pTbc->btc); } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 7a0bcc00a42a95d1f27b23384e494416e110b2e1..8ce294a3c6565ccffb3efa214fe75a147768143b 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -131,13 +131,14 @@ typedef struct SBtInfo { #define TDB_CELLDECODER_FREE_VAL(pCellDecoder) ((pCellDecoder)->freeKV & TDB_CELLD_F_VAL) typedef struct { - int kLen; - u8 *pKey; - int vLen; - u8 *pVal; - SPgno pgno; - u8 *pBuf; - u8 freeKV; + int kLen; + u8 *pKey; + int vLen; + u8 *pVal; + SPgno pgno; + u8 *pBuf; + u8 freeKV; + SArray *ofps; } SCellDecoder; struct SBTC { @@ -198,9 +199,10 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn); int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, TXN *pTxn); void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); -int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); -int tdbPagerRestoreJournals(SPager *pPager); -int tdbPagerRollback(SPager *pPager); +int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn); +// int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); +int tdbPagerRestoreJournals(SPager *pPager); +int tdbPagerRollback(SPager *pPager); // tdbPCache.c ==================================== #define TDB_PCACHE_PAGE \ @@ -373,6 +375,7 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { #ifdef USE_MAINDB #define TDB_MAINDB_NAME "main.tdb" +#define TDB_FREEDB_NAME "_free.db" #endif struct STDB { @@ -386,6 +389,7 @@ struct STDB { SPager **pgrHash; #ifdef USE_MAINDB TTB *pMainDb; + TTB *pFreeDb; #endif int64_t txnId; }; @@ -403,6 +407,8 @@ struct SPager { SRBTree rbt; // u8 inTran; TXN *pActiveTxn; + SArray *ofps; + SArray *frps; SPager *pNext; // used by TDB SPager *pHashNext; // used by TDB #ifdef USE_MAINDB diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index fd4d7c101d06ce73de37678e858716326e83feee..4715ccbd4157e65eb8665677a94788e2c0ed234b 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -14,3 +14,7 @@ target_link_libraries(tdbExOVFLTest tdb gtest gtest_main) add_executable(tdbPageDefragmentTest "tdbPageDefragmentTest.cpp") target_link_libraries(tdbPageDefragmentTest tdb gtest gtest_main) +# page recycling testing +add_executable(tdbPageRecycleTest "tdbPageRecycleTest.cpp") +target_link_libraries(tdbPageRecycleTest tdb gtest gtest_main) + diff --git a/source/libs/tdb/test/tdbExOVFLTest.cpp b/source/libs/tdb/test/tdbExOVFLTest.cpp index b16bc643d3687e00b04cfdcdc9f544e56cd0d0c4..325703c946842308249dcd0ec7170447cdb2bd92 100644 --- a/source/libs/tdb/test/tdbExOVFLTest.cpp +++ b/source/libs/tdb/test/tdbExOVFLTest.cpp @@ -190,6 +190,15 @@ static void insertOfp(void) { // commit current transaction tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); } // TEST(TdbOVFLPagesTest, DISABLED_TbInsertTest) { @@ -233,6 +242,13 @@ TEST(TdbOVFLPagesTest, TbGetTest) { tdbFree(pVal); } + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); } // TEST(TdbOVFLPagesTest, DISABLED_TbDeleteTest) { @@ -334,6 +350,15 @@ tdbBegin(pEnv, &txn); // commit current transaction tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); } // TEST(tdb_test, DISABLED_simple_insert1) { @@ -407,6 +432,8 @@ TEST(tdb_test, simple_insert1) { tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); + closePool(pPool); + { // Query the data void *pVal = NULL; int vLen; diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4d7b314917a822fde5d4c2ad091ba6130ed47b22 --- /dev/null +++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp @@ -0,0 +1,835 @@ +#include + +#define ALLOW_FORBID_FUNC +#include "os.h" +#include "tdb.h" + +#include +#include +#include +#include +#include "tlog.h" + +typedef struct SPoolMem { + int64_t size; + struct SPoolMem *prev; + struct SPoolMem *next; +} SPoolMem; + +static SPoolMem *openPool() { + SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool)); + + pPool->prev = pPool->next = pPool; + pPool->size = 0; + + return pPool; +} + +static void clearPool(SPoolMem *pPool) { + SPoolMem *pMem; + + do { + pMem = pPool->next; + + if (pMem == pPool) break; + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); + } while (1); + + assert(pPool->size == 0); +} + +static void closePool(SPoolMem *pPool) { + clearPool(pPool); + taosMemoryFree(pPool); +} + +static void *poolMalloc(void *arg, size_t size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void poolFree(void *arg, void *ptr) { + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = &(((SPoolMem *)ptr)[-1]); + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); +} + +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + int k1, k2; + + std::string s1((char *)pKey1 + 3, kLen1 - 3); + std::string s2((char *)pKey2 + 3, kLen2 - 3); + k1 = stoi(s1); + k2 = stoi(s2); + + if (k1 < k2) { + return -1; + } else if (k1 > k2) { + return 1; + } else { + return 0; + } +} + +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; +} + +static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) { + TDB *pEnv = NULL; + + int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0); + if (ret) { + pEnv = NULL; + } + + return pEnv; +} + +static void generateBigVal(char *val, int valLen) { + for (int i = 0; i < valLen; ++i) { + char c = char(i & 0xff); + if (c == 0) { + c = 1; + } + val[i] = c; + } +} + +static void insertOfp(void) { + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + char const *key = "key123456789"; + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); +} + +static void clearDb(char const *db) { taosRemoveDir(db); } + +TEST(TdbPageRecycleTest, DISABLED_TbInsertTest) { + // TEST(TdbPageRecycleTest, TbInsertTest) { + // ofp inserting + clearDb("tdb"); + insertOfp(); +} + +TEST(TdbPageRecycleTest, DISABLED_TbGetTest) { + // TEST(TdbPageRecycleTest, TbGetTest) { + clearDb("tdb"); + insertOfp(); + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // int ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + int ret = tdbTbOpen("ofp_insert.db", 12, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + { // Query the data + void *pVal = NULL; + int vLen; + + // char const *key = "key1"; + char const *key = "key123456789"; + ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, valLen); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + + tdbFree(pVal); + } +} + +TEST(TdbPageRecycleTest, DISABLED_TbDeleteTest) { + // TEST(TdbPageRecycleTest, TbDeleteTest) { + int ret = 0; + + taosRemoveDir("tdb"); + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[((4083 - 4 - 3 - 2) + 1) * 2]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + { // insert the generated big data + ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + { // query the data + void *pVal = NULL; + int vLen; + + ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, valLen); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + + tdbFree(pVal); + } + /* open to debug committed file +tdbCommit(pEnv, &txn); +tdbTxnClose(&txn); + +++txnid; +tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); +tdbBegin(pEnv, &txn); + */ + { // upsert the data + ret = tdbTbUpsert(pDb, "key1", strlen("key1"), "value1", strlen("value1"), txn); + GTEST_ASSERT_EQ(ret, 0); + } + + { // query the upserted data + void *pVal = NULL; + int vLen; + + ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, strlen("value1")); + GTEST_ASSERT_EQ(memcmp("value1", pVal, vLen), 0); + + tdbFree(pVal); + } + + { // delete the data + ret = tdbTbDelete(pDb, "key1", strlen("key1"), txn); + GTEST_ASSERT_EQ(ret, 0); + } + + { // query the deleted data + void *pVal = NULL; + int vLen = -1; + + ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); + ASSERT(ret == -1); + GTEST_ASSERT_EQ(ret, -1); + + GTEST_ASSERT_EQ(vLen, -1); + GTEST_ASSERT_EQ(pVal, nullptr); + + tdbFree(pVal); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); +} + +TEST(TdbPageRecycleTest, DISABLED_simple_insert1) { + // TEST(TdbPageRecycleTest, simple_insert1) { + int ret; + TDB *pEnv; + TTB *pDb; + tdb_cmpr_fn_t compFunc; + int nData = 1; + TXN *txn; + int const pageSize = 4096; + + taosRemoveDir("tdb"); + + // Open Env + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + compFunc = tKeyCmpr; + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + { + char key[64]; + // char val[(4083 - 4 - 3 - 2)]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int iData = 1; iData <= nData; iData++) { + sprintf(key, "key0"); + sprintf(val, "value%d", iData); + + // ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + // GTEST_ASSERT_EQ(ret, 0); + + // generate value payload + int valLen = sizeof(val) / sizeof(val[0]); + for (int i = 6; i < valLen; ++i) { + char c = char(i & 0xff); + if (c == 0) { + c = 1; + } + val[i] = c; + } + + ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + + // commit the transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + { // Query the data + void *pVal = NULL; + int vLen; + + for (int i = 1; i <= nData; i++) { + sprintf(key, "key%d", i); + // sprintf(val, "value%d", i); + + ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, sizeof(val) / sizeof(val[0])); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + } + + tdbFree(pVal); + } + + { // Iterate to query the DB data + TBC *pDBC; + void *pKey = NULL; + void *pVal = NULL; + int vLen, kLen; + int count = 0; + + ret = tdbTbcOpen(pDb, &pDBC, NULL); + GTEST_ASSERT_EQ(ret, 0); + + tdbTbcMoveToFirst(pDBC); + + for (;;) { + ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); + if (ret < 0) break; + + // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; + // std::cout.write((char *)pVal, vLen) /* << " " << vLen */; + // std::cout << std::endl; + + count++; + } + + GTEST_ASSERT_EQ(count, nData); + + tdbTbcClose(pDBC); + + tdbFree(pKey); + tdbFree(pVal); + } + } + + ret = tdbTbDrop(pDb); + GTEST_ASSERT_EQ(ret, 0); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); +} + +static void insertDb(int nData) { + int ret = 0; + TDB *pEnv = NULL; + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc; + TXN *txn = NULL; + int const pageSize = 4 * 1024; + + // Open Env + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + compFunc = tKeyCmpr; + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // 1, insert nData kv + { + char key[64]; + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int iData = 0; iData < nData; ++iData) { + sprintf(key, "key%03d", iData); + sprintf(val, "value%03d", iData); + + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); + GTEST_ASSERT_EQ(ret, 0); + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + + // commit the transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // 2, delete nData/2 records + + closePool(pPool); + } + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); + + system("ls -l ./tdb"); +} + +static void deleteDb(int nData) { + int ret = 0; + TDB *pEnv = NULL; + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc; + TXN *txn = NULL; + int const pageSize = 4 * 1024; + + // Open Env + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + compFunc = tKeyCmpr; + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // 2, delete nData/2 records + { + char key[64]; + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int iData = 0; iData < nData; iData++) { + // if (iData % 2 == 0) continue; + + sprintf(key, "key%03d", iData); + sprintf(val, "value%03d", iData); + + { // delete the data + ret = tdbTbDelete(pDb, key, strlen(key), txn); + GTEST_ASSERT_EQ(ret, 0); + } + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + + // commit the transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + } + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); + + system("ls -l ./tdb"); +} + +static const int nDataConst = 256 * 19; + +// TEST(TdbPageRecycleTest, DISABLED_seq_insert) { +TEST(TdbPageRecycleTest, seq_insert) { + clearDb("tdb"); + insertDb(nDataConst); +} + +// TEST(TdbPageRecycleTest, DISABLED_seq_delete) { +TEST(TdbPageRecycleTest, seq_delete) { deleteDb(nDataConst); } + +// TEST(TdbPageRecycleTest, DISABLED_recycly_insert) { +TEST(TdbPageRecycleTest, recycly_insert) { insertDb(nDataConst); } + +// TEST(TdbPageRecycleTest, DISABLED_recycly_seq_insert_ofp) { +TEST(TdbPageRecycleTest, recycly_seq_insert_ofp) { + clearDb("tdb"); + insertOfp(); + system("ls -l ./tdb"); +} + +static void deleteOfp(void) { + // open Env + int ret = 0; + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + { // delete the data + char const *key = "key123456789"; + ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + ret = tdbTbDrop(pDb); + GTEST_ASSERT_EQ(ret, 0); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); +} + +// TEST(TdbPageRecycleTest, DISABLED_seq_delete_ofp) { +TEST(TdbPageRecycleTest, seq_delete_ofp) { + deleteOfp(); + system("ls -l ./tdb"); +} + +// TEST(TdbPageRecycleTest, DISABLED_recycly_seq_insert_ofp_again) { +TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_again) { + insertOfp(); + system("ls -l ./tdb"); +} + +// TEST(TdbPageRecycleTest, DISABLED_recycly_seq_insert_ofp_nocommit) { +TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_nocommit) { + clearDb("tdb"); + insertOfp(); + system("ls -l ./tdb"); + + // open Env + int ret = 0; + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + { // delete the data + char const *key = "key123456789"; + ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // 1, insert nData kv + { + int nData = nDataConst; + char key[64]; + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + + for (int iData = 0; iData < nData; ++iData) { + sprintf(key, "key%03d", iData); + sprintf(val, "value%03d", iData); + + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); + GTEST_ASSERT_EQ(ret, 0); + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); + + system("ls -l ./tdb"); +} + +// TEST(TdbPageRecycleTest, DISABLED_recycly_delete_interior_ofp_nocommit) { +TEST(TdbPageRecycleTest, recycly_delete_interior_ofp_nocommit) { + clearDb("tdb"); + + // open Env + int ret = 0; + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = NULL; // tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + char key[1024] = {0}; + int count = sizeof(key) / sizeof(key[0]); + for (int i = 0; i < count - 1; ++i) { + key[i] = 'a'; + } + + // insert n ofp keys to form 2-layer btree + { + for (int i = 0; i < 7; ++i) { + // sprintf(&key[count - 2], "%c", i); + key[count - 2] = '0' + i; + + ret = tdbTbInsert(pDb, key, count, NULL, NULL, txn); + GTEST_ASSERT_EQ(ret, 0); + } + } + /* + // delete one interior key + { + sprintf(&key[count - 2], "%c", 2); + key[count - 2] = '0' + 2; + + ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + GTEST_ASSERT_EQ(ret, 0); + } + */ + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); + + system("ls -l ./tdb"); +} diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index bedd14353fa94379416d2c1d0f44421e3c3b450c..445c24159f5b3a3906d1adc9100285b6db508c0b 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include "osEnv.h" #include "tfsInt.h" static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg); @@ -113,6 +114,48 @@ SDiskSize tfsGetSize(STfs *pTfs) { return size; } +bool tfsDiskSpaceAvailable(STfs *pTfs, int32_t level) { + if (level < 0 || level >= pTfs->nlevel) { + return false; + } + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + for (int32_t id = 0; id < pTier->ndisk; id++) { + SDiskID diskId = {.level = level, .id = id}; + STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); + if (pDisk == NULL) { + return false; + } + if (pDisk->size.avail <= 0) { + fError("tfs disk space unavailable. level:%d, disk:%d, path:%s", level, id, pDisk->path); + return false; + } + } + return true; +} + +bool tfsDiskSpaceSufficient(STfs *pTfs, int32_t level, int32_t disk) { + if (level < 0 || level >= pTfs->nlevel) { + return false; + } + + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + if (disk < 0 || disk >= pTier->ndisk) { + return false; + } + SDiskID diskId = {.level = level, .id = disk}; + STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); + return pDisk->size.avail >= tsDataSpace.reserved; +} + +int32_t tfsGetDisksAtLevel(STfs *pTfs, int32_t level) { + if (level < 0 || level >= pTfs->nlevel) { + return 0; + } + + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + return pTier->ndisk; +} + int32_t tfsGetLevel(STfs *pTfs) { return pTfs->nlevel; } int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) { @@ -227,6 +270,9 @@ int32_t tfsMkdirAt(STfs *pTfs, const char *rname, SDiskID diskId) { STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); char aname[TMPNAME_LEN]; + if (pDisk == NULL) { + return -1; + } snprintf(aname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname); if (taosMkDir(aname) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -269,6 +315,20 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) { return 0; } +int32_t tfsMkdirRecur(STfs *pTfs, const char *rname) { + for (int32_t level = 0; level < pTfs->nlevel; level++) { + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + for (int32_t id = 0; id < pTier->ndisk; id++) { + SDiskID did = {.id = id, .level = level}; + if (tfsMkdirRecurAt(pTfs, rname, did) < 0) { + return -1; + } + } + } + + return 0; +} + int32_t tfsMkdir(STfs *pTfs, const char *rname) { for (int32_t level = 0; level < pTfs->nlevel; level++) { STfsTier *pTier = TFS_TIER_AT(pTfs, level); @@ -311,25 +371,60 @@ int32_t tfsRmdir(STfs *pTfs, const char *rname) { return 0; } -int32_t tfsRename(STfs *pTfs, const char *orname, const char *nrname) { +static int32_t tfsRenameAt(STfs *pTfs, SDiskID diskId, const char *orname, const char *nrname) { char oaname[TMPNAME_LEN] = "\0"; char naname[TMPNAME_LEN] = "\0"; - for (int32_t level = 0; level < pTfs->nlevel; level++) { + int32_t level = diskId.level; + int32_t id = diskId.id; + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + STfsDisk *pDisk = pTier->disks[id]; + snprintf(oaname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, orname); + snprintf(naname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, nrname); + + if (taosRenameFile(oaname, naname) != 0 && errno != ENOENT) { + terrno = TAOS_SYSTEM_ERROR(errno); + fError("failed to rename %s to %s since %s", oaname, naname, terrstr()); + return -1; + } + + return 0; +} + +int32_t tfsRename(STfs *pTfs, int32_t diskPrimary, const char *orname, const char *nrname) { + for (int32_t level = pTfs->nlevel - 1; level >= 0; level--) { STfsTier *pTier = TFS_TIER_AT(pTfs, level); - for (int32_t id = 0; id < pTier->ndisk; id++) { - STfsDisk *pDisk = pTier->disks[id]; - snprintf(oaname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, orname); - snprintf(naname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, nrname); - if (taosRenameFile(oaname, naname) != 0 && errno != ENOENT) { - terrno = TAOS_SYSTEM_ERROR(errno); - fError("failed to rename %s to %s since %s", oaname, naname, terrstr()); + for (int32_t id = pTier->ndisk - 1; id >= 0; id--) { + if (level == 0 && id == diskPrimary) { + continue; + } + + SDiskID diskId = {.level = level, .id = id}; + if (tfsRenameAt(pTfs, diskId, orname, nrname)) { return -1; } } } - return 0; + SDiskID diskId = {.level = 0, .id = diskPrimary}; + return tfsRenameAt(pTfs, diskId, orname, nrname); +} + +int32_t tfsSearch(STfs *pTfs, int32_t level, const char *fname) { + if (level < 0 || level >= pTfs->nlevel) { + return -1; + } + char path[TMPNAME_LEN] = {0}; + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + + for (int32_t id = 0; id < pTier->ndisk; id++) { + STfsDisk *pDisk = pTier->disks[id]; + snprintf(path, TMPNAME_LEN - 1, "%s%s%s", pDisk->path, TD_DIRSEP, fname); + if (taosCheckExistFile(path)) { + return id; + } + } + return -1; } STfsDir *tfsOpendir(STfs *pTfs, const char *rname) { diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index df37630fd784e0e561b2675202a9a714370865cc..9bbf6bc729f38141b4ef38ba60af90bc87402654 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -156,7 +156,7 @@ TEST_F(TfsTest, 03_Dir) { EXPECT_NE(taosDirExist(ap4), 1); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0); EXPECT_EQ(taosDirExist(ap4), 1); - EXPECT_EQ(tfsRename(pTfs, p44, p45), 0); + EXPECT_EQ(tfsRename(pTfs, 0, p44, p45), 0); EXPECT_EQ(tfsRmdir(pTfs, p4), 0); EXPECT_NE(taosDirExist(ap4), 1); @@ -609,7 +609,7 @@ TEST_F(TfsTest, 05_MultiDisk) { EXPECT_NE(taosDirExist(_ap22), 1); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0); EXPECT_EQ(taosDirExist(_ap22), 1); - EXPECT_EQ(tfsRename(pTfs, p44, p45), 0); + EXPECT_EQ(tfsRename(pTfs, 0, p44, p45), 0); EXPECT_EQ(tfsRmdir(pTfs, p4), 0); EXPECT_NE(taosDirExist(_ap22), 1); } @@ -721,4 +721,4 @@ TEST_F(TfsTest, 05_MultiDisk) { } tfsClose(pTfs); -} \ No newline at end of file +} diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index a2c486767f931ac7644217dba24d9cab55b48c7e..a6b7a20f7658043033f8360176b5ae1e8cafcd45 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -154,6 +154,7 @@ typedef struct { #pragma pack(push, 1) +#define TRANS_VER 2 typedef struct { char version : 4; // RPC version char comp : 2; // compression algorithm, 0:no compression 1:lz4 @@ -166,6 +167,7 @@ typedef struct { uint64_t timestamp; char user[TSDB_UNI_LEN]; + int32_t compatibilityVer; uint32_t magicNum; STraceId traceId; uint64_t ahandle; // ahandle assigned by client @@ -291,7 +293,7 @@ bool transReadComplete(SConnBuffer* connBuf); int transResetBuffer(SConnBuffer* connBuf); int transDumpFromBuffer(SConnBuffer* connBuf, char** buf); -int transSetConnOption(uv_tcp_t* stream); +int transSetConnOption(uv_tcp_t* stream, int keepalive); void transRefSrvHandle(void* handle); void transUnrefSrvHandle(void* handle); diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 8ea0064d4485c031a49b39bad70fd41e0aec7a85..cc2c0d4e845fd0f970f15ae7ce049024579d07c7 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -46,14 +46,14 @@ typedef struct { int8_t connType; char label[TSDB_LABEL_LEN]; char user[TSDB_UNI_LEN]; // meter ID - - int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size - int8_t encryption; // encrypt or not + int32_t compatibilityVer; + int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size + int8_t encryption; // encrypt or not int32_t retryMinInterval; // retry init interval int32_t retryStepFactor; // retry interval factor int32_t retryMaxInterval; // retry max interval - int32_t retryMaxTimouet; + int32_t retryMaxTimeout; int32_t failFastThreshold; int32_t failFastInterval; diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 04b546b36a48bd1df071c7b077d82c57c2ddea8a..c483d82027ae971da2644fb5c2ef8d2ee1f94c6c 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -391,7 +391,13 @@ static void httpHandleReq(SHttpMsg* msg) { // set up timeout to avoid stuck; int32_t fd = taosCreateSocketWithTimeout(5); - int ret = uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); + if (fd < 0) { + tError("http-report failed to open socket, dst:%s:%d", cli->addr, cli->port); + taosReleaseRef(httpRefMgt, httpRef); + destroyHttpClient(cli); + return; + } + int ret = uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); if (ret != 0) { tError("http-report failed to open socket, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, cli->port); taosReleaseRef(httpRefMgt, httpRef); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 0771f9198aa5e20fe37b95070781b81d01200351..ed94521df09b52df65985ed11b1e21760512f4e0 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -50,11 +50,12 @@ void* rpcOpen(const SRpcInit* pInit) { } pRpc->encryption = pInit->encryption; + pRpc->compatibilityVer = pInit->compatibilityVer; pRpc->retryMinInterval = pInit->retryMinInterval; // retry init interval pRpc->retryStepFactor = pInit->retryStepFactor; pRpc->retryMaxInterval = pInit->retryMaxInterval; - pRpc->retryMaxTimouet = pInit->retryMaxTimouet; + pRpc->retryMaxTimeout = pInit->retryMaxTimeout; pRpc->failFastThreshold = pInit->failFastThreshold; pRpc->failFastInterval = pInit->failFastInterval; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index ea35f1cfe5e806d3e79f03c97aafb13350742d5c..cfdc5b5e8bd8eff783af0001e38ca093ea2fe582 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -73,7 +73,7 @@ typedef struct SCliConn { SDelayTask* task; - char* ip; + char* dstAddr; char src[32]; char dst[32]; @@ -196,6 +196,7 @@ static FORCE_INLINE int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* resp); static FORCE_INLINE uint32_t cliGetIpFromFqdnCache(SHashObj* cache, char* fqdn); static FORCE_INLINE void cliUpdateFqdnCache(SHashObj* cache, char* fqdn); +static FORCE_INLINE void cliMayUpdateFqdnCache(SHashObj* cache, char* dst); // process data read from server, add decompress etc later static void cliHandleResp(SCliConn* conn); // handle except about conn @@ -391,6 +392,7 @@ void cliHandleResp(SCliConn* conn) { transMsg.info.ahandle = NULL; transMsg.info.traceId = pHead->traceId; transMsg.info.hasEpSet = pHead->hasEpSet; + transMsg.info.cliVer = htonl(pHead->compatibilityVer); SCliMsg* pMsg = NULL; STransConnCtx* pCtx = NULL; @@ -488,6 +490,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { transMsg.code = code == -1 ? (pConn->broken ? TSDB_CODE_RPC_BROKEN_LINK : TSDB_CODE_RPC_NETWORK_UNAVAIL) : code; transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; transMsg.info.ahandle = NULL; + transMsg.info.cliVer = pTransInst->compatibilityVer; if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { transMsg.info.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType); @@ -541,6 +544,7 @@ void cliConnTimeout(uv_timer_t* handle) { taosArrayPush(pThrd->timerList, &conn->timer); conn->timer = NULL; + cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr); cliHandleFastFail(conn, UV_ECANCELED); } void cliReadTimeoutCb(uv_timer_t* handle) { @@ -717,7 +721,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { cliDestroyConnMsgs(conn, false); if (conn->list == NULL) { - conn->list = taosHashGet((SHashObj*)pool, conn->ip, strlen(conn->ip) + 1); + conn->list = taosHashGet((SHashObj*)pool, conn->dstAddr, strlen(conn->dstAddr) + 1); } SConnList* pList = conn->list; @@ -876,7 +880,7 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { connList->list->numOfConn--; connList->size--; } else { - SConnList* connList = taosHashGet((SHashObj*)pThrd->pool, conn->ip, strlen(conn->ip) + 1); + SConnList* connList = taosHashGet((SHashObj*)pThrd->pool, conn->dstAddr, strlen(conn->dstAddr) + 1); if (connList != NULL) connList->list->numOfConn--; } conn->list = NULL; @@ -921,7 +925,7 @@ static void cliDestroy(uv_handle_t* handle) { transReleaseExHandle(transGetRefMgt(), conn->refId); transRemoveExHandle(transGetRefMgt(), conn->refId); - taosMemoryFree(conn->ip); + taosMemoryFree(conn->dstAddr); taosMemoryFree(conn->stream); cliDestroyConnMsgs(conn, true); @@ -966,7 +970,7 @@ static void cliSendCb(uv_write_t* req, int status) { } if (status == 0) { - tTrace("%s conn %p data already was written out", CONN_GET_INST_LABEL(pConn), pConn); + tDebug("%s conn %p data already was written out", CONN_GET_INST_LABEL(pConn), pConn); } else { if (!uv_is_closing((uv_handle_t*)&pConn->stream)) { tError("%s conn %p failed to write:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_err_name(status)); @@ -984,11 +988,10 @@ void cliSendBatch(SCliConn* pConn) { SCliThrd* pThrd = pConn->hostThrd; STrans* pTransInst = pThrd->pTransInst; - SCliBatch* pBatch = pConn->pBatch; - SCliBatchList* pList = pBatch->pList; - pList->connCnt += 1; + SCliBatch* pBatch = pConn->pBatch; + int32_t wLen = pBatch->wLen; - int32_t wLen = pBatch->wLen; + pBatch->pList->connCnt += 1; uv_buf_t* wb = taosMemoryCalloc(wLen, sizeof(uv_buf_t)); int i = 0; @@ -1018,6 +1021,8 @@ void cliSendBatch(SCliConn* pConn) { memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user)); pHead->traceId = pMsg->info.traceId; pHead->magicNum = htonl(TRANS_MAGIC_NUM); + pHead->version = TRANS_VER; + pHead->compatibilityVer = htonl(pTransInst->compatibilityVer); } pHead->timestamp = taosHton64(taosGetTimestampUs()); @@ -1074,6 +1079,8 @@ void cliSend(SCliConn* pConn) { memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user)); pHead->traceId = pMsg->info.traceId; pHead->magicNum = htonl(TRANS_MAGIC_NUM); + pHead->version = TRANS_VER; + pHead->compatibilityVer = htonl(pTransInst->compatibilityVer); } pHead->timestamp = taosHton64(taosGetTimestampUs()); @@ -1163,7 +1170,7 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { if (conn == NULL) { conn = cliCreateConn(pThrd); conn->pBatch = pBatch; - conn->ip = taosStrdup(pList->dst); + conn->dstAddr = taosStrdup(pList->dst); uint32_t ipaddr = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, pList->ip); if (ipaddr == 0xffffffff) { @@ -1195,7 +1202,7 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { cliHandleFastFail(conn, -1); return; } - ret = transSetConnOption((uv_tcp_t*)conn->stream); + ret = transSetConnOption((uv_tcp_t*)conn->stream, 20); if (ret != 0) { tError("%s conn %p failed to set socket opt, reason:%s", transLabel(pTransInst), conn, uv_err_name(ret)); cliHandleFastFail(conn, -1); @@ -1208,6 +1215,8 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { conn->timer->data = NULL; taosArrayPush(pThrd->timerList, &conn->timer); conn->timer = NULL; + + cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr); cliHandleFastFail(conn, -1); return; } @@ -1266,11 +1275,11 @@ static void cliHandleFastFail(SCliConn* pConn, int status) { STraceId* trace = &pMsg->msg.info.traceId; tGError("%s msg %s failed to send, conn %p failed to connect to %s, reason: %s", CONN_GET_INST_LABEL(pConn), - TMSG_INFO(pMsg->msg.msgType), pConn, pConn->ip, uv_strerror(status)); + TMSG_INFO(pMsg->msg.msgType), pConn, pConn->dstAddr, uv_strerror(status)); if (pMsg != NULL && REQUEST_NO_RESP(&pMsg->msg) && (pTransInst->failFastFp != NULL && pTransInst->failFastFp(pMsg->msg.msgType))) { - SFailFastItem* item = taosHashGet(pThrd->failFastCache, pConn->ip, strlen(pConn->ip) + 1); + SFailFastItem* item = taosHashGet(pThrd->failFastCache, pConn->dstAddr, strlen(pConn->dstAddr) + 1); int64_t cTimestamp = taosGetTimestampMs(); if (item != NULL) { int32_t elapse = cTimestamp - item->timestamp; @@ -1282,12 +1291,12 @@ static void cliHandleFastFail(SCliConn* pConn, int status) { } } else { SFailFastItem item = {.count = 1, .timestamp = cTimestamp}; - taosHashPut(pThrd->failFastCache, pConn->ip, strlen(pConn->ip) + 1, &item, sizeof(SFailFastItem)); + taosHashPut(pThrd->failFastCache, pConn->dstAddr, strlen(pConn->dstAddr) + 1, &item, sizeof(SFailFastItem)); } } } else { tError("%s batch msg failed to send, conn %p failed to connect to %s, reason: %s", CONN_GET_INST_LABEL(pConn), - pConn, pConn->ip, uv_strerror(status)); + pConn, pConn->dstAddr, uv_strerror(status)); cliDestroyBatch(pConn->pBatch); pConn->pBatch = NULL; } @@ -1309,6 +1318,7 @@ void cliConnCb(uv_connect_t* req, int status) { } if (status != 0) { + cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, pConn->dstAddr); if (timeout == false) { cliHandleFastFail(pConn, status); } else if (timeout == true) { @@ -1346,6 +1356,7 @@ static void doNotifyApp(SCliMsg* pMsg, SCliThrd* pThrd) { transMsg.info.ahandle = pMsg->ctx->ahandle; transMsg.info.traceId = pMsg->msg.info.traceId; transMsg.info.hasEpSet = false; + transMsg.info.cliVer = pTransInst->compatibilityVer; if (pCtx->pSem != NULL) { if (pCtx->pRsp == NULL) { } else { @@ -1477,9 +1488,34 @@ static FORCE_INLINE uint32_t cliGetIpFromFqdnCache(SHashObj* cache, char* fqdn) } static FORCE_INLINE void cliUpdateFqdnCache(SHashObj* cache, char* fqdn) { // impl later + uint32_t addr = taosGetIpv4FromFqdn(fqdn); + if (addr != 0xffffffff) { + uint32_t* v = taosHashGet(cache, fqdn, strlen(fqdn) + 1); + if (addr != *v) { + char old[64] = {0}, new[64] = {0}; + tinet_ntoa(old, *v); + tinet_ntoa(new, addr); + tWarn("update ip of fqdn:%s, old: %s, new: %s", fqdn, old, new); + taosHashPut(cache, fqdn, strlen(fqdn) + 1, &addr, sizeof(addr)); + } + } return; } +static void cliMayUpdateFqdnCache(SHashObj* cache, char* dst) { + if (dst == NULL) return; + + int16_t i = 0, len = strlen(dst); + for (i = len - 1; i >= 0; i--) { + if (dst[i] == ':') break; + } + if (i > 0) { + char fqdn[TSDB_FQDN_LEN + 1] = {0}; + memcpy(fqdn, dst, i); + cliUpdateFqdnCache(cache, fqdn); + } +} + static void doFreeTimeoutMsg(void* param) { STaskArg* arg = param; SCliMsg* pMsg = arg->param1; @@ -1527,6 +1563,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { // persist conn already release by server STransMsg resp; cliBuildExceptResp(pMsg, &resp); + // refactorr later + resp.info.cliVer = pTransInst->compatibilityVer; + if (pMsg->type != Release) { pTransInst->cfp(pTransInst->parent, &resp, NULL); } @@ -1551,7 +1590,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { transCtxMerge(&conn->ctx, &pMsg->ctx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); - conn->ip = taosStrdup(addr); + conn->dstAddr = taosStrdup(addr); uint32_t ipaddr = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, fqdn); if (ipaddr == 0xffffffff) { @@ -1569,9 +1608,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { addr.sin_addr.s_addr = ipaddr; addr.sin_port = (uint16_t)htons(port); - tGTrace("%s conn %p try to connect to %s", pTransInst->label, conn, conn->ip); + tGTrace("%s conn %p try to connect to %s", pTransInst->label, conn, conn->dstAddr); pThrd->newConnCount++; - int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT * 4); + int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT * 10); if (fd == -1) { tGError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn, tstrerror(TAOS_SYSTEM_ERROR(errno))); @@ -1585,7 +1624,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { cliHandleExcept(conn); return; } - ret = transSetConnOption((uv_tcp_t*)conn->stream); + ret = transSetConnOption((uv_tcp_t*)conn->stream, tsKeepAliveIdle); if (ret != 0) { tGError("%s conn %p failed to set socket opt, reason:%s", transLabel(pTransInst), conn, uv_err_name(ret)); cliHandleExcept(conn); @@ -1599,6 +1638,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { taosArrayPush(pThrd->timerList, &conn->timer); conn->timer = NULL; + cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr); cliHandleFastFail(conn, ret); return; } @@ -1836,6 +1876,7 @@ void cliIteraConnMsgs(SCliConn* conn) { if (-1 == cliBuildExceptResp(cmsg, &resp)) { continue; } + resp.info.cliVer = pTransInst->compatibilityVer; pTransInst->cfp(pTransInst->parent, &resp, NULL); cmsg->ctx->ahandle = NULL; @@ -2246,7 +2287,7 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pCtx->retryMinInterval = pTransInst->retryMinInterval; pCtx->retryMaxInterval = pTransInst->retryMaxInterval; pCtx->retryStepFactor = pTransInst->retryStepFactor; - pCtx->retryMaxTimeout = pTransInst->retryMaxTimouet; + pCtx->retryMaxTimeout = pTransInst->retryMaxTimeout; pCtx->retryInitTimestamp = taosGetTimestampMs(); pCtx->retryNextInterval = pCtx->retryMinInterval; pCtx->retryStep = 0; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 0dfc7677b315ea03d66500867fe9bedf26c0cec7..5e602b1ea2457536572a02d8b786212c75180880 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -192,7 +192,7 @@ bool transReadComplete(SConnBuffer* connBuf) { memcpy((char*)&head, connBuf->buf, sizeof(head)); int32_t msgLen = (int32_t)htonl(head.msgLen); p->total = msgLen; - p->invalid = TRANS_NOVALID_PACKET(htonl(head.magicNum)); + p->invalid = TRANS_NOVALID_PACKET(htonl(head.magicNum)) || head.version != TRANS_VER; } if (p->total >= p->len) { p->left = p->total - p->len; @@ -203,10 +203,10 @@ bool transReadComplete(SConnBuffer* connBuf) { return (p->left == 0 || p->invalid) ? true : false; } -int transSetConnOption(uv_tcp_t* stream) { +int transSetConnOption(uv_tcp_t* stream, int keepalive) { #if defined(WINDOWS) || defined(DARWIN) #else - uv_tcp_keepalive(stream, 1, 20); + uv_tcp_keepalive(stream, 1, keepalive); #endif return uv_tcp_nodelay(stream, 1); // int ret = uv_tcp_keepalive(stream, 5, 60); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 269c7ecf9b69cc886e897dba5293084ccc3d0f53..c6c412022a001487b9f50939c3c6a0c6b2de6844 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -196,6 +196,7 @@ static bool uvHandleReq(SSvrConn* pConn) { tError("%s conn %p recv invalid packet, failed to decompress", transLabel(pTransInst), pConn); return false; } + tDebug("head version: %d 2", pHead->version); pHead->code = htonl(pHead->code); pHead->msgLen = htonl(pHead->msgLen); @@ -236,17 +237,17 @@ static bool uvHandleReq(SSvrConn* pConn) { if (pConn->status == ConnNormal && pHead->noResp == 0) { transRefSrvHandle(pConn); if (cost >= EXCEPTION_LIMIT_US) { - tGWarn("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus, recv exception", transLabel(pTransInst), - pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, (int)cost); + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus, recv exception", + transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, (int)cost); } else { tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus", transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, (int)cost); } } else { if (cost >= EXCEPTION_LIMIT_US) { - tGWarn("%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus, recv exception", - transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, - transMsg.code, (int)(cost)); + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus, recv exception", + transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, + transMsg.code, (int)(cost)); } else { tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus", transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, @@ -262,6 +263,7 @@ static bool uvHandleReq(SSvrConn* pConn) { transMsg.info.handle = (void*)transAcquireExHandle(transGetRefMgt(), pConn->refId); transMsg.info.refId = pConn->refId; transMsg.info.traceId = pHead->traceId; + transMsg.info.cliVer = htonl(pHead->compatibilityVer); tGTrace("%s handle %p conn:%p translated to app, refId:%" PRIu64, transLabel(pTransInst), transMsg.info.handle, pConn, pConn->refId); @@ -410,6 +412,8 @@ static int uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { pHead->traceId = pMsg->info.traceId; pHead->hasEpSet = pMsg->info.hasEpSet; pHead->magicNum = htonl(TRANS_MAGIC_NUM); + pHead->compatibilityVer = htonl(((STrans*)pConn->pTransInst)->compatibilityVer); + pHead->version = TRANS_VER; // handle invalid drop_task resp, TD-20098 if (pConn->inType == TDMT_SCH_DROP_TASK && pMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { @@ -722,7 +726,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { tError("read error %s", uv_err_name(nread)); } // TODO(log other failure reason) - tWarn("failed to create connect:%p", q); + tWarn("failed to create connect:%p, reason: %s", q, uv_err_name(nread)); taosMemoryFree(buf->base); uv_close((uv_handle_t*)q, NULL); return; @@ -737,10 +741,17 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_pipe_t* pipe = (uv_pipe_t*)q; if (!uv_pipe_pending_count(pipe)) { tError("No pending count"); + uv_close((uv_handle_t*)q, NULL); + return; + } + if (pThrd->quit) { + tWarn("thread already received quit msg, ignore incoming conn"); + + uv_close((uv_handle_t*)q, NULL); return; } - uv_handle_type pending = uv_pipe_pending_type(pipe); + // uv_handle_type pending = uv_pipe_pending_type(pipe); SSvrConn* pConn = createConn(pThrd); @@ -756,7 +767,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_tcp_init(pThrd->loop, pConn->pTcp); pConn->pTcp->data = pConn; - transSetConnOption((uv_tcp_t*)pConn->pTcp); + // transSetConnOption((uv_tcp_t*)pConn->pTcp); if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) { uv_os_fd_t fd; diff --git a/source/libs/transport/test/cliBench.c b/source/libs/transport/test/cliBench.c index aaee162cd754eb1e209fc21d8568fd703c57d303..8a5276b814d8e6e30e2e71e8ab4814ae33919bd7 100644 --- a/source/libs/transport/test/cliBench.c +++ b/source/libs/transport/test/cliBench.c @@ -19,6 +19,7 @@ #include "transLog.h" #include "trpc.h" #include "tutil.h" +#include "tversion.h" typedef struct { int index; @@ -155,7 +156,7 @@ int main(int argc, char *argv[]) { } initLogEnv(); - + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { tError("failed to initialize RPC"); diff --git a/source/libs/transport/test/svrBench.c b/source/libs/transport/test/svrBench.c index 4e2395b17bc2cc65e258d77a5907e56a538634b5..a3fa81662ccf60152b455e7e8761fc93e1eee5ad 100644 --- a/source/libs/transport/test/svrBench.c +++ b/source/libs/transport/test/svrBench.c @@ -13,12 +13,13 @@ * along with this program. If not, see . */ -//#define _DEFAULT_SOURCE +// #define _DEFAULT_SOURCE #include "os.h" #include "tglobal.h" #include "tqueue.h" #include "transLog.h" #include "trpc.h" +#include "tversion.h" int msgSize = 128; int commit = 0; @@ -151,6 +152,8 @@ int main(int argc, char *argv[]) { rpcInit.numOfThreads = 1; rpcInit.cfp = processRequestMsg; rpcInit.idleTime = 2 * 1500; + + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); rpcDebugFlag = 131; for (int i = 1; i < argc; ++i) { @@ -187,7 +190,7 @@ int main(int argc, char *argv[]) { rpcInit.connType = TAOS_CONN_SERVER; initLogEnv(); - + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { tError("failed to start RPC server"); diff --git a/source/libs/transport/test/transUT.cpp b/source/libs/transport/test/transUT.cpp index 88a1e2564f6221c0c88d24d99fc371007b296b5c..2fa94c358f7f26e7bb78915d117bd156718aa675 100644 --- a/source/libs/transport/test/transUT.cpp +++ b/source/libs/transport/test/transUT.cpp @@ -18,10 +18,10 @@ #include "tdatablock.h" #include "tglobal.h" #include "tlog.h" +#include "tmisce.h" #include "transLog.h" #include "trpc.h" -#include "tmisce.h" - +#include "tversion.h" using namespace std; const char *label = "APP"; @@ -54,6 +54,8 @@ class Client { rpcInit_.user = (char *)user; rpcInit_.parent = this; rpcInit_.connType = TAOS_CONN_CLIENT; + + taosVersionStrToInt(version, &(rpcInit_.compatibilityVer)); this->transCli = rpcOpen(&rpcInit_); tsem_init(&this->sem, 0, 0); } @@ -66,6 +68,7 @@ class Client { void Restart(CB cb) { rpcClose(this->transCli); rpcInit_.cfp = cb; + taosVersionStrToInt(version, &(rpcInit_.compatibilityVer)); this->transCli = rpcOpen(&rpcInit_); } void Stop() { @@ -117,6 +120,7 @@ class Server { rpcInit_.cfp = processReq; rpcInit_.user = (char *)user; rpcInit_.connType = TAOS_CONN_SERVER; + taosVersionStrToInt(version, &(rpcInit_.compatibilityVer)); } void Start() { this->transSrv = rpcOpen(&this->rpcInit_); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index a12f8051ba982ed627ed0767b76d344678748ca9..01d23a7e96a8313e0e111ee0a1fe3f361bf9b219 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -47,9 +47,7 @@ static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { } static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { - int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - terrno = TSDB_CODE_SUCCESS; - + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); @@ -74,13 +72,12 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { int64_t capacity = 0; int64_t readSize = 0; char* buf = NULL; - bool firstTrial = pFileInfo->fileSize < fileSize; int64_t offset = TMIN(pFileInfo->fileSize, fileSize); - int64_t offsetForward = offset - stepSize + walCkHeadSz - 1; - int64_t offsetBackward = offset; int64_t retVer = -1; int64_t lastEntryBeginOffset = 0; int64_t lastEntryEndOffset = 0; + int64_t recordLen = 0; + bool forwardStage = false; // check recover size if (2 * tsWalFsyncDataSizeLimit + offset < end) { @@ -91,14 +88,8 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { // search for the valid last WAL entry, e.g. block by block while (1) { - offset = (firstTrial) ? TMIN(fileSize, offsetForward + stepSize - walCkHeadSz + 1) - : TMAX(0, offsetBackward - stepSize + walCkHeadSz - 1); + offset = (lastEntryEndOffset > 0) ? offset : TMAX(0, offset - stepSize + walCkHeadSz - 1); end = TMIN(offset + stepSize, fileSize); - if (firstTrial) { - offsetForward = offset; - } else { - offsetBackward = offset; - } readSize = end - offset; capacity = readSize + sizeof(magic); @@ -129,7 +120,16 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { int64_t pos = 0; SWalCkHead* logContent = NULL; - while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic))) != NULL) { + while (true) { + forwardStage = (lastEntryEndOffset > 0 || offset == 0); + terrno = TSDB_CODE_SUCCESS; + if (forwardStage) { + candidate = (readSize - (haystack - buf)) > 0 ? haystack : NULL; + } else { + candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic)); + } + + if (candidate == NULL) break; pos = candidate - buf; // validate head @@ -137,13 +137,14 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { if (len < walCkHeadSz) { break; } + logContent = (SWalCkHead*)(buf + pos); if (walValidHeadCksum(logContent) != 0) { terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; wWarn("vgId:%d, failed to validate checksum of wal entry header. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, offset + pos, fnameStr); haystack = buf + pos + 1; - if (firstTrial) { + if (forwardStage) { break; } else { continue; @@ -151,9 +152,9 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { } // validate body - int64_t size = walCkHeadSz + logContent->head.bodyLen; - if (len < size) { - int64_t extraSize = size - len; + recordLen = walCkHeadSz + logContent->head.bodyLen; + if (len < recordLen) { + int64_t extraSize = recordLen - len; if (capacity < readSize + extraSize + sizeof(magic)) { capacity += extraSize; void* ptr = taosMemoryRealloc(buf, capacity); @@ -184,7 +185,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, offset + pos, fnameStr); haystack = buf + pos + 1; - if (firstTrial) { + if (forwardStage) { break; } else { continue; @@ -194,21 +195,14 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { // found one retVer = logContent->head.version; lastEntryBeginOffset = offset + pos; - lastEntryEndOffset = offset + pos + sizeof(SWalCkHead) + logContent->head.bodyLen; + lastEntryEndOffset = offset + pos + recordLen; // try next - haystack = buf + pos + 1; + haystack = buf + pos + recordLen; } - if (end == fileSize) firstTrial = false; - if (firstTrial) { - if (terrno == TSDB_CODE_SUCCESS) { - continue; - } else { - firstTrial = false; - } - } - if (retVer >= 0 || offset == 0) break; + offset = (lastEntryEndOffset > 0) ? lastEntryEndOffset : offset; + if (forwardStage && (terrno != TSDB_CODE_SUCCESS || end == fileSize)) break; } if (retVer < 0) { @@ -602,18 +596,18 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { // ftruncate idx file if (offset < fileSize) { if (taosFtruncateFile(pIdxFile, offset) < 0) { - wError("vgId:%d, failed to ftruncate file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, - strerror(errno), offset, fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, failed to ftruncate file since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), + offset, fnameStr); goto _err; } } // rebuild idx file if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) { - wError("vgId:%d, failed to seek file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), - offset, fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, failed to seek file since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), offset, + fnameStr); goto _err; } @@ -625,11 +619,12 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen; if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { - wError("vgId:%d, failed to read wal log head since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), - idxEntry.offset, fLogNameStr); + wError("vgId:%d, failed to read wal log head since %s. index:%" PRId64 ", offset:%" PRId64 ", file:%s", + pWal->cfg.vgId, terrstr(), idxEntry.ver, idxEntry.offset, fLogNameStr); goto _err; } if (taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); goto _err; } @@ -637,6 +632,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { } if (taosFsyncFile(pIdxFile) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); goto _err; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index d575b8687bc77b6d1ed75d8c587b7f0e5f6b56bb..54b9576eb1dae585b4bf2cdbe8bc026b9e050322 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -62,9 +62,6 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { void walCloseReader(SWalReader *pReader) { taosCloseFile(&pReader->pIdxFile); taosCloseFile(&pReader->pLogFile); - /*if (pReader->cond.enableRef) {*/ - /*taosHashRemove(pReader->pWal->pRefHash, &pReader->readerId, sizeof(int64_t));*/ - /*}*/ taosMemoryFreeClear(pReader->pHead); taosMemoryFree(pReader); } @@ -74,22 +71,22 @@ int32_t walNextValidMsg(SWalReader *pReader) { int64_t lastVer = walGetLastVer(pReader->pWal); int64_t committedVer = walGetCommittedVer(pReader->pWal); int64_t appliedVer = walGetAppliedVer(pReader->pWal); - if(appliedVer < committedVer){ // wait apply ver equal to commit ver, otherwise may lost data when consume data [TD-24010] - wDebug("vgId:%d, wal apply ver:%"PRId64" smaller than commit ver:%"PRId64, pReader->pWal->cfg.vgId, appliedVer, committedVer); -// taosMsleep(10); - } -// int64_t endVer = pReader->cond.scanUncommited ? lastVer : committedVer; - int64_t endVer = TMIN(appliedVer, committedVer); wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64 - ", applied index:%" PRId64", end index:%" PRId64, - pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer, endVer); - while (fetchVer <= endVer) { + ", applied index:%" PRId64, + pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer); + if (fetchVer > appliedVer){ + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; + } + while (fetchVer <= appliedVer) { if (walFetchHeadNew(pReader, fetchVer) < 0) { return -1; } - if (pReader->pHead->head.msgType == TDMT_VND_SUBMIT || - (IS_META_MSG(pReader->pHead->head.msgType) && pReader->cond.scanMeta)) { + + int32_t type = pReader->pHead->head.msgType; + if (type == TDMT_VND_SUBMIT || ((type == TDMT_VND_DELETE) && (pReader->cond.deleteMsg == 1)) || + (IS_META_MSG(type) && pReader->cond.scanMeta)) { if (walFetchBodyNew(pReader) < 0) { return -1; } @@ -98,14 +95,24 @@ int32_t walNextValidMsg(SWalReader *pReader) { if (walSkipFetchBodyNew(pReader) < 0) { return -1; } + fetchVer = pReader->curVersion; } } + return -1; } int64_t walReaderGetCurrentVer(const SWalReader *pReader) { return pReader->curVersion; } int64_t walReaderGetValidFirstVer(const SWalReader *pReader) { return walGetFirstVer(pReader->pWal); } +void walReaderSetSkipToVersion(SWalReader *pReader, int64_t ver) { atomic_store_64(&pReader->skipToVersion, ver); } + +// this function is NOT multi-thread safe, and no need to be. +int64_t walReaderGetSkipToVersion(SWalReader *pReader) { + int64_t newVersion = pReader->skipToVersion; + pReader->skipToVersion = 0; + return newVersion; +} void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64_t *ever) { *sver = walGetFirstVer(pReader->pWal); @@ -114,6 +121,17 @@ void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64_t *eve *ever = pReader->cond.scanUncommited ? lastVer : committedVer; } +void walReaderVerifyOffset(SWalReader *pWalReader, STqOffsetVal* pOffset){ + // if offset version is small than first version , let's seek to first version + taosThreadMutexLock(&pWalReader->pWal->mutex); + int64_t firstVer = walGetFirstVer((pWalReader)->pWal); + taosThreadMutexUnlock(&pWalReader->pWal->mutex); + + if (pOffset->version < firstVer){ + pOffset->version = firstVer; + } +} + static int64_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int64_t ver) { int64_t ret = 0; @@ -287,6 +305,7 @@ static int32_t walFetchBodyNew(SWalReader *pReader) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + pReader->pHead = ptr; pReadHead = &pReader->pHead->head; pReader->capacity = pReadHead->bodyLen; @@ -302,19 +321,16 @@ static int32_t walFetchBodyNew(SWalReader *pReader) { pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } -// pRead->curInvalid = 1; return -1; } if (walValidBodyCksum(pReader->pHead) != 0) { wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId, ver); -// pRead->curInvalid = 1; - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } - wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pReader->pWal->cfg.vgId, ver); + wDebug("vgId:%d, index:%" PRId64 " is fetched, type:%d, cursor advance", pReader->pWal->cfg.vgId, ver, pReader->pHead->head.msgType); pReader->curVersion = ver + 1; return 0; } @@ -346,7 +362,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { pRead->pWal->vers.appliedVer); // TODO: valid ver - if (ver > pRead->pWal->vers.appliedVer) { + if (ver > pRead->pWal->vers.commitVer) { return -1; } diff --git a/source/libs/wal/src/walRef.c b/source/libs/wal/src/walRef.c index 6aba66192684e262baddf4215d043a875c15f6fc..b7169dec5387e247284cb95afc11eca452e97741 100644 --- a/source/libs/wal/src/walRef.c +++ b/source/libs/wal/src/walRef.c @@ -63,43 +63,20 @@ int32_t walSetRefVer(SWalRef *pRef, int64_t ver) { return 0; } -SWalRef *walRefFirstVer(SWal *pWal, SWalRef *pRef) { - if (pRef == NULL) { - pRef = walOpenRef(pWal); - if (pRef == NULL) { - return NULL; - } - } +void walRefFirstVer(SWal *pWal, SWalRef *pRef) { taosThreadMutexLock(&pWal->mutex); int64_t ver = walGetFirstVer(pWal); pRef->refVer = ver; taosThreadMutexUnlock(&pWal->mutex); wDebug("vgId:%d, wal ref version %" PRId64 " for first", pWal->cfg.vgId, ver); - - return pRef; } -SWalRef *walRefCommittedVer(SWal *pWal) { - SWalRef *pRef = walOpenRef(pWal); - if (pRef == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } +void walRefLastVer(SWal *pWal, SWalRef *pRef) { taosThreadMutexLock(&pWal->mutex); - - int64_t ver = walGetCommittedVer(pWal); - - wDebug("vgId:%d, wal ref version %" PRId64 " for committed", pWal->cfg.vgId, ver); - + int64_t ver = walGetLastVer(pWal); pRef->refVer = ver; - // bsearch in fileSet - SWalFileInfo tmpInfo; - tmpInfo.firstVer = ver; - SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - ASSERT(pRet != NULL); - // pRef->refFile = pRet->firstVer; taosThreadMutexUnlock(&pWal->mutex); - return pRef; + wDebug("vgId:%d, wal ref version %" PRId64 " for last", pWal->cfg.vgId, ver); } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 9b7b3dfd5006857eb57b5e74edd39a3835fdbd71..ef97bff89600333b06f16a77c809e35d0567db0c 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -473,7 +473,10 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { // check alignment of idx entries int64_t endOffset = taosLSeekFile(pWal->pIdxFile, 0, SEEK_END); if (endOffset < 0) { - wFatal("vgId:%d, failed to seek end of idxfile due to %s. ver:%" PRId64 "", pWal->cfg.vgId, strerror(errno), ver); + wFatal("vgId:%d, failed to seek end of WAL idxfile due to %s. ver:%" PRId64 "", pWal->cfg.vgId, strerror(errno), + ver); + taosMsleep(100); + exit(EXIT_FAILURE); } return 0; } @@ -533,16 +536,20 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy END: // recover in a reverse order if (taosFtruncateFile(pWal->pLogFile, offset) < 0) { - wFatal("vgId:%d, failed to ftruncate logfile to offset:%" PRId64 " during recovery due to %s", pWal->cfg.vgId, - offset, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); + wFatal("vgId:%d, failed to recover WAL logfile from write error since %s, offset:%" PRId64, pWal->cfg.vgId, + terrstr(), offset); + taosMsleep(100); + exit(EXIT_FAILURE); } int64_t idxOffset = (index - pFileInfo->firstVer) * sizeof(SWalIdxEntry); if (taosFtruncateFile(pWal->pIdxFile, idxOffset) < 0) { - wFatal("vgId:%d, failed to ftruncate idxfile to offset:%" PRId64 "during recovery due to %s", pWal->cfg.vgId, - idxOffset, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); + wFatal("vgId:%d, failed to recover WAL idxfile from write error since %s, offset:%" PRId64, pWal->cfg.vgId, + terrstr(), idxOffset); + taosMsleep(100); + exit(EXIT_FAILURE); } return -1; } diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index db066a82b6647462eb3b17e7723411477f7fa1e1..46e28b529db1769d4e47c2694d6a9f19037997bf 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -64,7 +64,7 @@ else() endif() IF (JEMALLOC_ENABLED) - target_link_libraries(os PUBLIC -ljemalloc) + target_link_libraries(os PUBLIC -L${CMAKE_BINARY_DIR}/build/lib -ljemalloc) ENDIF () if(${BUILD_TEST}) diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 7f0e6d1dee22972cc633c78421a3ee1e6b297e36..0fc136c6930374dfaefd41b3345ac1767dc04480 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -95,10 +95,10 @@ void osCleanup() {} bool osLogSpaceAvailable() { return tsLogSpace.size.avail > 0; } -bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; } - bool osTempSpaceAvailable() { return tsTempSpace.size.avail > 0; } +bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; } + bool osLogSpaceSufficient() { return tsLogSpace.size.avail > tsLogSpace.reserved; } bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reserved; } diff --git a/source/os/src/osMath.c b/source/os/src/osMath.c index 0cff0f78a618b5d82f05b55d45e2cc6303a186fd..10d02ab25c65c3ccf8b74c49f04b004f9571eb92 100644 --- a/source/os/src/osMath.c +++ b/source/os/src/osMath.c @@ -25,7 +25,7 @@ int32_t qsortHelper(const void* p1, const void* p2, const void* param) { // todo refactor: 1) move away; 2) use merge sort instead; 3) qsort is not a stable sort actually. void taosSort(void* base, int64_t sz, int64_t width, __compar_fn_t compar) { -#if defined(WINDOWS) || defined(_ALPINE) +#if defined(WINDOWS_STASH) || defined(_ALPINE) void* param = compar; taosqsort(base, sz, width, param, qsortHelper); #else diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 84004ed3c18d1bb4baa218d450e7c825d25cd385..5f73251e3b9dff37395198709269dc75c599f8a9 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -327,17 +327,19 @@ bool getWinVersionReleaseName(char *releaseName, int32_t maxLen) { } #endif -int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { +int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t maxLen) { #ifdef WINDOWS if (!getWinVersionReleaseName(releaseName, maxLen)) { snprintf(releaseName, maxLen, "Windows"); } + if(sName) snprintf(sName, maxLen, "Windows"); return 0; #elif defined(_TD_DARWIN_64) char osversion[32]; size_t osversion_len = sizeof(osversion) - 1; int osversion_name[] = { CTL_KERN, KERN_OSRELEASE }; + if(sName) snprintf(sName, maxLen, "macOS"); if (sysctl(osversion_name, 2, osversion, &osversion_len, NULL, 0) == -1) { return -1; } @@ -357,24 +359,35 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { return 0; #else char line[1024]; + char *dest = NULL; size_t size = 0; int32_t code = -1; + int32_t cnt = 0; TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) return false; + if (pFile == NULL) return code; while ((size = taosGetsFile(pFile, sizeof(line), line)) != -1) { line[size - 1] = '\0'; - if (strncmp(line, "PRETTY_NAME", 11) == 0) { - const char *p = strchr(line, '=') + 1; - if (*p == '"') { - p++; - line[size - 2] = 0; - } - tstrncpy(releaseName, p, maxLen); + if (strncmp(line, "NAME", 4) == 0) { + dest = sName; + } else if (strncmp(line, "PRETTY_NAME", 11) == 0) { + dest = releaseName; code = 0; - break; + } else if (strncmp(line, "VERSION_ID", 10) == 0) { + dest = ver; + } else { + continue; } + if (!dest) continue; + const char *p = strchr(line, '=') + 1; + if (*p == '"') { + p++; + line[size - 2] = 0; + } + tstrncpy(dest, p, maxLen); + + if (++cnt >= 3) break; } taosCloseFile(&pFile); @@ -948,6 +961,18 @@ char *taosGetCmdlineByPID(int pid) { #endif } +int64_t taosGetOsUptime() { +#ifdef WINDOWS +#elif defined(_TD_DARWIN_64) +#else + struct sysinfo info; + if (0 == sysinfo(&info)) { + return (int64_t)info.uptime * 1000; + } +#endif + return 0; +} + void taosSetCoreDump(bool enable) { if (!enable) return; #ifdef WINDOWS diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index 39ba92fdc5e93d45facd85a2015461ad1fd68d8c..4c4e22bdd92846631026a74064f4f4d6a0a14a4f 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -17,6 +17,15 @@ #include #include "os.h" +#ifdef WINDOWS +#define THREAD_PTR_CHECK(p) \ + do { \ + if (!(p) || !(*(p))) return 0; \ + } while (0); +#else +#define THREAD_PTR_CHECK(p) +#endif + int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void *(*start)(void *), void *arg) { return pthread_create(tid, attr, start, arg); } @@ -83,9 +92,13 @@ int32_t taosThreadCondSignal(TdThreadCond *cond) { return pthread_cond_signal(co int32_t taosThreadCondBroadcast(TdThreadCond *cond) { return pthread_cond_broadcast(cond); } -int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex) { return pthread_cond_wait(cond, mutex); } +int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex) { + THREAD_PTR_CHECK(mutex) + return pthread_cond_wait(cond, mutex); +} int32_t taosThreadCondTimedWait(TdThreadCond *cond, TdThreadMutex *mutex, const struct timespec *abstime) { + THREAD_PTR_CHECK(mutex) return pthread_cond_timedwait(cond, mutex, abstime); } @@ -124,24 +137,37 @@ int32_t taosThreadKeyDelete(TdThreadKey key) { return pthread_key_delete(key); } int32_t taosThreadKill(TdThread thread, int32_t sig) { return pthread_kill(thread, sig); } // int32_t taosThreadMutexConsistent(TdThreadMutex* mutex) { +// THREAD_PTR_CHECK(mutex) // return pthread_mutex_consistent(mutex); // } -int32_t taosThreadMutexDestroy(TdThreadMutex *mutex) { return pthread_mutex_destroy(mutex); } +int32_t taosThreadMutexDestroy(TdThreadMutex *mutex) { + THREAD_PTR_CHECK(mutex) + return pthread_mutex_destroy(mutex); +} int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr) { return pthread_mutex_init(mutex, attr); } -int32_t taosThreadMutexLock(TdThreadMutex *mutex) { return pthread_mutex_lock(mutex); } +int32_t taosThreadMutexLock(TdThreadMutex *mutex) { + THREAD_PTR_CHECK(mutex) + return pthread_mutex_lock(mutex); +} // int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime) { // return pthread_mutex_timedlock(mutex, abstime); // } -int32_t taosThreadMutexTryLock(TdThreadMutex *mutex) { return pthread_mutex_trylock(mutex); } +int32_t taosThreadMutexTryLock(TdThreadMutex *mutex) { + THREAD_PTR_CHECK(mutex) + return pthread_mutex_trylock(mutex); +} -int32_t taosThreadMutexUnlock(TdThreadMutex *mutex) { return pthread_mutex_unlock(mutex); } +int32_t taosThreadMutexUnlock(TdThreadMutex *mutex) { + THREAD_PTR_CHECK(mutex) + return pthread_mutex_unlock(mutex); +} int32_t taosThreadMutexAttrDestroy(TdThreadMutexAttr *attr) { return pthread_mutexattr_destroy(attr); } @@ -224,6 +250,7 @@ int32_t taosThreadSetSchedParam(TdThread thread, int32_t policy, const struct sc int32_t taosThreadSetSpecific(TdThreadKey key, const void *value) { return pthread_setspecific(key, value); } int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock) { + THREAD_PTR_CHECK(lock) #ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_destroy((pthread_mutex_t *)lock); #else @@ -242,6 +269,7 @@ int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int32_t pshared) { } int32_t taosThreadSpinLock(TdThreadSpinlock *lock) { + THREAD_PTR_CHECK(lock) #ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_lock((pthread_mutex_t *)lock); #else @@ -250,6 +278,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock *lock) { } int32_t taosThreadSpinTrylock(TdThreadSpinlock *lock) { + THREAD_PTR_CHECK(lock) #ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_trylock((pthread_mutex_t *)lock); #else @@ -258,6 +287,7 @@ int32_t taosThreadSpinTrylock(TdThreadSpinlock *lock) { } int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock) { + THREAD_PTR_CHECK(lock) #ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_unlock((pthread_mutex_t *)lock); #else diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 39d1de043713b83301c8f15a4030345d00536658..05233065fa75375592877eadba143c069759f547 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -367,8 +367,49 @@ int32_t taosGetTimeOfDay(struct timeval *tv) { time_t taosTime(time_t *t) { return time(t); } +/* + * mktime64 - Converts date to seconds. + * Converts Gregorian date to seconds since 1970-01-01 00:00:00. + * Assumes input in normal date format, i.e. 1980-12-31 23:59:59 + * => year=1980, mon=12, day=31, hour=23, min=59, sec=59. + * + * [For the Julian calendar (which was used in Russia before 1917, + * Britain & colonies before 1752, anywhere else before 1582, + * and is still in use by some communities) leave out the + * -year/100+year/400 terms, and add 10.] + * + * This algorithm was first published by Gauss (I think). + * + * A leap second can be indicated by calling this function with sec as + * 60 (allowable under ISO 8601). The leap second is treated the same + * as the following second since they don't exist in UNIX time. + * + * An encoding of midnight at the end of the day as 24:00:00 - ie. midnight + * tomorrow - (allowable under ISO 8601) is supported. + */ +int64_t user_mktime64(const uint32_t year, const uint32_t mon, const uint32_t day, const uint32_t hour, + const uint32_t min, const uint32_t sec, int64_t time_zone) { + uint32_t _mon = mon, _year = year; + + /* 1..12 -> 11,12,1..10 */ + if (0 >= (int32_t)(_mon -= 2)) { + _mon += 12; /* Puts Feb last since it has leap day */ + _year -= 1; + } + + // int64_t _res = (((((int64_t) (_year/4 - _year/100 + _year/400 + 367*_mon/12 + day) + + // _year*365 - 719499)*24 + hour)*60 + min)*60 + sec); + int64_t _res = 367 * ((int64_t)_mon) / 12; + _res += _year / 4 - _year / 100 + _year / 400 + day + ((int64_t)_year) * 365 - 719499; + _res *= 24; + _res = ((_res + hour) * 60 + min) * 60 + sec; + + return _res + time_zone; +} + time_t taosMktime(struct tm *timep) { #ifdef WINDOWS +#if 0 struct tm tm1 = {0}; LARGE_INTEGER t; FILETIME f; @@ -405,6 +446,19 @@ time_t taosMktime(struct tm *timep) { t.QuadPart -= offset.QuadPart; return (time_t)(t.QuadPart / 10000000); +#else + time_t result = mktime(timep); + if (result != -1) { + return result; + } +#ifdef _MSC_VER +#if _MSC_VER >= 1900 + int64_t tz = _timezone; +#endif +#endif + return user_mktime64(timep->tm_year + 1900, timep->tm_mon + 1, timep->tm_mday, timep->tm_hour, timep->tm_min, + timep->tm_sec, tz); +#endif #else return mktime(timep); #endif diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index cd6ad7cdb5b45c35bae19093a2a97b0c0d8e9bdf..4280490c683c9d94fe1b0a200c65db6ed4b7b9fa 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -768,7 +768,7 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8 keyValue[4] = (keyValue[4] == '+' ? '-' : '+'); keyValue[10] = 0; sprintf(winStr, "TZ=%s:00", &(keyValue[1])); - *tsTimezone = taosStr2Int32(&keyValue[4], NULL, 10); + *tsTimezone = -taosStr2Int32(&keyValue[4], NULL, 10); } break; } @@ -789,7 +789,7 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8 indexStr = ppp - pp + 3; } sprintf(&winStr[indexStr], "%c%c%c:%c%c:00", (p[0] == '+' ? '-' : '+'), p[1], p[2], p[3], p[4]); - *tsTimezone = taosStr2Int32(p, NULL, 10); + *tsTimezone = -taosStr2Int32(p, NULL, 10); } else { *tsTimezone = 0; } diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index 1d6542e78cd987e5228c94609cf0dd865a6d6396..a2ccc4de021f90bc83279960d2dacb8959aca445 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -37,7 +37,7 @@ TEST(osTest, osSystem) { const int sysLen = 64; char osSysName[sysLen]; - int ret = taosGetOsReleaseName(osSysName, sysLen); + int ret = taosGetOsReleaseName(osSysName, NULL, NULL, sysLen); printf("os systeme name:%s\n", osSysName); ASSERT_EQ(ret, 0); } diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 8f3bd42a47fad94dc9d63d213f926e13475cb273..9c2ed190c1f7f1f16936e6ef8ebac5289435c290 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -4,6 +4,14 @@ add_library(util STATIC ${UTIL_SRC}) if (DEFINED GRANT_CFG_INCLUDE_DIR) add_definitions(-DGRANTS_CFG) endif() + +IF (${ASSERT_NOT_CORE}) + ADD_DEFINITIONS(-DASSERT_NOT_CORE) + MESSAGE(STATUS "disable assert core") +ELSE () + MESSAGE(STATUS "enable assert core") +ENDIF (${ASSERT_NOT_CORE}) + target_include_directories( util PUBLIC "${TD_SOURCE_DIR}/include/util" @@ -15,10 +23,9 @@ target_link_libraries( util PUBLIC os PUBLIC lz4_static - PUBLIC api cjson + PUBLIC api cjson geos_c ) if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) endif(${BUILD_TEST}) - diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index f8ce680531a92182e0dc7bfea642e2af38f9c6de..f5e15e7436240e612e4dae3263acaa09363a38d1 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -191,7 +191,7 @@ void* taosArrayGet(const SArray* pArray, size_t index) { } if (index >= pArray->size) { - uError("index is out of range, current:%"PRIzu" max:%d", index, pArray->capacity); + uError("index is out of range, current:%" PRIzu " max:%d", index, pArray->capacity); return NULL; } @@ -221,7 +221,7 @@ size_t taosArrayGetSize(const SArray* pArray) { return TARRAY_SIZE(pArray); } -void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { +void* taosArrayInsert(SArray* pArray, size_t index, const void* pData) { if (pArray == NULL || pData == NULL) { return NULL; } @@ -316,6 +316,10 @@ SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) { } SArray* taosArrayDup(const SArray* pSrc, __array_item_dup_fn_t fn) { + if (NULL == pSrc) { + return NULL; + } + if (pSrc->size == 0) { // empty array list return taosArrayInit(8, pSrc->elemSize); } @@ -472,13 +476,13 @@ int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode) { return tlen; } -void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz) { +void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz, int8_t sver) { int32_t sz; buf = taosDecodeFixedI32(buf, &sz); *pArray = taosArrayInit(sz, sizeof(void*)); for (int32_t i = 0; i < sz; i++) { void* data = taosMemoryCalloc(1, dataSz); - buf = decode(buf, data); + buf = decode(buf, data, sver); taosArrayPush(*pArray, &data); } return (void*)buf; @@ -488,7 +492,7 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t // order array void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) { taosqsort(pArray->pData, pArray->size, pArray->elemSize, param, fn); -// taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); + // taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); } void taosArraySwap(SArray* a, SArray* b) { diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index d6f0e898efa4fb32d66880ebb781c0384c9d2fc4..843f9c56dced580f188e2efb01c2f0e20415fc2d 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -225,6 +225,23 @@ int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) { return compareLenPrefixedWStr(pRight, pLeft); } +int32_t compareLenBinaryVal(const void *pLeft, const void *pRight) { + int32_t len1 = varDataLen(pLeft); + int32_t len2 = varDataLen(pRight); + + int32_t minLen = TMIN(len1, len2); + int32_t ret = memcmp(varDataVal(pLeft), varDataVal(pRight), minLen); + if (ret == 0) { + if (len1 == len2) { + return 0; + } else { + return len1 > len2 ? 1 : -1; + } + } else { + return ret > 0 ? 1 : -1; + } +} + // string > number > bool > null // ref: https://dev.mysql.com/doc/refman/8.0/en/json.html#json-comparison int32_t compareJsonVal(const void *pLeft, const void *pRight) { @@ -1284,7 +1301,7 @@ int32_t comparewcsPatternNMatch(const void *pLeft, const void *pRight) { __compar_fn_t getComparFunc(int32_t type, int32_t optr) { __compar_fn_t comparFn = NULL; - if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -1307,7 +1324,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } } - if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -1351,7 +1368,8 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; - case TSDB_DATA_TYPE_BINARY: { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_GEOMETRY: { if (optr == OP_TYPE_MATCH) { comparFn = comparestrRegexMatch; } else if (optr == OP_TYPE_NMATCH) { @@ -1436,6 +1454,7 @@ __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { case TSDB_DATA_TYPE_UBIGINT: return (order == TSDB_ORDER_ASC) ? compareUint64Val : compareUint64ValDesc; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_GEOMETRY: return (order == TSDB_ORDER_ASC) ? compareLenPrefixedStr : compareLenPrefixedStrDesc; case TSDB_DATA_TYPE_NCHAR: return (order == TSDB_ORDER_ASC) ? compareLenPrefixedWStr : compareLenPrefixedWStrDesc; diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 7631da1b568406f03e520bb7153c59aee3e4f343..c31115d14f953a6ef291d8879b76c5bfcb7ca072 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -912,11 +912,11 @@ int32_t tsCompressDoubleImp(const char *const input, const int32_t nelements, ch return opos; } -uint64_t decodeDoubleValue(const char *const input, int32_t *const ipos, uint8_t flag) { +FORCE_INLINE uint64_t decodeDoubleValue(const char *const input, int32_t *const ipos, uint8_t flag) { uint64_t diff = 0ul; - int32_t nbytes = (flag & INT8MASK(3)) + 1; + int32_t nbytes = (flag & 0x7) + 1; for (int32_t i = 0; i < nbytes; i++) { - diff = diff | ((INT64MASK(8) & input[(*ipos)++]) << BITS_PER_BYTE * i); + diff |= (((uint64_t)0xff & input[(*ipos)++]) << BITS_PER_BYTE * i); } int32_t shift_width = (LONG_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); diff <<= shift_width; @@ -936,25 +936,22 @@ int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, uint8_t flags = 0; int32_t ipos = 1; int32_t opos = 0; - uint64_t prev_value = 0; + uint64_t diff = 0; + union { + uint64_t bits; + double real; + } curr; + + curr.bits = 0; for (int32_t i = 0; i < nelements; i++) { if ((i & 0x01) == 0) { flags = input[ipos++]; } - uint8_t flag = flags & INT8MASK(4); + diff = decodeDoubleValue(input, &ipos, flags & 0x0f); flags >>= 4; - - uint64_t diff = decodeDoubleValue(input, &ipos, flag); - union { - uint64_t bits; - double real; - } curr; - - uint64_t predicted = prev_value; - curr.bits = predicted ^ diff; - prev_value = curr.bits; + curr.bits ^= diff; ostream[opos++] = curr.real; } @@ -1388,6 +1385,14 @@ static struct { .endFn = tCompBinaryEnd, .getI64 = NULL, .putI64 = NULL}, + {.type = TSDB_DATA_TYPE_GEOMETRY, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd, + .getI64 = NULL, + .putI64 = NULL}, }; struct SCompressor { diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 288ea6052b580909198788f528871633048e19f1..3681c7a423c82cbd343d3660cdd6432614a6c919 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -380,43 +380,43 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { return 0; } -int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, bool tsc) { - SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .tsc = tsc}; +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope) { + SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .scope = scope}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, bool tsc) { +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval, .tsc = tsc}; + SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval, .scope = scope}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, bool tsc) { +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval, .tsc = tsc}; + SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval, .scope = scope}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, bool tsc) { +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, int8_t scope) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval, .tsc = tsc}; + SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval, .scope = scope}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc) { - SConfigItem item = {.dtype = CFG_DTYPE_STRING, .tsc = tsc}; +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { + SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope}; item.str = taosStrdup(defaultVal); if (item.str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -425,8 +425,8 @@ int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, bo return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc) { - SConfigItem item = {.dtype = CFG_DTYPE_DIR, .tsc = tsc}; +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { + SConfigItem item = {.dtype = CFG_DTYPE_DIR, .scope = scope}; if (cfgCheckAndSetDir(&item, defaultVal) != 0) { return -1; } @@ -434,8 +434,8 @@ int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, bool return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal) { - SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .tsc = 1}; +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { + SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .scope = scope}; if (cfgCheckAndSetLocale(&item, defaultVal) != 0) { return -1; } @@ -443,8 +443,8 @@ int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal) { return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal) { - SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .tsc = 1}; +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { + SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .scope = scope}; if (cfgCheckAndSetCharset(&item, defaultVal) != 0) { return -1; } @@ -452,8 +452,8 @@ int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal) { return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal) { - SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .tsc = 1}; +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { + SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .scope = scope}; if (cfgCheckAndSetTimezone(&item, defaultVal) != 0) { return -1; } @@ -543,6 +543,27 @@ void cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *p *pLen = len; } +void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen) { + int32_t len = 0; + switch (pItem->scope) { + case CFG_SCOPE_SERVER: + len = snprintf(buf, bufSize, "server"); + break; + case CFG_SCOPE_CLIENT: + len = snprintf(buf, bufSize, "client"); + break; + case CFG_SCOPE_BOTH: + len = snprintf(buf, bufSize, "both"); + break; + } + + if (len > bufSize) { + len = bufSize; + } + + *pLen = len; +} + void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { if (dump) { printf(" global config"); @@ -560,7 +581,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { int32_t size = taosArrayGetSize(pCfg->array); for (int32_t i = 0; i < size; ++i) { SConfigItem *pItem = taosArrayGet(pCfg->array, i); - if (tsc && !pItem->tsc) continue; + if (tsc && pItem->scope == CFG_SCOPE_SERVER) continue; if (dump && strcmp(pItem->name, "scriptDir") == 0) continue; if (dump && strcmp(pItem->name, "simDebugFlag") == 0) continue; tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 58c7300bee4cd1b2071665ec2c1e7683022e3690..b0b407e2a5bbe402a8c67330c5f1ae644e6094b3 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -261,8 +261,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_ALTER_OPTION, "Invalid stable alter TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_OPTION_UNCHNAGED, "STable option unchanged") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC,"Field used by topic") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SINGLE_STB_MODE_DB, "Database is single stable mode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SCHEMA_VER, "Invalid schema version while alter stb") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_STABLE_UID_NOT_MATCH, "Invalid stable uid while alter stb") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SCHEMA_VER, "Invalid schema version") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STABLE_UID_NOT_MATCH, "Invalid stable uid") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_CONFLICT_WITH_TSMA, "Field used by tsma") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_IN_CREATING, "Dnode in creating status") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_IN_DROPPING, "Dnode in dropping status") @@ -275,7 +275,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT, "Conflict transaction TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CLOG_IS_NULL, "Transaction commitlog is null") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL, "Unable to establish connection While execute transaction and will continue in the background") TAOS_DEFINE_ERROR(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED, "Last Transaction not finished") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRNAS_SYNC_TIMEOUT, "Sync timeout While execute transaction and will continue in the background") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_SYNC_TIMEOUT, "Sync timeout While execute transaction and will continue in the background") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") // mnode-mq @@ -324,9 +324,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_NOT_DEPLOYED, "Snode not deployed") TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NOT_CATCH_UP, "Mnode didn't catch the leader") TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ALREADY_IS_VOTER, "Mnode already is a leader") TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ONLY_TWO_MNODE, "Only two mnodes exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NO_NEED_RESTORE, "No need to restore on this dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE, "Please use this command when the dnode is offline") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INIT_FAILED, "Vnode init failure") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_EXIST, "Vnode not exist") TAOS_DEFINE_ERROR(TSDB_CODE_VND_ALREADY_EXIST, "Vnode already exist") @@ -342,6 +345,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_QUERY_BUSY, "Query busy") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_CATCH_UP, "Vnode didn't catch up its leader") TAOS_DEFINE_ERROR(TSDB_CODE_VND_ALREADY_IS_VOTER, "Vnode already is a voter") TAOS_DEFINE_ERROR(TSDB_CODE_VND_DIR_ALREADY_EXIST, "Vnode directory already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_META_DATA_UNSAFE_DELETE, "Single replica vnode data will lost permanently after this operation, if you make sure this, please use drop dnode unsafe to execute") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") @@ -379,7 +383,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, "Tag conditon too many TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, "Multiple retrieval of this query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many time window in query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many groups/time window in query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit has reached") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error") @@ -402,6 +406,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR, "Json not support in t TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JSON_IN_GROUP_ERROR, "Json not support in group/partition by") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JOB_NOT_EXIST, "Job not exist") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_QWORKER_QUIT, "Vnode/Qnode is quitting") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR, "Geometry not support in this operator") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, "License expired") @@ -539,7 +544,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TIMELINE_FUNC, "Invalid timeline fu TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_PASSWD, "Invalid password") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ALTER_TABLE, "Invalid alter table statement") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY, "Primary timestamp column cannot be dropped") -TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_MODIFY_COL, "Only binary/nchar column length could be modified, and the length can only be increased, not decreased") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_MODIFY_COL, "Only binary/nchar/geometry column length could be modified, and the length can only be increased, not decreased") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TBNAME, "Invalid tbname pseudo column") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_FUNCTION_NAME, "Invalid function name") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_COMMENT_TOO_LONG, "Comment too long") @@ -551,7 +556,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COL_JSON, "Only tag can be jso TAOS_DEFINE_ERROR(TSDB_CODE_PAR_VALUE_TOO_LONG, "Value too long for column/tag") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DELETE_WHERE, "The DELETE statement must have a definite time window range") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG, "The REDISTRIBUTE VGROUP statement only support 1 to 3 dnodes") -TAOS_DEFINE_ERROR(TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC, "Fill now allowed") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC, "Fill not allowed") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_WINDOW_PC, "Invalid windows pc") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC, "Window not allowed") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_STREAM_NOT_ALLOWED_FUNC, "Stream not allowed") @@ -563,6 +568,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_SMA_INDEX, "Invalid sma index") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_SELECTED_EXPR, "Invalid SELECTed expression") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_GET_META_ERROR, "Fail to get table info") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_UNIQUE_TABLE_ALIAS, "Not unique table/alias") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC, "System table not allowed") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED, "System table not allowed") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error") //planner @@ -613,19 +620,30 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_QTASKINFO_CREATE, "Rsma qtaskinfo crea TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_SCHEMA, "Rsma invalid schema") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_STREAM_STATE_OPEN, "Rsma stream state open") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_STREAM_STATE_COMMIT, "Rsma stream state commit") -TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FS_REF, "Rsma fs ref error") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FS_SYNC, "Rsma fs sync error") -TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FS_UPDATE, "Rsma fs update error") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_RESULT, "Rsma result error") //index TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Index is rebuilding") TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_INVALID_FILE, "Index file is invalid") +//scalar +TAOS_DEFINE_ERROR(TSDB_CODE_SCALAR_CONVERT_ERROR, "Cannot convert to specific type") + //tmq TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NEED_INITIALIZED, "Assignment or poll interface need to be called first") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_SNAPSHOT_ERROR, "Can not operate in snapshot mode") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NO_COMMITTED, "No committed info") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE, "Offset out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_VGID, "VgId does not belong to this consumer") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_TOPIC, "Topic does not belong to this consumer") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED, "Consumer closed") -TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_ERROR, "Consumer error, to see log") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_ERROR, "Consumer error, to see log") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE, "Topic num out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE, "Group num out of range 100") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_SAME_COMMITTED_VALUE, "Same committed value") // stream TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist") @@ -668,7 +686,7 @@ const char* tstrerror(int32_t err) { if ((err & 0x00ff0000) == 0x00ff0000) { int32_t code = err & 0x0000ffff; // strerror can handle any invalid code - // invalid code return Unknown error + // invalid code return Unknown error return strerror(code); } int32_t s = 0; diff --git a/source/util/src/tgeosctx.c b/source/util/src/tgeosctx.c new file mode 100644 index 0000000000000000000000000000000000000000..b4e7e953dfbbcb6632059baf6ff9799ccd22c123 --- /dev/null +++ b/source/util/src/tgeosctx.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdef.h" +#include "tgeosctx.h" + +static threadlocal SGeosContext tlGeosCtx = {0}; + +SGeosContext* getThreadLocalGeosCtx() { + return &tlGeosCtx; +} + +void destroyThreadLocalGeosCtx() { + if (tlGeosCtx.WKTReader) { + GEOSWKTReader_destroy_r(tlGeosCtx.handle, tlGeosCtx.WKTReader); + tlGeosCtx.WKTReader = NULL; + } + + if (tlGeosCtx.WKTWriter) { + GEOSWKTWriter_destroy_r(tlGeosCtx.handle, tlGeosCtx.WKTWriter); + tlGeosCtx.WKTWriter = NULL; + } + + if (tlGeosCtx.WKBReader) { + GEOSWKBReader_destroy_r(tlGeosCtx.handle, tlGeosCtx.WKBReader); + tlGeosCtx.WKBReader = NULL; + } + + if (tlGeosCtx.WKBWriter) { + GEOSWKBWriter_destroy_r(tlGeosCtx.handle, tlGeosCtx.WKBWriter); + tlGeosCtx.WKBWriter = NULL; + } + + if(tlGeosCtx.handle) { + GEOS_finish_r(tlGeosCtx.handle); + tlGeosCtx.handle = NULL; + } +} diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index f9c7eb1f5687440d5156d1999741ef777774aa67..891c09cfb5dc142deca47e16dcf6f6d401e39aa4 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -194,6 +194,7 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: fn = MurmurHash3_32; break; case TSDB_DATA_TYPE_UINT: diff --git a/source/util/src/theap.c b/source/util/src/theap.c index 8c1a1db05709e3ef3eceb3329185d82f0b5485ed..315ddf93676831f41b820e82ca70fb2ac993b644 100644 --- a/source/util/src/theap.c +++ b/source/util/src/theap.c @@ -187,3 +187,178 @@ void heapRemove(Heap* heap, HeapNode* node) { } void heapDequeue(Heap* heap) { heapRemove(heap, heap->min); } + + +struct PriorityQueue { + SArray* container; + pq_comp_fn fn; + FDelete deleteFn; + void* param; +}; +PriorityQueue* createPriorityQueue(pq_comp_fn fn, FDelete deleteFn, void* param) { + PriorityQueue* pq = (PriorityQueue*)taosMemoryCalloc(1, sizeof(PriorityQueue)); + pq->container = taosArrayInit(1, sizeof(PriorityQueueNode)); + pq->fn = fn; + pq->deleteFn = deleteFn; + pq->param = param; + return pq; +} + +void taosPQSetFn(PriorityQueue* pq, pq_comp_fn fn) { + pq->fn = fn; +} + +void destroyPriorityQueue(PriorityQueue* pq) { + if (pq->deleteFn) + taosArrayDestroyP(pq->container, pq->deleteFn); + else + taosArrayDestroy(pq->container); + taosMemoryFree(pq); +} + +static size_t pqParent(size_t i) { return (--i) >> 1; /* (i - 1) / 2 */ } +static size_t pqLeft(size_t i) { return (i << 1) | 1; /* i * 2 + 1 */ } +static size_t pqRight(size_t i) { return (++i) << 1; /* (i + 1) * 2 */} +static void pqSwapPQNode(PriorityQueueNode* a, PriorityQueueNode* b) { + void * tmp = a->data; + a->data = b->data; + b->data = tmp; +} + +#define pqContainerGetEle(pq, i) ((PriorityQueueNode*)taosArrayGet((pq)->container, (i))) +#define pqContainerSize(pq) (taosArrayGetSize((pq)->container)) + +size_t taosPQSize(PriorityQueue* pq) { return pqContainerSize(pq); } + +static PriorityQueueNode* pqHeapify(PriorityQueue* pq, size_t from, size_t last) { + size_t largest = from; + do { + from = largest; + size_t l = pqLeft(from); + size_t r = pqRight(from); + if (l < last && pq->fn(pqContainerGetEle(pq, from)->data, pqContainerGetEle(pq, l)->data, pq->param)) { + largest = l; + } + if (r < last && pq->fn(pqContainerGetEle(pq, largest)->data, pqContainerGetEle(pq, r)->data, pq->param)) { + largest = r; + } + if (largest != from) { + pqSwapPQNode(pqContainerGetEle(pq, from), pqContainerGetEle(pq, largest)); + } + } while (largest != from); + return pqContainerGetEle(pq, largest); +} + +static void pqBuildHeap(PriorityQueue* pq) { + if (pqContainerSize(pq) > 1) { + for (size_t i = pqContainerSize(pq) - 1; i > 0; --i) { + pqHeapify(pq, i, pqContainerSize(pq)); + } + pqHeapify(pq, 0, pqContainerSize(pq)); + } +} + +static PriorityQueueNode* pqReverseHeapify(PriorityQueue* pq, size_t i) { + while (i > 0 && !pq->fn(pqContainerGetEle(pq, i)->data, pqContainerGetEle(pq, pqParent(i))->data, pq->param)) { + size_t parentIdx = pqParent(i); + pqSwapPQNode(pqContainerGetEle(pq, i), pqContainerGetEle(pq, parentIdx)); + i = parentIdx; + } + return pqContainerGetEle(pq, i); +} + +static void pqUpdate(PriorityQueue* pq, size_t i) { + if (i == 0 || pq->fn(pqContainerGetEle(pq, i)->data, pqContainerGetEle(pq, pqParent(i))->data, pq->param)) { + // if value in pos i is smaller than parent, heapify down from i to the end + pqHeapify(pq, i, pqContainerSize(pq)); + } else { + // if value in pos i is big than parent, heapify up from i + pqReverseHeapify(pq, i); + } +} + +static void pqRemove(PriorityQueue* pq, size_t i) { + if (i == pqContainerSize(pq) - 1) { + taosArrayPop(pq->container); + return; + } + + taosArraySet(pq->container, i, taosArrayGet(pq->container, pqContainerSize(pq) - 1)); + taosArrayPop(pq->container); + pqUpdate(pq, i); +} + +PriorityQueueNode* taosPQTop(PriorityQueue* pq) { + return pqContainerGetEle(pq, 0); +} + +PriorityQueueNode* taosPQPush(PriorityQueue* pq, const PriorityQueueNode* node) { + taosArrayPush(pq->container, node); + return pqReverseHeapify(pq, pqContainerSize(pq) - 1); +} + +void taosPQPop(PriorityQueue* pq) { + PriorityQueueNode* top = taosPQTop(pq); + if (pq->deleteFn) pq->deleteFn(top->data); + pqRemove(pq, 0); +} + +struct BoundedQueue { + PriorityQueue* queue; + uint32_t maxSize; +}; + +BoundedQueue* createBoundedQueue(uint32_t maxSize, pq_comp_fn fn, FDelete deleteFn, void* param) { + BoundedQueue* q = (BoundedQueue*)taosMemoryCalloc(1, sizeof(BoundedQueue)); + q->queue = createPriorityQueue(fn, deleteFn, param); + taosArrayEnsureCap(q->queue->container, maxSize + 1); + q->maxSize = maxSize; + return q; +} + +void taosBQSetFn(BoundedQueue* q, pq_comp_fn fn) { + taosPQSetFn(q->queue, fn); +} + +void destroyBoundedQueue(BoundedQueue* q) { + if (!q) return; + destroyPriorityQueue(q->queue); + taosMemoryFree(q); +} + +PriorityQueueNode* taosBQPush(BoundedQueue* q, PriorityQueueNode* n) { + if (pqContainerSize(q->queue) == q->maxSize + 1) { + PriorityQueueNode* top = pqContainerGetEle(q->queue, 0); + if (q->queue->fn(top->data, n->data, q->queue->param)) { + return NULL; + } else { + void* p = top->data; + top->data = n->data; + n->data = p; + if (q->queue->deleteFn) q->queue->deleteFn(n->data); + } + return pqHeapify(q->queue, 0, taosBQSize(q)); + } else { + return taosPQPush(q->queue, n); + } +} + +PriorityQueueNode* taosBQTop(BoundedQueue* q) { + return taosPQTop(q->queue); +} + +void taosBQBuildHeap(BoundedQueue *q) { + pqBuildHeap(q->queue); +} + +size_t taosBQMaxSize(BoundedQueue* q) { + return q->maxSize; +} + +size_t taosBQSize(BoundedQueue* q) { + return taosPQSize(q->queue); +} + +void taosBQPop(BoundedQueue* q) { + taosPQPop(q->queue); +} diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index c8ac15786f5abe1e3ffcb203fb82225a195f7202..de7ad848ed5dd522bbab43bc9a40c921f1ee8d26 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -76,7 +76,11 @@ static int32_t tsDaylightActive; /* Currently in daylight saving time. */ bool tsLogEmbedded = 0; bool tsAsyncLog = true; +#ifdef ASSERT_NOT_CORE +bool tsAssert = false; +#else bool tsAssert = true; +#endif int32_t tsNumOfLogLines = 10000000; int32_t tsLogKeepDays = 0; LogFp tsLogFp = NULL; @@ -253,15 +257,15 @@ static void taosKeepOldLog(char *oldName) { (void)taosRenameFile(oldName, fileName); - if (tsLogKeepDays < 0) { - char compressFileName[LOG_FILE_NAME_LEN + 20]; - snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); - if (taosCompressFile(fileName, compressFileName) == 0) { - (void)taosRemoveFile(fileName); - } + char compressFileName[LOG_FILE_NAME_LEN + 20]; + snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); + if (taosCompressFile(fileName, compressFileName) == 0) { + (void)taosRemoveFile(fileName); } - taosRemoveOldFiles(tsLogDir, TABS(tsLogKeepDays)); + if (tsLogKeepDays > 0) { + taosRemoveOldFiles(tsLogDir, tsLogKeepDays); + } } static void *taosThreadToOpenNewFile(void *param) { @@ -486,24 +490,11 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *buffer, int32_t len) { if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL && osLogSpaceAvailable()) { taosUpdateLogNums(level); -#if 0 - // DEBUG_FATAL and DEBUG_ERROR are duplicated - // fsync will cause thread blocking and may also generate log misalignment in case of asyncLog - if (tsAsyncLog && level != DEBUG_FATAL) { - taosPushLogBuffer(tsLogObj.logHandle, buffer, len); - } else { - taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); - if (level == DEBUG_FATAL) { - taosFsyncFile(tsLogObj.logHandle->pFile); - } - } -#else if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); } -#endif if (tsLogObj.maxLines > 0) { atomic_add_fetch_32(&tsLogObj.lines, 1); @@ -556,6 +547,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons len += vsnprintf(buffer + len, LOG_MAX_LINE_DUMP_BUFFER_SIZE - 2 - len, format, argpointer); va_end(argpointer); + len = len > LOG_MAX_LINE_DUMP_BUFFER_SIZE - 2 ? LOG_MAX_LINE_DUMP_BUFFER_SIZE - 2 : len; buffer[len++] = '\n'; buffer[len] = 0; @@ -664,7 +656,7 @@ static int32_t taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t msg int32_t end = 0; int32_t remainSize = 0; static int64_t lostLine = 0; - char tmpBuf[128] = {0}; + char tmpBuf[128]; int32_t tmpBufLen = 0; if (pLogBuf == NULL || pLogBuf->stop) return -1; diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index c476baa7908172458cb5ffba718cd55fca62772d..f85ab0ecad01e8deb6d99315469e2133b5fc8282 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -71,12 +71,12 @@ int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, return 0; } -void tMergeTreeDestroy(SMultiwayMergeTreeInfo* pTree) { - if (pTree == NULL) { +void tMergeTreeDestroy(SMultiwayMergeTreeInfo** pTree) { + if (pTree == NULL || *pTree == NULL) { return; } - taosMemoryFreeClear(pTree); + taosMemoryFreeClear(*pTree); } void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) { diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index e182800d9c6219c0f0326a39319a3ddf67ae54b4..3de159797fc6027c3553e5b46c8f74b547de3406 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -39,6 +39,7 @@ enum { struct SLRUEntry { void *value; _taos_lru_deleter_t deleter; + void *ud; SLRUEntry *nextHash; SLRUEntry *next; SLRUEntry *prev; @@ -94,7 +95,7 @@ static void taosLRUEntryFree(SLRUEntry *entry) { ASSERT(entry->refs == 0); if (entry->deleter) { - (*entry->deleter)(entry->keyData, entry->keyLength, entry->value); + (*entry->deleter)(entry->keyData, entry->keyLength, entry->value, entry->ud); } taosMemoryFree(entry); @@ -146,6 +147,25 @@ static void taosLRUEntryTableCleanup(SLRUEntryTable *table) { taosMemoryFree(table->list); } +static int taosLRUEntryTableApplyF(SLRUEntryTable *table, _taos_lru_functor_t functor, void *ud) { + int ret = 0; + uint32_t end = 1 << table->lengthBits; + for (uint32_t i = 0; i < end; ++i) { + SLRUEntry *h = table->list[i]; + while (h) { + SLRUEntry *n = h->nextHash; + ASSERT(TAOS_LRU_ENTRY_IN_CACHE(h)); + ret = functor(h->keyData, h->keyLength, h->value, ud); + if (ret) { + return ret; + } + h = n; + } + } + + return ret; +} + static SLRUEntry **taosLRUEntryTableFindPtr(SLRUEntryTable *table, const void *key, size_t keyLen, uint32_t hash) { SLRUEntry **entry = &table->list[hash >> (32 - table->lengthBits)]; while (*entry && ((*entry)->hash != hash || memcmp(key, (*entry)->keyData, keyLen) != 0)) { @@ -424,7 +444,7 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * static LRUStatus taosLRUCacheShardInsert(SLRUCacheShard *shard, const void *key, size_t keyLen, uint32_t hash, void *value, size_t charge, _taos_lru_deleter_t deleter, LRUHandle **handle, - LRUPriority priority) { + LRUPriority priority, void *ud) { SLRUEntry *e = taosMemoryCalloc(1, sizeof(SLRUEntry) - 1 + keyLen); if (!e) { return TAOS_LRU_STATUS_FAIL; @@ -433,6 +453,7 @@ static LRUStatus taosLRUCacheShardInsert(SLRUCacheShard *shard, const void *key, e->value = value; e->flags = 0; e->deleter = deleter; + e->ud = ud; e->keyLength = keyLen; e->hash = hash; e->refs = 0; @@ -490,6 +511,18 @@ static void taosLRUCacheShardErase(SLRUCacheShard *shard, const void *key, size_ } } +static int taosLRUCacheShardApply(SLRUCacheShard *shard, _taos_lru_functor_t functor, void *ud) { + int ret; + + taosThreadMutexLock(&shard->mutex); + + ret = taosLRUEntryTableApplyF(&shard->table, functor, ud); + + taosThreadMutexUnlock(&shard->mutex); + + return ret; +} + static void taosLRUCacheShardEraseUnrefEntries(SLRUCacheShard *shard) { SArray *lastReferenceList = taosArrayInit(16, POINTER_BYTES); @@ -700,12 +733,12 @@ void taosLRUCacheCleanup(SLRUCache *cache) { } LRUStatus taosLRUCacheInsert(SLRUCache *cache, const void *key, size_t keyLen, void *value, size_t charge, - _taos_lru_deleter_t deleter, LRUHandle **handle, LRUPriority priority) { + _taos_lru_deleter_t deleter, LRUHandle **handle, LRUPriority priority, void *ud) { uint32_t hash = TAOS_LRU_CACHE_SHARD_HASH32(key, keyLen); uint32_t shardIndex = hash & cache->shardedCache.shardMask; return taosLRUCacheShardInsert(&cache->shards[shardIndex], key, keyLen, hash, value, charge, deleter, handle, - priority); + priority, ud); } LRUHandle *taosLRUCacheLookup(SLRUCache *cache, const void *key, size_t keyLen) { @@ -722,6 +755,15 @@ void taosLRUCacheErase(SLRUCache *cache, const void *key, size_t keyLen) { return taosLRUCacheShardErase(&cache->shards[shardIndex], key, keyLen, hash); } +void taosLRUCacheApply(SLRUCache *cache, _taos_lru_functor_t functor, void *ud) { + int numShards = cache->numShards; + for (int i = 0; i < numShards; ++i) { + if (taosLRUCacheShardApply(&cache->shards[i], functor, ud)) { + break; + } + } +} + void taosLRUCacheEraseUnrefEntries(SLRUCache *cache) { int numShards = cache->numShards; for (int i = 0; i < numShards; ++i) { diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index fa8b5d33b7e8f45fe37d658f1ba336d21a43e5b4..8f3a80ded42894e86763631198e220bb5f14410b 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -166,7 +166,7 @@ static char* doFlushBufPage(SDiskbasedBuf* pBuf, SPageInfo* pg) { char* t = NULL; if ((!HAS_DATA_IN_DISK(pg)) || pg->dirty) { void* payload = GET_PAYLOAD_DATA(pg); - t = doCompressData(payload, pBuf->pageSize, &size, pBuf); + t = doCompressData(payload, pBuf->pageSize + sizeof(SFilePage), &size, pBuf); if (size < 0) { uError("failed to compress data when flushing data to disk, %s", pBuf->id); terrno = TSDB_CODE_INVALID_PARA; @@ -482,6 +482,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { SPageInfo** pInfo = (SPageInfo**)((*pi)->pn->data); if (*pInfo != *pi) { + terrno = TSDB_CODE_APP_ERROR; uError("inconsistently data in paged buffer, pInfo:%p, pi:%p, %s", *pInfo, *pi, pBuf->id); return NULL; } diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index e7386d5912dd83c5a76af3c902bcd910b5ffef87..e1000f7bc153176d76c676f81601f2a49f7d0213 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -105,7 +105,7 @@ static void tRBTreeTransplant(SRBTree *pTree, SRBTreeNode *u, SRBTreeNode *v) { v->parent = u->parent; } -static SRBTreeNode *tRBTreeSuccessor(SRBTree *pTree, SRBTreeNode *pNode) { +static SRBTreeNode *tRBTreeSuccessor(const SRBTree *pTree, SRBTreeNode *pNode) { if (pNode->right != pTree->NIL) { pNode = pNode->right; while (pNode->left != pTree->NIL) { @@ -125,7 +125,7 @@ static SRBTreeNode *tRBTreeSuccessor(SRBTree *pTree, SRBTreeNode *pNode) { return pNode; } -static SRBTreeNode *tRBTreePredecessor(SRBTree *pTree, SRBTreeNode *pNode) { +static SRBTreeNode *tRBTreePredecessor(const SRBTree *pTree, SRBTreeNode *pNode) { if (pNode->left != pTree->NIL) { pNode = pNode->left; while (pNode->right != pTree->NIL) { @@ -443,7 +443,7 @@ SRBTreeNode *tRBTreeDropMax(SRBTree *pTree) { return pNode; } -SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode) { +SRBTreeNode *tRBTreeGet(const SRBTree *pTree, const SRBTreeNode *pKeyNode) { SRBTreeNode *pNode = pTree->root; while (pNode != pTree->NIL) { diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 5c1706e405565b6bab81c39eeceacf4fb4164514..8ad7ccc7c21c18d43e0773710d4d4345f997d3ce 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "tsched.h" #include "tdef.h" +#include "tgeosctx.h" #include "tlog.h" #include "ttimer.h" #include "tutil.h" @@ -164,6 +165,8 @@ void *taosProcessSchedQueue(void *scheduler) { (*(msg.tfp))(msg.ahandle, msg.thandle); } + destroyThreadLocalGeosCtx(); + return NULL; } diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 222e0e8a51e019d76f2de71a79423fcfe7b3fa65..6344af523fb06c3168047b088af853e0b18d9c0b 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -376,6 +376,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { fprintf(stdout, "%d: %" PRId64 " \n", id++, *(int64_t *)key); break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_GEOMETRY: fprintf(stdout, "%d: %s \n", id++, key); break; case TSDB_DATA_TYPE_DOUBLE: diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 7e99d6a35cf2eafeb8205e768152b5fb0deb0bd7..cb01fb2d133932ee388973ba21882d6c61361c89 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -113,7 +113,7 @@ typedef struct time_wheel_t { static int32_t tsMaxTmrCtrl = TSDB_MAX_VNODES_PER_DB + 100; -static TdThreadOnce tmrModuleInit = PTHREAD_ONCE_INIT; +static int32_t tmrModuleInit = 0; static TdThreadMutex tmrCtrlMutex; static tmr_ctrl_t* tmrCtrls; static tmr_ctrl_t* unusedTmrCtrl = NULL; @@ -512,11 +512,11 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* han return stopped; } -static void taosTmrModuleInit(void) { +static int32_t taosTmrModuleInit(void) { tmrCtrls = taosMemoryMalloc(sizeof(tmr_ctrl_t) * tsMaxTmrCtrl); if (tmrCtrls == NULL) { tmrError("failed to allocate memory for timer controllers."); - return; + return -1; } memset(&timerMap, 0, sizeof(timerMap)); @@ -535,14 +535,14 @@ static void taosTmrModuleInit(void) { time_wheel_t* wheel = wheels + i; if (taosThreadMutexInit(&wheel->mutex, NULL) != 0) { tmrError("failed to create the mutex for wheel, reason:%s", strerror(errno)); - return; + return -1; } wheel->nextScanAt = now + wheel->resolution; wheel->index = 0; wheel->slots = (tmr_obj_t**)taosMemoryCalloc(wheel->size, sizeof(tmr_obj_t*)); if (wheel->slots == NULL) { tmrError("failed to allocate wheel slots"); - return; + return -1; } timerMap.size += wheel->size; } @@ -551,20 +551,48 @@ static void taosTmrModuleInit(void) { timerMap.slots = (timer_list_t*)taosMemoryCalloc(timerMap.size, sizeof(timer_list_t)); if (timerMap.slots == NULL) { tmrError("failed to allocate hash map"); - return; + return -1; } tmrQhandle = taosInitScheduler(10000, taosTmrThreads, "tmr", NULL); taosInitTimer(taosTimerLoopFunc, MSECONDS_PER_TICK); tmrDebug("timer module is initialized, number of threads: %d", taosTmrThreads); + + return 2; +} + +static int32_t taosTmrInitModule(void) { + if (atomic_load_32(&tmrModuleInit) == 2) { + return 0; + } + + if (atomic_load_32(&tmrModuleInit) < 0) { + return -1; + } + + while (true) { + if (0 == atomic_val_compare_exchange_32(&tmrModuleInit, 0, 1)) { + atomic_store_32(&tmrModuleInit, taosTmrModuleInit()); + } else if (atomic_load_32(&tmrModuleInit) < 0) { + return -1; + } else if (atomic_load_32(&tmrModuleInit) == 2) { + return 0; + } else { + taosMsleep(1); + } + } + + return -1; } void* taosTmrInit(int32_t maxNumOfTmrs, int32_t resolution, int32_t longest, const char* label) { const char* ret = taosMonotonicInit(); tmrDebug("ttimer monotonic clock source:%s", ret); - taosThreadOnce(&tmrModuleInit, taosTmrModuleInit); + if (taosTmrInitModule() < 0) { + return NULL; + } taosThreadMutexLock(&tmrCtrlMutex); tmr_ctrl_t* ctrl = unusedTmrCtrl; @@ -581,6 +609,7 @@ void* taosTmrInit(int32_t maxNumOfTmrs, int32_t resolution, int32_t longest, con } tstrncpy(ctrl->label, label, sizeof(ctrl->label)); + tmrDebug("%s timer controller is initialized, number of timer controllers: %d.", label, numOfTmrCtrl); return ctrl; } @@ -629,8 +658,6 @@ void taosTmrCleanUp(void* handle) { tmrCtrls = NULL; unusedTmrCtrl = NULL; -#if defined(LINUX) - tmrModuleInit = PTHREAD_ONCE_INIT; // to support restart -#endif + atomic_store_32(&tmrModuleInit, 0); } } diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 6d95660103e2c78203e7531af927e8d59ae4c358..6b6878ec83e9da35957c9c6aa5bbc6b37b1a404d 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -351,10 +351,10 @@ int32_t titoa(uint64_t val, size_t radix, char str[]) { int32_t i = 0; uint64_t v = val; - while(v > 0) { + do { buf[i++] = s[v % radix]; v /= radix; - } + } while (v > 0); // reverse order for(int32_t j = 0; j < i; ++j) { diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 6edee27c052d474355812a4b46d53331a5ab6324..57dc60e539bb825f99d9d222bc174c0cdcf888db 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "tworker.h" +#include "tgeosctx.h" #include "taoserror.h" #include "tlog.h" @@ -92,6 +93,8 @@ static void *tQWorkerThreadFp(SQueueWorker *worker) { taosUpdateItemSize(qinfo.queue, 1); } + destroyThreadLocalGeosCtx(); + return NULL; } diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 2e307771b7ed6c5473c4edc0b1947de199f82f10..0bf06e6f44ece7587573f7f0131f62e1c46df62f 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -75,4 +75,12 @@ target_link_libraries(rbtreeTest os util gtest_main) add_test( NAME rbtreeTest COMMAND rbtreeTest -) \ No newline at end of file +) + +# pageBufferTest +add_executable(pageBufferTest "pageBufferTest.cpp") +target_link_libraries(pageBufferTest os util gtest_main) +add_test( + NAME pageBufferTest + COMMAND pageBufferTest +) diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp index 00ed80493073c74822fed86d3cc6ef970f810bcb..50d3656ccd66f49e4875ba775ead48c4c6559c5e 100644 --- a/source/util/test/pageBufferTest.cpp +++ b/source/util/test/pageBufferTest.cpp @@ -157,6 +157,68 @@ void recyclePageTest() { destroyDiskbasedBuf(pBuf); } + +int saveDataToPage(SFilePage* pPg, const char* data, uint32_t len) { + memcpy(pPg->data + pPg->num, data, len); + pPg->num += len; + setBufPageDirty(pPg, true); + return 0; +} + +bool checkBufVarData(SFilePage* pPg, const char* varData, uint32_t varLen) { + const char* start = pPg->data + sizeof(SFilePage); + for (uint32_t i = 0; i < (pPg->num - sizeof(SFilePage)) / varLen; ++i) { + if (0 != strncmp(start + 6 * i + 3, varData, varLen - 3)) { + using namespace std; + cout << "pos: " << sizeof(SFilePage) + 6 * i + 3 << " should be " << varData << " but is: " << start + 6 * i + 3 + << endl; + return false; + } + } + return true; +} + +// SPageInfo.pData: | sizeof(void*) 8 bytes | sizeof(SFilePage) 4 bytes| 4096 bytes | +// ^ +// | +// SFilePage: flush to disk from here +void testFlushAndReadBackBuffer() { + SDiskbasedBuf* pBuf = NULL; + uint32_t totalLen = 4096; + auto code = createDiskbasedBuf(&pBuf, totalLen, totalLen * 2, "1", TD_TMP_DIR_PATH); + int32_t pageId = -1; + auto* pPg = (SFilePage*)getNewBufPage(pBuf, &pageId); + ASSERT_TRUE(pPg != nullptr); + pPg->num = sizeof(SFilePage); + + // save data into page + uint32_t len = 6; // sizeof(SFilePage) + 6 * 682 = 4096 + // nullbitmap(1) + len(2) + AA\0(3) + char* rowData = (char*)taosMemoryCalloc(1, len); + *(uint16_t*)(rowData + 2) = (uint16_t)2; + rowData[3] = 'A'; + rowData[4] = 'A'; + + while (pPg->num + len <= getBufPageSize(pBuf)) { + saveDataToPage(pPg, rowData, len); + } + ASSERT_EQ(pPg->num, totalLen); + ASSERT_TRUE(checkBufVarData(pPg, rowData + 3, len)); + releaseBufPage(pBuf, pPg); + + // flush to disk + int32_t newPgId = -1; + pPg = (SFilePage*)getNewBufPage(pBuf, &newPgId); + releaseBufPage(pBuf, pPg); + pPg = (SFilePage*)getNewBufPage(pBuf, &newPgId); + releaseBufPage(pBuf, pPg); + + // reload it from disk + pPg = (SFilePage*)getBufPage(pBuf, pageId); + ASSERT_TRUE(checkBufVarData(pPg, rowData + 3, len)); + destroyDiskbasedBuf(pBuf); +} + } // namespace TEST(testCase, resultBufferTest) { @@ -164,6 +226,7 @@ TEST(testCase, resultBufferTest) { simpleTest(); writeDownTest(); recyclePageTest(); + testFlushAndReadBackBuffer(); } -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop diff --git a/tests/develop-test/2-query/show_create_db.py b/tests/develop-test/2-query/show_create_db.py index 5574a59ec2e328f10a215adb3061e5626e28b100..af31d7f03aa2aef4f533e3701b5f4bc7200c15ad 100644 --- a/tests/develop-test/2-query/show_create_db.py +++ b/tests/develop-test/2-query/show_create_db.py @@ -1,4 +1,4 @@ -import sys +import sys from util.log import * from util.cases import * from util.sql import * @@ -8,15 +8,15 @@ from math import inf class TDTestCase: def caseDescription(self): ''' - case1: [TD-11204]Difference improvement that can ignore negative - ''' + case1: [TD-11204]Difference improvement that can ignore negative + ''' return - + def init(self, conn, logSql, replicaVer=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), False) self._conn = conn - + def restartTaosd(self, index=1, dbname="db"): tdDnodes.stop(index) tdDnodes.startWithoutSleep(index) @@ -42,17 +42,17 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 1 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0") self.restartTaosd(1, dbname='scd') @@ -60,17 +60,17 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 1 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0") tdSql.execute('drop database scd') diff --git a/tests/develop-test/win-test-file b/tests/develop-test/win-test-file index e4f3bcf56ee0ea3154232d384e005855b60e3c82..b640ef6bfe60d94dd4d3d2430edd71e89af120d8 100644 --- a/tests/develop-test/win-test-file +++ b/tests/develop-test/win-test-file @@ -9,5 +9,6 @@ python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py +python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R diff --git a/tests/docs-examples-test/test_R.sh b/tests/docs-examples-test/test_R.sh new file mode 100755 index 0000000000000000000000000000000000000000..707ea027047579ca4530aaccdf9c0f6051a1a1be --- /dev/null +++ b/tests/docs-examples-test/test_R.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +pgrep taosd || taosd >> /dev/null 2>&1 & +pgrep taosadapter || taosadapter >> /dev/null 2>&1 & + +cd ../../docs/examples/R +wget -N https://repo1.maven.org/maven2/com/taosdata/jdbc/taos-jdbcdriver/3.2.4/taos-jdbcdriver-3.2.4-dist.jar + +jar_path=`find . -name taos-jdbcdriver-*-dist.jar` +echo jar_path=$jar_path +R -f connect_native.r --args $jar_path +# R -f connect_rest.r --args $jar_path # bug 14704 + diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 3e656c03024d715d2c03aee83e0ac428828dc88c..c8b8ce87fd9effdc2e62989674a0f814fe977ff2 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1,427 +1,10 @@ -#Coulumn Define +#Column Define #caseID,rerunTimes,Run with Sanitizer,casePath,caseCommand #NA,NA,y or n,script,./test.sh -f tsim/user/basic.sim #unit-test ,,y,unit-test,bash test.sh -#tsim test -,,y,script,./test.sh -f tsim/user/basic.sim -,,y,script,./test.sh -f tsim/user/password.sim -,,y,script,./test.sh -f tsim/user/privilege_db.sim -,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim -,,y,script,./test.sh -f tsim/user/privilege_topic.sim -,,y,script,./test.sh -f tsim/db/alter_option.sim -,,y,script,./test.sh -f tsim/db/alter_replica_13.sim -,,y,script,./test.sh -f tsim/db/alter_replica_31.sim -,,y,script,./test.sh -f tsim/db/basic1.sim -,,y,script,./test.sh -f tsim/db/basic2.sim -,,y,script,./test.sh -f tsim/db/basic3.sim -,,y,script,./test.sh -f tsim/db/basic4.sim -,,y,script,./test.sh -f tsim/db/basic5.sim -,,y,script,./test.sh -f tsim/db/basic6.sim -,,y,script,./test.sh -f tsim/db/commit.sim -,,y,script,./test.sh -f tsim/db/create_all_options.sim -,,y,script,./test.sh -f tsim/db/delete_reuse1.sim -,,y,script,./test.sh -f tsim/db/delete_reuse2.sim -,,y,script,./test.sh -f tsim/db/delete_reusevnode.sim -,,y,script,./test.sh -f tsim/db/delete_reusevnode2.sim -,,y,script,./test.sh -f tsim/db/delete_writing1.sim -,,y,script,./test.sh -f tsim/db/delete_writing2.sim -,,y,script,./test.sh -f tsim/db/error1.sim -,,y,script,./test.sh -f tsim/db/keep.sim -,,y,script,./test.sh -f tsim/db/len.sim -,,y,script,./test.sh -f tsim/db/repeat.sim -,,y,script,./test.sh -f tsim/db/show_create_db.sim -,,y,script,./test.sh -f tsim/db/show_create_table.sim -,,y,script,./test.sh -f tsim/db/tables.sim -,,y,script,./test.sh -f tsim/db/taosdlog.sim -,,y,script,./test.sh -f tsim/dnode/balance_replica1.sim -,,y,script,./test.sh -f tsim/dnode/balance_replica3.sim -,,y,script,./test.sh -f tsim/dnode/balance1.sim -,,y,script,./test.sh -f tsim/dnode/balance2.sim -,,y,script,./test.sh -f tsim/dnode/balance3.sim -,,y,script,./test.sh -f tsim/dnode/balancex.sim -,,y,script,./test.sh -f tsim/dnode/create_dnode.sim -,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim -,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim -,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim -,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim -,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim -,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim -,,y,script,./test.sh -f tsim/dnode/drop_dnode_force.sim -,,y,script,./test.sh -f tsim/dnode/offline_reason.sim -,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim -,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim -,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim -,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim -,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim -,,y,script,./test.sh -f tsim/dnode/vnode_clean.sim -,,y,script,./test.sh -f tsim/dnode/use_dropped_dnode.sim -,,y,script,./test.sh -f tsim/dnode/split_vgroup_replica1.sim -,,y,script,./test.sh -f tsim/dnode/split_vgroup_replica3.sim -,,y,script,./test.sh -f tsim/import/basic.sim -,,y,script,./test.sh -f tsim/import/commit.sim -,,y,script,./test.sh -f tsim/import/large.sim -,,y,script,./test.sh -f tsim/import/replica1.sim -,,y,script,./test.sh -f tsim/insert/backquote.sim -,,y,script,./test.sh -f tsim/insert/basic.sim -,,y,script,./test.sh -f tsim/insert/basic0.sim -,,y,script,./test.sh -f tsim/insert/basic1.sim -,,y,script,./test.sh -f tsim/insert/basic2.sim -,,y,script,./test.sh -f tsim/insert/commit-merge0.sim -,,y,script,./test.sh -f tsim/insert/insert_drop.sim -,,y,script,./test.sh -f tsim/insert/insert_select.sim -,,y,script,./test.sh -f tsim/insert/null.sim -,,y,script,./test.sh -f tsim/insert/query_block1_file.sim -,,y,script,./test.sh -f tsim/insert/query_block1_memory.sim -,,y,script,./test.sh -f tsim/insert/query_block2_file.sim -,,y,script,./test.sh -f tsim/insert/query_block2_memory.sim -,,y,script,./test.sh -f tsim/insert/query_file_memory.sim -,,y,script,./test.sh -f tsim/insert/query_multi_file.sim -,,y,script,./test.sh -f tsim/insert/tcp.sim -,,y,script,./test.sh -f tsim/insert/update0.sim -,,y,script,./test.sh -f tsim/insert/update1_sort_merge.sim -,,y,script,./test.sh -f tsim/insert/update2.sim -,,y,script,./test.sh -f tsim/parser/alter__for_community_version.sim -,,y,script,./test.sh -f tsim/parser/alter_column.sim -,,y,script,./test.sh -f tsim/parser/alter_stable.sim -,,y,script,./test.sh -f tsim/parser/alter.sim -,,y,script,./test.sh -f tsim/parser/alter1.sim -,,y,script,./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim -,,y,script,./test.sh -f tsim/parser/auto_create_tb.sim -,,y,script,./test.sh -f tsim/parser/between_and.sim -,,y,script,./test.sh -f tsim/parser/binary_escapeCharacter.sim -,,y,script,./test.sh -f tsim/parser/col_arithmetic_operation.sim -,,y,script,./test.sh -f tsim/parser/columnValue_bigint.sim -,,y,script,./test.sh -f tsim/parser/columnValue_bool.sim -,,y,script,./test.sh -f tsim/parser/columnValue_double.sim -,,y,script,./test.sh -f tsim/parser/columnValue_float.sim -,,y,script,./test.sh -f tsim/parser/columnValue_int.sim -,,y,script,./test.sh -f tsim/parser/columnValue_smallint.sim -,,y,script,./test.sh -f tsim/parser/columnValue_tinyint.sim -,,y,script,./test.sh -f tsim/parser/columnValue_unsign.sim -,,y,script,./test.sh -f tsim/parser/commit.sim -,,y,script,./test.sh -f tsim/parser/condition.sim -,,y,script,./test.sh -f tsim/parser/constCol.sim -,,y,script,./test.sh -f tsim/parser/create_db.sim -,,y,script,./test.sh -f tsim/parser/create_mt.sim -,,y,script,./test.sh -f tsim/parser/create_tb_with_tag_name.sim -,,y,script,./test.sh -f tsim/parser/create_tb.sim -,,y,script,./test.sh -f tsim/parser/dbtbnameValidate.sim -,,y,script,./test.sh -f tsim/parser/distinct.sim -,,y,script,./test.sh -f tsim/parser/fill_us.sim -,,y,script,./test.sh -f tsim/parser/fill.sim -,,y,script,./test.sh -f tsim/parser/first_last.sim -,,y,script,./test.sh -f tsim/parser/fill_stb.sim -,,y,script,./test.sh -f tsim/parser/interp.sim -,,y,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim -,,y,script,./test.sh -f tsim/parser/function.sim -,,y,script,./test.sh -f tsim/parser/groupby-basic.sim -,,y,script,./test.sh -f tsim/parser/groupby.sim -,,y,script,./test.sh -f tsim/parser/having_child.sim -,,y,script,./test.sh -f tsim/parser/having.sim -,,y,script,./test.sh -f tsim/parser/import_commit1.sim -,,y,script,./test.sh -f tsim/parser/import_commit2.sim -,,y,script,./test.sh -f tsim/parser/import_commit3.sim -,,y,script,./test.sh -f tsim/parser/import_file.sim -,,y,script,./test.sh -f tsim/parser/import.sim -,,y,script,./test.sh -f tsim/parser/insert_multiTbl.sim -,,y,script,./test.sh -f tsim/parser/insert_tb.sim -,,y,script,./test.sh -f tsim/parser/join_manyblocks.sim -,,y,script,./test.sh -f tsim/parser/join_multitables.sim -,,y,script,./test.sh -f tsim/parser/join_multivnode.sim -,,y,script,./test.sh -f tsim/parser/join.sim -,,y,script,./test.sh -f tsim/parser/last_cache.sim -,,y,script,./test.sh -f tsim/parser/last_groupby.sim -,,y,script,./test.sh -f tsim/parser/lastrow.sim -,,y,script,./test.sh -f tsim/parser/lastrow2.sim -,,y,script,./test.sh -f tsim/parser/like.sim -,,y,script,./test.sh -f tsim/parser/limit.sim -,,y,script,./test.sh -f tsim/parser/limit1.sim -,,y,script,./test.sh -f tsim/parser/mixed_blocks.sim -,,y,script,./test.sh -f tsim/parser/nchar.sim -,,y,script,./test.sh -f tsim/parser/nestquery.sim -,,y,script,./test.sh -f tsim/parser/null_char.sim -,,y,script,./test.sh -f tsim/parser/precision_ns.sim -,,y,script,./test.sh -f tsim/parser/projection_limit_offset.sim -,,y,script,./test.sh -f tsim/parser/regex.sim -,,y,script,./test.sh -f tsim/parser/regressiontest.sim -,,y,script,./test.sh -f tsim/parser/select_across_vnodes.sim -,,y,script,./test.sh -f tsim/parser/select_distinct_tag.sim -,,y,script,./test.sh -f tsim/parser/select_from_cache_disk.sim -,,y,script,./test.sh -f tsim/parser/select_with_tags.sim -,,y,script,./test.sh -f tsim/parser/selectResNum.sim -,,y,script,./test.sh -f tsim/parser/set_tag_vals.sim -,,y,script,./test.sh -f tsim/parser/single_row_in_tb.sim -,,y,script,./test.sh -f tsim/parser/sliding.sim -,,y,script,./test.sh -f tsim/parser/slimit_alter_tags.sim -,,y,script,./test.sh -f tsim/parser/slimit.sim -,,y,script,./test.sh -f tsim/parser/slimit1.sim -,,y,script,./test.sh -f tsim/parser/stableOp.sim -,,y,script,./test.sh -f tsim/parser/tags_dynamically_specifiy.sim -,,y,script,./test.sh -f tsim/parser/tags_filter.sim -,,y,script,./test.sh -f tsim/parser/tbnameIn.sim -,,y,script,./test.sh -f tsim/parser/timestamp.sim -,,y,script,./test.sh -f tsim/parser/top_groupby.sim -,,y,script,./test.sh -f tsim/parser/topbot.sim -,,y,script,./test.sh -f tsim/parser/union.sim -,,y,script,./test.sh -f tsim/parser/union_sysinfo.sim -,,y,script,./test.sh -f tsim/parser/where.sim -,,y,script,./test.sh -f tsim/query/tagLikeFilter.sim -,,y,script,./test.sh -f tsim/query/charScalarFunction.sim -,,y,script,./test.sh -f tsim/query/explain.sim -,,y,script,./test.sh -f tsim/query/interval-offset.sim -,,y,script,./test.sh -f tsim/query/interval.sim -,,y,script,./test.sh -f tsim/query/scalarFunction.sim -,,y,script,./test.sh -f tsim/query/scalarNull.sim -,,y,script,./test.sh -f tsim/query/session.sim -,,y,script,./test.sh -f tsim/query/udf.sim -,,y,script,./test.sh -f tsim/query/udf_with_const.sim -,,y,script,./test.sh -f tsim/query/sys_tbname.sim -,,y,script,./test.sh -f tsim/query/groupby.sim -,,y,script,./test.sh -f tsim/query/event.sim -,,y,script,./test.sh -f tsim/query/forceFill.sim -,,y,script,./test.sh -f tsim/query/emptyTsRange.sim -,,y,script,./test.sh -f tsim/query/partitionby.sim -,,y,script,./test.sh -f tsim/qnode/basic1.sim -,,y,script,./test.sh -f tsim/snode/basic1.sim -,,y,script,./test.sh -f tsim/mnode/basic1.sim -,,y,script,./test.sh -f tsim/mnode/basic2.sim -,,y,script,./test.sh -f tsim/mnode/basic3.sim -,,y,script,./test.sh -f tsim/mnode/basic4.sim -,,y,script,./test.sh -f tsim/mnode/basic5.sim -,,y,script,./test.sh -f tsim/show/basic.sim -,,y,script,./test.sh -f tsim/table/autocreate.sim -,,y,script,./test.sh -f tsim/table/basic1.sim -,,y,script,./test.sh -f tsim/table/basic2.sim -,,y,script,./test.sh -f tsim/table/basic3.sim -,,y,script,./test.sh -f tsim/table/bigint.sim -,,y,script,./test.sh -f tsim/table/binary.sim -,,y,script,./test.sh -f tsim/table/bool.sim -,,y,script,./test.sh -f tsim/table/column_name.sim -,,y,script,./test.sh -f tsim/table/column_num.sim -,,y,script,./test.sh -f tsim/table/column_value.sim -,,y,script,./test.sh -f tsim/table/column2.sim -,,y,script,./test.sh -f tsim/table/createmulti.sim -,,y,script,./test.sh -f tsim/table/date.sim -,,y,script,./test.sh -f tsim/table/db.table.sim -,,y,script,./test.sh -f tsim/table/delete_reuse1.sim -,,y,script,./test.sh -f tsim/table/delete_reuse2.sim -,,y,script,./test.sh -f tsim/table/delete_writing.sim -,,y,script,./test.sh -f tsim/table/describe.sim -,,y,script,./test.sh -f tsim/table/double.sim -,,y,script,./test.sh -f tsim/table/float.sim -,,y,script,./test.sh -f tsim/table/hash.sim -,,y,script,./test.sh -f tsim/table/int.sim -,,y,script,./test.sh -f tsim/table/limit.sim -,,y,script,./test.sh -f tsim/table/smallint.sim -,,y,script,./test.sh -f tsim/table/table_len.sim -,,y,script,./test.sh -f tsim/table/table.sim -,,y,script,./test.sh -f tsim/table/tinyint.sim -,,y,script,./test.sh -f tsim/table/vgroup.sim -,,n,script,./test.sh -f tsim/stream/basic0.sim -g -,,y,script,./test.sh -f tsim/stream/basic1.sim -,,y,script,./test.sh -f tsim/stream/basic2.sim -,,y,script,./test.sh -f tsim/stream/basic3.sim -,,y,script,./test.sh -f tsim/stream/basic4.sim -,,y,script,./test.sh -f tsim/stream/checkStreamSTable1.sim -,,y,script,./test.sh -f tsim/stream/checkStreamSTable.sim -,,y,script,./test.sh -f tsim/stream/deleteInterval.sim -,,y,script,./test.sh -f tsim/stream/deleteSession.sim -,,y,script,./test.sh -f tsim/stream/deleteState.sim -,,y,script,./test.sh -f tsim/stream/distributeInterval0.sim -,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim -,,y,script,./test.sh -f tsim/stream/distributeSession0.sim -,,y,script,./test.sh -f tsim/stream/drop_stream.sim -,,y,script,./test.sh -f tsim/stream/fillHistoryBasic1.sim -,,y,script,./test.sh -f tsim/stream/fillHistoryBasic2.sim -,,y,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim -,,y,script,./test.sh -f tsim/stream/fillIntervalDelete0.sim -,,y,script,./test.sh -f tsim/stream/fillIntervalDelete1.sim -,,y,script,./test.sh -f tsim/stream/fillIntervalLinear.sim -,,y,script,./test.sh -f tsim/stream/fillIntervalPartitionBy.sim -,,y,script,./test.sh -f tsim/stream/fillIntervalPrevNext1.sim -,,y,script,./test.sh -f tsim/stream/fillIntervalPrevNext.sim -,,y,script,./test.sh -f tsim/stream/fillIntervalRange.sim -,,y,script,./test.sh -f tsim/stream/fillIntervalValue.sim -,,y,script,./test.sh -f tsim/stream/ignoreCheckUpdate.sim -,,y,script,./test.sh -f tsim/stream/ignoreExpiredData.sim -,,y,script,./test.sh -f tsim/stream/partitionby1.sim -,,y,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim -,,y,script,./test.sh -f tsim/stream/partitionbyColumnSession.sim -,,y,script,./test.sh -f tsim/stream/partitionbyColumnState.sim -,,y,script,./test.sh -f tsim/stream/partitionby.sim -,,y,script,./test.sh -f tsim/stream/pauseAndResume.sim -,,y,script,./test.sh -f tsim/stream/schedSnode.sim -,,y,script,./test.sh -f tsim/stream/session0.sim -,,y,script,./test.sh -f tsim/stream/session1.sim -,,y,script,./test.sh -f tsim/stream/sliding.sim -,,y,script,./test.sh -f tsim/stream/state0.sim -,,y,script,./test.sh -f tsim/stream/state1.sim -,,y,script,./test.sh -f tsim/stream/triggerInterval0.sim -,,y,script,./test.sh -f tsim/stream/triggerSession0.sim -,,y,script,./test.sh -f tsim/stream/udTableAndTag0.sim -,,y,script,./test.sh -f tsim/stream/udTableAndTag1.sim -,,y,script,./test.sh -f tsim/stream/udTableAndTag2.sim -,,y,script,./test.sh -f tsim/stream/windowClose.sim -,,y,script,./test.sh -f tsim/trans/lossdata1.sim -,,y,script,./test.sh -f tsim/trans/create_db.sim -,,y,script,./test.sh -f tsim/tmq/basic1.sim -,,y,script,./test.sh -f tsim/tmq/basic2.sim -,,y,script,./test.sh -f tsim/tmq/basic3.sim -,,y,script,./test.sh -f tsim/tmq/basic4.sim -,,y,script,./test.sh -f tsim/tmq/basic1Of2Cons.sim -,,y,script,./test.sh -f tsim/tmq/basic2Of2Cons.sim -,,y,script,./test.sh -f tsim/tmq/basic3Of2Cons.sim -,,y,script,./test.sh -f tsim/tmq/basic4Of2Cons.sim -,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim -,,y,script,./test.sh -f tsim/tmq/topic.sim -,,y,script,./test.sh -f tsim/tmq/snapshot.sim -,,y,script,./test.sh -f tsim/tmq/snapshot1.sim -,,y,script,./test.sh -f tsim/stable/alter_comment.sim -,,y,script,./test.sh -f tsim/stable/alter_count.sim -,,y,script,./test.sh -f tsim/stable/alter_import.sim -,,y,script,./test.sh -f tsim/stable/alter_insert1.sim -,,y,script,./test.sh -f tsim/stable/alter_insert2.sim -,,y,script,./test.sh -f tsim/stable/alter_metrics.sim -,,y,script,./test.sh -f tsim/stable/column_add.sim -,,y,script,./test.sh -f tsim/stable/column_drop.sim -,,y,script,./test.sh -f tsim/stable/column_modify.sim -,,y,script,./test.sh -f tsim/stable/disk.sim -,,y,script,./test.sh -f tsim/stable/dnode3.sim -,,y,script,./test.sh -f tsim/stable/metrics.sim -,,y,script,./test.sh -f tsim/stable/refcount.sim -,,y,script,./test.sh -f tsim/stable/tag_add.sim -,,y,script,./test.sh -f tsim/stable/tag_drop.sim -,,y,script,./test.sh -f tsim/stable/tag_filter.sim -,,y,script,./test.sh -f tsim/stable/tag_modify.sim -,,y,script,./test.sh -f tsim/stable/tag_rename.sim -,,y,script,./test.sh -f tsim/stable/values.sim -,,y,script,./test.sh -f tsim/stable/vnode3.sim -,,y,script,./test.sh -f tsim/stable/metrics_idx.sim -,,n,script,./test.sh -f tsim/sma/drop_sma.sim -,,y,script,./test.sh -f tsim/sma/sma_leak.sim -,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim -,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim -,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim -,,n,script,./test.sh -f tsim/valgrind/checkError1.sim -,,n,script,./test.sh -f tsim/valgrind/checkError2.sim -,,n,script,./test.sh -f tsim/valgrind/checkError3.sim -,,n,script,./test.sh -f tsim/valgrind/checkError4.sim -,,n,script,./test.sh -f tsim/valgrind/checkError5.sim -,,n,script,./test.sh -f tsim/valgrind/checkError6.sim -,,n,script,./test.sh -f tsim/valgrind/checkError7.sim -,,n,script,./test.sh -f tsim/valgrind/checkError8.sim -,,n,script,./test.sh -f tsim/valgrind/checkUdf.sim -,,y,script,./test.sh -f tsim/vnode/replica3_basic.sim -,,y,script,./test.sh -f tsim/vnode/replica3_repeat.sim -,,y,script,./test.sh -f tsim/vnode/replica3_vgroup.sim -,,y,script,./test.sh -f tsim/vnode/replica3_many.sim -,,y,script,./test.sh -f tsim/vnode/replica3_import.sim -,,y,script,./test.sh -f tsim/vnode/stable_balance_replica1.sim -,,y,script,./test.sh -f tsim/vnode/stable_dnode2_stop.sim -,,y,script,./test.sh -f tsim/vnode/stable_dnode2.sim -,,y,script,./test.sh -f tsim/vnode/stable_dnode3.sim -,,y,script,./test.sh -f tsim/vnode/stable_replica3_dnode6.sim -,,y,script,./test.sh -f tsim/vnode/stable_replica3_vnode3.sim -,,y,script,./test.sh -f tsim/sync/3Replica1VgElect.sim -,,y,script,./test.sh -f tsim/sync/3Replica5VgElect.sim -,,y,script,./test.sh -f tsim/sync/oneReplica1VgElect.sim -,,y,script,./test.sh -f tsim/sync/oneReplica5VgElect.sim -,,y,script,./test.sh -f tsim/catalog/alterInCurrent.sim -,,y,script,./test.sh -f tsim/scalar/in.sim -,,y,script,./test.sh -f tsim/scalar/scalar.sim -,,y,script,./test.sh -f tsim/scalar/filter.sim -,,y,script,./test.sh -f tsim/scalar/caseWhen.sim -,,y,script,./test.sh -f tsim/scalar/tsConvert.sim -,,y,script,./test.sh -f tsim/alter/cached_schema_after_alter.sim -,,y,script,./test.sh -f tsim/alter/dnode.sim -,,y,script,./test.sh -f tsim/alter/table.sim -,,y,script,./test.sh -f tsim/cache/new_metrics.sim -,,y,script,./test.sh -f tsim/cache/restart_table.sim -,,y,script,./test.sh -f tsim/cache/restart_metrics.sim -,,y,script,./test.sh -f tsim/column/commit.sim -,,y,script,./test.sh -f tsim/column/metrics.sim -,,y,script,./test.sh -f tsim/column/table.sim -,,y,script,./test.sh -f tsim/compress/commitlog.sim -,,y,script,./test.sh -f tsim/compress/compress2.sim -,,y,script,./test.sh -f tsim/compress/compress.sim -,,y,script,./test.sh -f tsim/compress/uncompress.sim -,,y,script,./test.sh -f tsim/compute/avg.sim -,,y,script,./test.sh -f tsim/compute/block_dist.sim -,,y,script,./test.sh -f tsim/compute/bottom.sim -,,y,script,./test.sh -f tsim/compute/count.sim -,,y,script,./test.sh -f tsim/compute/diff.sim -,,y,script,./test.sh -f tsim/compute/diff2.sim -,,y,script,./test.sh -f tsim/compute/first.sim -,,y,script,./test.sh -f tsim/compute/interval.sim -,,y,script,./test.sh -f tsim/compute/last_row.sim -,,y,script,./test.sh -f tsim/compute/last.sim -,,y,script,./test.sh -f tsim/compute/leastsquare.sim -,,y,script,./test.sh -f tsim/compute/max.sim -,,y,script,./test.sh -f tsim/compute/min.sim -,,y,script,./test.sh -f tsim/compute/null.sim -,,y,script,./test.sh -f tsim/compute/percentile.sim -,,y,script,./test.sh -f tsim/compute/stddev.sim -,,y,script,./test.sh -f tsim/compute/sum.sim -,,y,script,./test.sh -f tsim/compute/top.sim -,,y,script,./test.sh -f tsim/field/2.sim -,,y,script,./test.sh -f tsim/field/3.sim -,,y,script,./test.sh -f tsim/field/4.sim -,,y,script,./test.sh -f tsim/field/5.sim -,,y,script,./test.sh -f tsim/field/6.sim -,,y,script,./test.sh -f tsim/field/binary.sim -,,y,script,./test.sh -f tsim/field/bigint.sim -,,y,script,./test.sh -f tsim/field/bool.sim -,,y,script,./test.sh -f tsim/field/double.sim -,,y,script,./test.sh -f tsim/field/float.sim -,,y,script,./test.sh -f tsim/field/int.sim -,,y,script,./test.sh -f tsim/field/single.sim -,,y,script,./test.sh -f tsim/field/smallint.sim -,,y,script,./test.sh -f tsim/field/tinyint.sim -,,y,script,./test.sh -f tsim/field/unsigined_bigint.sim -,,y,script,./test.sh -f tsim/vector/metrics_field.sim -,,y,script,./test.sh -f tsim/vector/metrics_mix.sim -,,y,script,./test.sh -f tsim/vector/metrics_query.sim -,,y,script,./test.sh -f tsim/vector/metrics_tag.sim -,,y,script,./test.sh -f tsim/vector/metrics_time.sim -,,y,script,./test.sh -f tsim/vector/multi.sim -,,y,script,./test.sh -f tsim/vector/single.sim -,,y,script,./test.sh -f tsim/vector/table_field.sim -,,y,script,./test.sh -f tsim/vector/table_mix.sim -,,y,script,./test.sh -f tsim/vector/table_query.sim -,,y,script,./test.sh -f tsim/vector/table_time.sim -,,y,script,./test.sh -f tsim/wal/kill.sim -,,y,script,./test.sh -f tsim/tag/3.sim -,,y,script,./test.sh -f tsim/tag/4.sim -,,y,script,./test.sh -f tsim/tag/5.sim -,,y,script,./test.sh -f tsim/tag/6.sim -,,y,script,./test.sh -f tsim/tag/add.sim -,,y,script,./test.sh -f tsim/tag/bigint.sim -,,y,script,./test.sh -f tsim/tag/binary_binary.sim -,,y,script,./test.sh -f tsim/tag/binary.sim -,,y,script,./test.sh -f tsim/tag/bool_binary.sim -,,y,script,./test.sh -f tsim/tag/bool_int.sim -,,y,script,./test.sh -f tsim/tag/bool.sim -,,y,script,./test.sh -f tsim/tag/change.sim -,,y,script,./test.sh -f tsim/tag/column.sim -,,y,script,./test.sh -f tsim/tag/commit.sim -,,y,script,./test.sh -f tsim/tag/create.sim -,,y,script,./test.sh -f tsim/tag/delete.sim -,,y,script,./test.sh -f tsim/tag/double.sim -,,y,script,./test.sh -f tsim/tag/filter.sim -,,y,script,./test.sh -f tsim/tag/float.sim -,,y,script,./test.sh -f tsim/tag/int_binary.sim -,,y,script,./test.sh -f tsim/tag/int_float.sim -,,y,script,./test.sh -f tsim/tag/int.sim -,,y,script,./test.sh -f tsim/tag/set.sim -,,y,script,./test.sh -f tsim/tag/smallint.sim -,,y,script,./test.sh -f tsim/tag/tinyint.sim -,,y,script,./test.sh -f tsim/tag/drop_tag.sim -,,y,script,./test.sh -f tsim/tag/tbNameIn.sim -,,y,script,./test.sh -f tmp/monitor.sim - #system test ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py @@ -442,6 +25,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_limit_opt.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py @@ -449,6 +33,14 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb3.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb0.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/ins_topics_test.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxTopic.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqClientConsLog.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqOffset.py +,,n,system-test,python3 ./test.py -f 7-tmq/tmqDropConsumer.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py @@ -479,6 +71,11 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreMnode.py -N 5 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreQnode.py -N 5 -M 3 + ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/basic5.py @@ -492,10 +89,12 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/db.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqError.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/schema.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbFilterWhere.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData1.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +,,n,system-test,python3 ./test.py -f 7-tmq/tmqConsumerGroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAlterSchema.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -N 3 -n 3 @@ -528,6 +127,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py +,,n,system-test,python3 ./test.py -f 7-tmq/tmq_offset.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/raw_block_interface_test.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 @@ -535,6 +135,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3404.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3581.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3311.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/balance_vgroups_r1.py -N 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py @@ -554,17 +157,21 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_control.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_manage.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_privilege.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_privilege_show.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_privilege_all.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/multilevel.py -#,,n,system-test,python3 ./test.py -f 0-others/compatibility.py +,,n,system-test,python3 ./test.py -f 0-others/compatibility.py ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py ,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py +#,,n,system-test,python3 ./test.py -f 0-others/splitVGroup.py -N 5 +,,n,system-test,python3 ./test.py -f 0-others/timeRangeWise.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py @@ -582,7 +189,36 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/db_tb_name_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/InsertFuturets.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_wide_column.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_benchmark.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionUS.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionNS.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -R @@ -695,6 +331,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/orderBy.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py @@ -709,8 +346,11 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaBasic.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sma_index.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml_TS-3724.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py @@ -777,6 +417,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/systable_func.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py @@ -785,6 +426,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tagFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3 + ,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 @@ -813,11 +456,11 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 #,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 ,,n,system-test,python3 ./test.py -f 6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py -N 6 -M 3 - +,,n,system-test,python ./test.py -f 6-cluster/5dnode3mnodeRoll.py -N 3 -C 1 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 #,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 -,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1 @@ -1056,7 +699,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 4 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 4 @@ -1122,6 +765,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/odbc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py +,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 +,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/stt.py -N 3 #tsim test ,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim @@ -1156,6 +801,7 @@ ,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim ,,y,script,./test.sh -f tsim/user/privilege_topic.sim ,,y,script,./test.sh -f tsim/user/privilege_table.sim +,,y,script,./test.sh -f tsim/user/privilege_create_db.sim ,,y,script,./test.sh -f tsim/db/alter_option.sim ,,y,script,./test.sh -f tsim/db/alter_replica_31.sim ,,y,script,./test.sh -f tsim/db/basic1.sim @@ -1241,6 +887,7 @@ ,,y,script,./test.sh -f tsim/parser/columnValue_tinyint.sim ,,y,script,./test.sh -f tsim/parser/columnValue_unsign.sim ,,y,script,./test.sh -f tsim/parser/condition.sim +,,y,script,./test.sh -f tsim/parser/condition_scl.sim ,,y,script,./test.sh -f tsim/parser/constCol.sim ,,y,script,./test.sh -f tsim/parser/create_db.sim ,,y,script,./test.sh -f tsim/parser/create_mt.sim @@ -1313,17 +960,22 @@ ,,n,script,./test.sh -f tsim/query/udfpy.sim ,,y,script,./test.sh -f tsim/query/udf_with_const.sim ,,y,script,./test.sh -f tsim/query/join_interval.sim +,,y,script,./test.sh -f tsim/query/join_pk.sim ,,y,script,./test.sh -f tsim/query/unionall_as_table.sim ,,y,script,./test.sh -f tsim/query/multi_order_by.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim ,,y,script,./test.sh -f tsim/query/groupby.sim +,,y,script,./test.sh -f tsim/query/groupby_distinct.sim ,,y,script,./test.sh -f tsim/query/event.sim ,,y,script,./test.sh -f tsim/query/forceFill.sim ,,y,script,./test.sh -f tsim/query/emptyTsRange.sim +,,y,script,./test.sh -f tsim/query/emptyTsRange_scl.sim ,,y,script,./test.sh -f tsim/query/partitionby.sim ,,y,script,./test.sh -f tsim/query/tableCount.sim ,,y,script,./test.sh -f tsim/query/tag_scan.sim ,,y,script,./test.sh -f tsim/query/nullColSma.sim +,,y,script,./test.sh -f tsim/query/bug3398.sim +,,y,script,./test.sh -f tsim/query/explain_tsorder.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim ,,y,script,./test.sh -f tsim/snode/basic1.sim ,,y,script,./test.sh -f tsim/mnode/basic1.sim @@ -1552,6 +1204,7 @@ ,,y,script,./test.sh -f tsim/tag/drop_tag.sim ,,y,script,./test.sh -f tsim/tag/tbNameIn.sim ,,y,script,./test.sh -f tmp/monitor.sim + #develop test ,,n,develop-test,python3 ./test.py -f 2-query/table_count_scan.py ,,n,develop-test,python3 ./test.py -f 2-query/show_create_db.py @@ -1574,3 +1227,4 @@ ,,n,docs-examples-test,bash csharp.sh ,,n,docs-examples-test,bash jdbc.sh ,,n,docs-examples-test,bash go.sh +,,n,docs-examples-test,bash test_R.sh diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 0fc29c241b24db24b92862b57a9e61278e2c6fe0..5ae061072af3de05b65ac2d2e8ea47f1d7dd33d6 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -68,7 +68,7 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j || exit 1" + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ if [[ -d ${WORKDIR}/debugNoSan ]] ;then @@ -97,7 +97,7 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j || exit 1 " + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh index 2d736e141497b084c5b0e215881683e0e8c67b15..206f99ff3df6bbf6a75ec1e2303a9eb42034ba6a 100755 --- a/tests/parallel_test/run_case.sh +++ b/tests/parallel_test/run_case.sh @@ -76,10 +76,10 @@ ulimit -c unlimited md5sum /usr/lib/libtaos.so.1 md5sum /home/TDinternal/debug/build/lib/libtaos.so -#define taospy 2.7.6 +#define taospy 2.7.10 pip3 list|grep taospy pip3 uninstall taospy -y -pip3 install --default-timeout=120 taospy==2.7.6 +pip3 install --default-timeout=120 taospy==2.7.10 $TIMEOUT_CMD $cmd RET=$? diff --git a/tests/parallel_test/split_case.sh b/tests/parallel_test/split_case.sh index af601ed9a6a1c4d371e56d94aa45bc56b8715793..4e2c535fafab9a7d593b11f624149ec1dd20c6bf 100755 --- a/tests/parallel_test/split_case.sh +++ b/tests/parallel_test/split_case.sh @@ -5,6 +5,8 @@ parm_path=$(pwd ${parm_path}) echo "execute path:${parm_path}" cd ${parm_path} cp cases.task ${case_file} +sed -i '/udf/d' ${case_file} +sed -i '/Udf/d' ${case_file} sed -i '/^$/d' ${case_file} sed -i '$a\%%FINISHED%%' ${case_file} diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index ec588659e93ffc30d21355b970e595d08f04a9f7..5024f1e2fe25b2ac26e6434da3218c304661f89c 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1722,12 +1722,14 @@ class TaskCreateDb(StateTransitionTask): vg_nums = random.randint(1, 8) cache_model = Dice.choice(['none', 'last_row', 'last_value', 'both']) buffer = random.randint(3, 128) + walRetentionPeriod = random.randint(1, 10000) dbName = self._db.getName() - self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} ".format(dbName, repStr, + self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} wal_retention_period {} ".format(dbName, repStr, updatePostfix, vg_nums, cache_model, - buffer)) + buffer, + walRetentionPeriod)) if dbName == "db_0" and Config.getConfig().use_shadow_db: self.execWtSql(wt, "create database {} {} {} ".format("db_s", repStr, updatePostfix)) @@ -2041,18 +2043,19 @@ class TdSuperTable: for topic in current_topic_list: topic_list.append(topic) - consumer.subscribe(topic_list) - - # consumer with random work life - time_start = time.time() - while 1: - res = consumer.poll(1) - consumer.commit(res) - if time.time() - time_start > random.randint(5, 50): - break try: + consumer.subscribe(topic_list) + + # consumer with random work life + time_start = time.time() + while 1: + res = consumer.poll(1) + consumer.commit(res) + if time.time() - time_start > random.randint(5, 50): + break consumer.unsubscribe() - except TmqError as e: + consumer.close() + except TmqError as err: # topic deleted by other threads pass return diff --git a/tests/pytest/insert/basic.py b/tests/pytest/insert/basic.py index f23f38651aa57590a14824077e003812ea10d8bd..203dda027a96053878cbd7db2e1c741eb1423df4 100644 --- a/tests/pytest/insert/basic.py +++ b/tests/pytest/insert/basic.py @@ -18,7 +18,7 @@ from util.sql import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar = 1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/pytest/insert/basic_unsigned.py b/tests/pytest/insert/basic_unsigned.py index ff7e0e5e4ad57a35d574d88ea179f713c8a8b611..c9f6eda53595b00c803876e28846298e8a53f927 100644 --- a/tests/pytest/insert/basic_unsigned.py +++ b/tests/pytest/insert/basic_unsigned.py @@ -18,7 +18,7 @@ from util.sql import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar = 1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) @@ -39,10 +39,9 @@ class TDTestCase: tdSql.execute('insert into tb values (now - 5m , NULL)') tdSql.execute('insert into tb values (now - 6m , 10)') tdSql.execute('insert into tb values (now - 7m , NULL)') - tdSql.execute('insert into tb values (now - 8m , 254)') tdSql.error('insert into tb values (now - 9m, -1)') - tdSql.error('insert into tb values (now - 9m, 255)') + tdSql.execute('insert into tb values (now - 9m, 255)') tdSql.query("select * from tb") tdSql.checkRows(insertRows + 4) diff --git a/tests/pytest/insert/bigint.py b/tests/pytest/insert/bigint.py index 5431cf8106fdbcd74bdd310de45039badb3bce7f..cfb54283db1a78ccfbe9177354ec0ea1d777cf28 100644 --- a/tests/pytest/insert/bigint.py +++ b/tests/pytest/insert/bigint.py @@ -7,7 +7,7 @@ from util.sql import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar = 1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py index 28621f777ba16fed985642551b4db3449204fb2e..94f712fea9e14a4d20b44d242004ff3ed1d7f85a 100644 --- a/tests/pytest/insert/binary.py +++ b/tests/pytest/insert/binary.py @@ -10,7 +10,7 @@ import os class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar = 1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/pytest/insert/bool.py b/tests/pytest/insert/bool.py index 1412567cd75721d99308ddadfe7b35032207faaa..a455cfa2e3ac50e3dcd0691d0fdc97004887d8ab 100644 --- a/tests/pytest/insert/bool.py +++ b/tests/pytest/insert/bool.py @@ -7,7 +7,7 @@ from util.sql import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar = 1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/pytest/insert/date.py b/tests/pytest/insert/date.py index d4f1176f613dfc48d141893d9f7c955e1852a04e..66cba0645a2a5d2b643d56b695ee78952bb19390 100644 --- a/tests/pytest/insert/date.py +++ b/tests/pytest/insert/date.py @@ -7,7 +7,7 @@ from util.sql import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar = 1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/pytest/insert/double.py b/tests/pytest/insert/double.py index 2699f4b0e95897de47d461f885690e2b8a7863d7..6fc6afd3bf13909e43346afae30791e7fe85f3df 100644 --- a/tests/pytest/insert/double.py +++ b/tests/pytest/insert/double.py @@ -7,7 +7,7 @@ from util.sql import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar = 1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/pytest/query/query.py b/tests/pytest/query/query.py index c759e7766827e9b8e30f1b9ceb812c755fb057ae..b37cef7b10623f3cd4eb7380b4ec22598bc520a7 100644 --- a/tests/pytest/query/query.py +++ b/tests/pytest/query/query.py @@ -19,23 +19,24 @@ from util.sql import tdSql from util.dnodes import tdDnodes class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar = 1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.ts = 1538548685000 def bug_6387(self): - tdSql.execute("create database bug6387 ") - tdSql.execute("use bug6387 ") + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db") + tdSql.execute("use db") tdSql.execute("create table test(ts timestamp, c1 int) tags(t1 int)") for i in range(5000): sql = "insert into t%d using test tags(1) values " % i for j in range(21): sql = sql + "(now+%ds,%d)" % (j ,j ) tdSql.execute(sql) - tdSql.query("select count(*) from test interval(1s) group by tbname") - tdSql.checkData(0,1,1) +# tdSql.query("select count(*) from test interval(1s) group by tbname") +# tdSql.checkData(0,1,1) def run(self): tdSql.prepare() @@ -58,10 +59,10 @@ class TDTestCase: tdSql.query("select * from db.st where ts='2020-05-13 10:00:00.000'") tdSql.checkRows(1) - tdSql.query("select tbname, dev from dev_001") - tdSql.checkRows(1) - tdSql.checkData(0, 0, 'dev_001') - tdSql.checkData(0, 1, 'dev_01') +# tdSql.query("select tbname, dev from dev_001") +# tdSql.checkRows(1) +# tdSql.checkData(0, 0, 'dev_001') +# tdSql.checkData(0, 1, 'dev_01') tdSql.query("select tbname, dev, tagtype from dev_001") tdSql.checkRows(2) @@ -124,10 +125,10 @@ class TDTestCase: tdSql.checkRows(5) # For jira: https://jira.taosdata.com:18080/browse/TD-2850 - tdSql.execute("create database 'Test' ") - tdSql.execute("use 'Test' ") - tdSql.execute("create table 'TB'(ts timestamp, 'Col1' int) tags('Tag1' int)") - tdSql.execute("insert into 'Tb0' using tb tags(1) values(now, 1)") + tdSql.execute("create database `Test` ") + tdSql.execute("use `Test` ") + tdSql.execute("create table TB(ts timestamp, `Col1` int) tags(`Tag1` int)") + tdSql.execute("insert into Tb0 using tb tags(1) values(now, 1)") tdSql.query("select * from tb") tdSql.checkRows(1) @@ -135,7 +136,7 @@ class TDTestCase: tdSql.checkRows(1) #For jira: https://jira.taosdata.com:18080/browse/TD-6387 - self.bug_6387() + #self.bug_6387() def stop(self): diff --git a/tests/pytest/query/queryError.py b/tests/pytest/query/queryError.py index e5c468600ba56a251057f204971084fe2844a85e..1993cda0c96ac7f9e686dcb87d9a2c9466a6fbbd 100644 --- a/tests/pytest/query/queryError.py +++ b/tests/pytest/query/queryError.py @@ -19,7 +19,7 @@ from util.sql import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar = 1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) @@ -42,10 +42,10 @@ class TDTestCase: ('2020-05-13 10:00:00.005', 3, 'third')""") # query first .. as .. - tdSql.error("select first(*) as one from st") + tdSql.execute("select first(*) as one from st") # query last .. as .. - tdSql.error("select last(*) as latest from st") + tdSql.execute("select last(*) as latest from st") # query last row .. as .. tdSql.error("select last_row as latest from st") @@ -54,7 +54,7 @@ class TDTestCase: #tdSql.error("select distinct tagtype from st") # query .. order by non-time field - tdSql.error("select * from st order by name") + tdSql.execute("select * from st order by name") # TD-2133 tdSql.error("select diff(tagtype),bottom(tagtype,1) from dev_001") @@ -66,8 +66,8 @@ class TDTestCase: tdSql.error("select diff(tagtype),top(tagtype,1) from dev_001") # TD-6006 - tdSql.error("select * from dev_001 where 'name' is not null") - tdSql.error("select * from dev_001 where \"name\" = 'first'") + tdSql.execute("select * from dev_001 where 'name' is not null") + tdSql.execute("select * from dev_001 where \"name\" = 'first'") def stop(self): tdSql.close() diff --git a/tests/pytest/query/queryFillTest.py b/tests/pytest/query/queryFillTest.py index e50d02faf27f33191f4a76c535443f6802e0ed28..f35c88470a3fa46c8686dd2920fbb491c60a084d 100644 --- a/tests/pytest/query/queryFillTest.py +++ b/tests/pytest/query/queryFillTest.py @@ -19,7 +19,7 @@ from util.sql import tdSql class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar = 1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) @@ -46,39 +46,39 @@ class TDTestCase: tdSql.execute(sql) tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h)") - tdSql.checkRows(139) - tdSql.checkData(0, 1, -1.5) - tdSql.checkData(138, 1, -1.0) + tdSql.checkRows(128) +# tdSql.checkData(0, 1, -1.5) +# tdSql.checkData(138, 1, -1.0) tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(none)") - tdSql.checkRows(139) - tdSql.checkData(0, 1, -1.5) - tdSql.checkData(138, 1, -1.0) + tdSql.checkRows(128) +# tdSql.checkData(0, 1, -1.5) +# tdSql.checkData(138, 1, -1.0) tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(value, 2.0)") tdSql.checkRows(141) - tdSql.checkData(0, 1, 2.0) - tdSql.checkData(140, 1, 2.0) +# tdSql.checkData(0, 1, 2.0) +# tdSql.checkData(140, 1, 2.0) tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(prev)") tdSql.checkRows(141) - tdSql.checkData(0, 1, None) - tdSql.checkData(140, 1, -1.0) +# tdSql.checkData(0, 1, None) +# tdSql.checkData(140, 1, -1.0) tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(null)") tdSql.checkRows(141) - tdSql.checkData(0, 1, None) - tdSql.checkData(140, 1, None) +# tdSql.checkData(0, 1, None) +# tdSql.checkData(140, 1, None) tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(linear)") tdSql.checkRows(141) - tdSql.checkData(0, 1, None) - tdSql.checkData(140, 1, None) +# tdSql.checkData(0, 1, None) +# tdSql.checkData(140, 1, None) tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(next)") tdSql.checkRows(141) - tdSql.checkData(0, 1, -1.5) - tdSql.checkData(140, 1, None) +# tdSql.checkData(0, 1, -1.5) +# tdSql.checkData(140, 1, None) tdSql.query("select max(col1) - min(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' and id = 1 group by loc, id") rows = tdSql.queryRows @@ -92,4 +92,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdumpTest.py b/tests/pytest/tools/taosdumpTest.py index 4bfb7d5ba33cbd105b248b5293029ce3606f01c5..aafd365f349e899ec55d64f50bf2f25962fec803 100644 --- a/tests/pytest/tools/taosdumpTest.py +++ b/tests/pytest/tools/taosdumpTest.py @@ -92,9 +92,9 @@ class TDTestCase: else: tdLog.info("taosdump found: %s" % binPath) - os.system("%s -y --databases db -o ./taosdumptest/tmp1" % binPath) + os.system("%s --databases db -o ./taosdumptest/tmp1" % binPath) os.system( - "%s -y --databases db1 -o ./taosdumptest/tmp2" % + "%s --databases db1 -o ./taosdumptest/tmp2" % binPath) tdSql.execute("drop database db") @@ -172,7 +172,7 @@ class TDTestCase: tdSql.query("show stables") tdSql.checkRows(2) os.system( - "%s -y --databases db12312313231231321312312312_323 -o ./taosdumptest/tmp1" % + "%s --databases db12312313231231321312312312_323 -o ./taosdumptest/tmp1" % binPath) tdSql.execute("drop database db12312313231231321312312312_323") os.system("%s -i ./taosdumptest/tmp1" % binPath) diff --git a/tests/pytest/tools/taosdumpTest2.py b/tests/pytest/tools/taosdumpTest2.py index 8a85ce10ed53946abe4f8ecd4a022752e07f94c1..36e0480e578a49cd6d30e8dd2d0c8d3ed602ad3a 100644 --- a/tests/pytest/tools/taosdumpTest2.py +++ b/tests/pytest/tools/taosdumpTest2.py @@ -97,7 +97,7 @@ class TDTestCase: tdSql.query("show databases") tdSql.checkRows(2) - os.system("%s -i ./taosdumptest/tmp -y" % binPath) + os.system("%s -i ./taosdumptest/tmp" % binPath) tdSql.query("show databases") tdSql.checkRows(3) @@ -125,13 +125,13 @@ class TDTestCase: os.system("rm ./taosdumptest/tmp/*.sql") os.system("rm ./taosdumptest/tmp/*.avro*") os.system("rm -rf ./taosdumptest/tmp/taosdump.*") - os.system("%s -D test -o ./taosdumptest/tmp -y" % binPath) + os.system("%s -D test -o ./taosdumptest/tmp" % binPath) tdSql.execute("drop database test") tdSql.query("show databases") tdSql.checkRows(3) - os.system("%s -i ./taosdumptest/tmp -y" % binPath) + os.system("%s -i ./taosdumptest/tmp" % binPath) tdSql.execute("use test") tdSql.query("show stables") diff --git a/tests/pytest/tools/taosdumpTestNanoSupport.py b/tests/pytest/tools/taosdumpTestNanoSupport.py index c40462b8db51629549040a4f906c927b3bf3154c..2a3990614acb1cf6bbc30d8b02132fe62a63c1b0 100644 --- a/tests/pytest/tools/taosdumpTestNanoSupport.py +++ b/tests/pytest/tools/taosdumpTestNanoSupport.py @@ -134,15 +134,15 @@ class TDTestCase: # dump all data os.system( - "%s -y -g --databases timedb1 -o ./taosdumptest/dumptmp1" % + "%s -g --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath) # dump part data with -S -E os.system( - '%s -y -g --databases timedb1 -S 1625068810000000000 -E 1625068860000000000 -o ./taosdumptest/dumptmp2 ' % + '%s -g --databases timedb1 -S 1625068810000000000 -E 1625068860000000000 -o ./taosdumptest/dumptmp2 ' % binPath) os.system( - '%s -y -g --databases timedb1 -S 1625068810000000000 -o ./taosdumptest/dumptmp3 ' % + '%s -g --databases timedb1 -S 1625068810000000000 -o ./taosdumptest/dumptmp3 ' % binPath) tdSql.execute("drop database timedb1") @@ -200,14 +200,14 @@ class TDTestCase: self.createdb(precision="us") os.system( - "%s -y -g --databases timedb1 -o ./taosdumptest/dumptmp1" % + "%s -g --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath) os.system( - '%s -y -g --databases timedb1 -S 1625068810000000 -E 1625068860000000 -o ./taosdumptest/dumptmp2 ' % + '%s -g --databases timedb1 -S 1625068810000000 -E 1625068860000000 -o ./taosdumptest/dumptmp2 ' % binPath) os.system( - '%s -y -g --databases timedb1 -S 1625068810000000 -o ./taosdumptest/dumptmp3 ' % + '%s -g --databases timedb1 -S 1625068810000000 -o ./taosdumptest/dumptmp3 ' % binPath) os.system("%s -i ./taosdumptest/dumptmp1" % binPath) @@ -269,14 +269,14 @@ class TDTestCase: self.createdb(precision="ms") os.system( - "%s -y -g --databases timedb1 -o ./taosdumptest/dumptmp1" % + "%s -g --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath) os.system( - '%s -y -g --databases timedb1 -S 1625068810000 -E 1625068860000 -o ./taosdumptest/dumptmp2 ' % + '%s -g --databases timedb1 -S 1625068810000 -E 1625068860000 -o ./taosdumptest/dumptmp2 ' % binPath) os.system( - '%s -y -g --databases timedb1 -S 1625068810000 -o ./taosdumptest/dumptmp3 ' % + '%s -g --databases timedb1 -S 1625068810000 -o ./taosdumptest/dumptmp3 ' % binPath) os.system("%s -i ./taosdumptest/dumptmp1" % binPath) diff --git a/tests/pytest/util/autogen.py b/tests/pytest/util/autogen.py index 90cd347b9aed49816bc0de4242bacff8891173bc..6f5c89d15b6ee36e6f4ee9664033bc7e761b9237 100644 --- a/tests/pytest/util/autogen.py +++ b/tests/pytest/util/autogen.py @@ -17,7 +17,7 @@ class AutoGen: def __init__(self): self.ts = 1600000000000 self.batch_size = 100 - seed = time.clock_gettime(time.CLOCK_REALTIME) + seed = time.time() % 10000 random.seed(seed) # set start ts @@ -93,9 +93,9 @@ class AutoGen: return ''.join(random.choice(letters) for i in range(count)) # create db - def create_db(self, dbname): + def create_db(self, dbname, vgroups = 2, replica = 1): self.dbname = dbname - tdSql.execute(f'create database {dbname}') + tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica}') tdSql.execute(f'use {dbname}') # create table or stable diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 80ab5baa9a8a23132abf75686ccdc6bebb6d8238..89e3df81b938d666e670f9096638ad0da91fa569 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -545,6 +545,8 @@ class TDDnode: def stoptaosd(self): + tdLog.debug("start to stop taosd on dnode: %d "% (self.index)) + # print(self.asan,self.running,self.remoteIP,self.valgrind) if self.asan: stopCmd = "%s -s stop -n dnode%d" % (self.execPath, self.index) tdLog.info("execute script: " + stopCmd) @@ -859,5 +861,4 @@ class TDDnodes: def getAsan(self): return self.asan - -tdDnodes = TDDnodes() +tdDnodes = TDDnodes() \ No newline at end of file diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index c041282bfce6d5cf1ba6723ddf476a50bc1f2c7a..2fa21b1983824de33b713d9b59008a69f9ffb8d3 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -78,29 +78,38 @@ class TDSql: self.cursor.execute(s) time.sleep(2) - def error(self, sql): + def error(self, sql, expectedErrno = None): + caller = inspect.getframeinfo(inspect.stack()[1][0]) expectErrNotOccured = True + try: self.cursor.execute(sql) except BaseException as e: expectErrNotOccured = False - caller = inspect.getframeinfo(inspect.stack()[1][0]) + self.errno = e.errno self.error_info = repr(e) # print(error_info) # self.error_info = error_info[error_info.index('(')+1:-1].split(",")[0].replace("'","") # self.error_info = (','.join(error_info.split(",")[:-1]).split("(",1)[1:][0]).replace("'","") # print("!!!!!!!!!!!!!!",self.error_info) - + if expectErrNotOccured: - caller = inspect.getframeinfo(inspect.stack()[1][0]) tdLog.exit("%s(%d) failed: sql:%s, expect error not occured" % (caller.filename, caller.lineno, sql)) else: self.queryRows = 0 self.queryCols = 0 self.queryResult = None - tdLog.info("sql:%s, expect error occured" % (sql)) + + if expectedErrno != None: + if expectedErrno == self.errno: + tdLog.info("sql:%s, expected errno %s occured" % (sql, expectedErrno)) + else: + tdLog.exit("%s(%d) failed: sql:%s, errno %s occured, but not expected errno %s" % (caller.filename, caller.lineno, sql, self.errno, expectedErrno)) + else: + tdLog.info("sql:%s, expect error occured" % (sql)) + return self.error_info - + def query(self, sql, row_tag=None,queryTimes=10): self.sql = sql @@ -431,8 +440,10 @@ class TDSql: time.sleep(1) continue - def execute(self, sql,queryTimes=30): + def execute(self, sql, queryTimes=30, show=False): self.sql = sql + if show: + tdLog.info(sql) i=1 while i <= queryTimes: try: diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 99507ef5c3ae58d786dbfe8f91e72400d471ec3f..604d6ade8944570d44331f9b3ecc9e722a096ee7 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -16,8 +16,8 @@ int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; -int32_t optrIdxList[] = {0, 7}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_NCHAR}; +int32_t optrIdxList[] = {5, 11}; typedef struct { char* oper; @@ -123,6 +123,7 @@ int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos); int queryColumnTest(TAOS_STMT *stmt, TAOS *taos); int queryMiscTest(TAOS_STMT *stmt, TAOS *taos); int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos); +int insertVarLenErr(TAOS_STMT *stmt, TAOS *taos); enum { TTYPE_INSERT = 1, @@ -190,6 +191,7 @@ CaseCfg gCase[] = { {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, {"query:NG-TBNEXISTS",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, false, insertNonExistsTb, 10, 10, 1, 3, 0, 0, 1, -1}, + {"query:NG-VARLENERR",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, true, insertVarLenErr, 10, 10, 1, 3, 0, 0, 1, -1}, // {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, // {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, @@ -319,7 +321,7 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper #if 0 CaseCtrl gCaseCtrl = { // query case with specified col&oper - .bindNullNum = 1, + .bindNullNum = 0, .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, @@ -329,18 +331,19 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindTagNum = 0, .bindRowNum = 0, .bindColTypeNum = 0, - .bindColTypeList = NULL, + .bindColTypeList = bindColTypeList, .optrIdxListNum = 0, - .optrIdxList = NULL, + .optrIdxList = optrIdxList, .checkParamNum = false, .printRes = true, .runTimes = 0, .caseRunIdx = -1, - //.optrIdxListNum = tListLen(optrIdxList), - //.optrIdxList = optrIdxList, - //.bindColTypeNum = tListLen(bindColTypeList), - //.bindColTypeList = bindColTypeList, - .caseIdx = 8, + .optrIdxListNum = tListLen(optrIdxList), + .optrIdxList = optrIdxList, + .bindColTypeNum = tListLen(bindColTypeList), + .bindColTypeList = bindColTypeList, + .caseRunIdx = -1, + .caseIdx = 24, .caseNum = 1, .caseRunNum = 1, }; @@ -453,6 +456,9 @@ void generateInsertSQL(BindData *data) { case TSDB_DATA_TYPE_UBIGINT: len += sprintf(data->sql + len, "tubigdata"); break; + case TSDB_DATA_TYPE_GEOMETRY: + len += sprintf(data->sql + len, "tgeometrydata"); + break; default: printf("!!!invalid tag type:%d", data->pTags[c].buffer_type); exit(1); @@ -521,6 +527,9 @@ void generateInsertSQL(BindData *data) { case TSDB_DATA_TYPE_UBIGINT: len += sprintf(data->sql + len, "ubigdata"); break; + case TSDB_DATA_TYPE_GEOMETRY: + len += sprintf(data->sql + len, "tgeometrydata"); + break; default: printf("!!!invalid col type:%d", data->pBind[c].buffer_type); exit(1); @@ -550,7 +559,7 @@ void bpAppendOperatorParam(BindData *data, int32_t *len, int32_t dataType, int32 if (gCaseCtrl.optrIdxListNum > 0) { pInfo = &operInfo[gCaseCtrl.optrIdxList[idx]]; } else { - if (TSDB_DATA_TYPE_VARCHAR == dataType || TSDB_DATA_TYPE_NCHAR == dataType) { + if (TSDB_DATA_TYPE_VARCHAR == dataType || TSDB_DATA_TYPE_NCHAR == dataType || TSDB_DATA_TYPE_GEOMETRY == dataType) { pInfo = &operInfo[varoperatorList[rand() % tListLen(varoperatorList)]]; } else { pInfo = &operInfo[operatorList[rand() % tListLen(operatorList)]]; @@ -635,6 +644,9 @@ int32_t bpAppendColumnName(BindData *data, int32_t type, int32_t len) { case TSDB_DATA_TYPE_UBIGINT: return sprintf(data->sql + len, "ubigdata"); break; + case TSDB_DATA_TYPE_GEOMETRY: + len += sprintf(data->sql + len, "tgeometrydata"); + break; default: printf("!!!invalid col type:%d", type); exit(1); @@ -868,6 +880,7 @@ int32_t prepareColData(BP_BIND_TYPE bType, BindData *data, int32_t bindIdx, int3 pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_GEOMETRY: pBase[bindIdx].buffer_length = gVarCharSize; pBase[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; pBase[bindIdx].length = data->binaryLen; @@ -1210,6 +1223,7 @@ int32_t bpAppendValueString(char *buf, int type, void *value, int32_t valueLen, case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: buf[*len] = '\''; ++(*len); memcpy(buf + *len, value, valueLen); @@ -1355,7 +1369,7 @@ void bpCheckColTagFields(TAOS_STMT *stmt, int32_t fieldNum, TAOS_FIELD_E* pField exit(1); } - if (pFields[i].type == TSDB_DATA_TYPE_BINARY) { + if (pFields[i].type == TSDB_DATA_TYPE_BINARY || pFields[i].type == TSDB_DATA_TYPE_GEOMETRY) { if (pFields[i].bytes != (pBind[i].buffer_length + 2)) { printf("!!!%s %dth field len %d mis-match expect len %d\n", BP_BIND_TYPE_STR(type), i, pFields[i].bytes, (pBind[i].buffer_length + 2)); exit(1); @@ -1439,14 +1453,17 @@ void bpShowBindParam(TAOS_MULTI_BIND *bind, int32_t num) { } } -int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { +int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, bool expectFail) { static int32_t n = 0; - bpCheckColFields(stmt, bind); + if (!expectFail) { + bpCheckColFields(stmt, bind); + } if (gCurCase->bindRowNum > 1) { if (0 == (n++%2)) { if (taos_stmt_bind_param_batch(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1454,6 +1471,7 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { } else { for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { if (taos_stmt_bind_single_param_batch(stmt, bind+i, i)) { + if (expectFail) continue; printf("!!!taos_stmt_bind_single_param_batch %d error:%s\n", taos_stmt_errstr(stmt), i); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1463,12 +1481,14 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { } else { if (0 == (n++%2)) { if (taos_stmt_bind_param_batch(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); } } else { if (taos_stmt_bind_param(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1531,7 +1551,7 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1583,7 +1603,7 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1641,7 +1661,7 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1691,7 +1711,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1759,7 +1779,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1811,7 +1831,7 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1872,7 +1892,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1938,7 +1958,7 @@ int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2005,7 +2025,7 @@ int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos) { if (gCaseCtrl.checkParamNum) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2065,7 +2085,7 @@ int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2119,7 +2139,7 @@ int queryColumnTest(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum, false)) { exit(1); } @@ -2167,7 +2187,7 @@ int queryMiscTest(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum, false)) { exit(1); } @@ -2234,6 +2254,42 @@ int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos) { return 0; } +void bpAddWrongVarBuffLen(TAOS_MULTI_BIND* pBind) { + for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { + if (pBind[i].buffer_type == TSDB_DATA_TYPE_BINARY || pBind[i].buffer_type == TSDB_DATA_TYPE_NCHAR) { + *pBind[i].length += 100; + } + } +} + +int insertVarLenErr(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + prepareInsertData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckIsInsert(stmt, 1); + + code = bpSetTableNameTags(&data, 0, "t0", stmt); + if (code != 0){ + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpAddWrongVarBuffLen(data.pBind); + + if (bpBindParam(stmt, data.pBind, true)) { + exit(1); + } + + destroyData(&data); + + return 0; +} int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { @@ -2499,6 +2555,9 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * case TSDB_DATA_TYPE_UBIGINT: blen += sprintf(buf + blen, "ubigdata bigint unsigned"); break; + case TSDB_DATA_TYPE_GEOMETRY: + blen += sprintf(buf + blen, "geometrydata geometry(%d)", gVarCharSize); + break; default: printf("invalid col type:%d", colList[c]); exit(1); @@ -2557,13 +2616,16 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * case TSDB_DATA_TYPE_UBIGINT: blen += sprintf(buf + blen, "tubigdata bigint unsigned"); break; + case TSDB_DATA_TYPE_GEOMETRY: + blen += sprintf(buf + blen, "tgeometrydata geometry(%d)", gVarCharSize); + break; default: printf("invalid col type:%d", colList[c]); exit(1); - } + } } - blen += sprintf(buf + blen, ")"); + blen += sprintf(buf + blen, ")"); } if (3 == tableType) { @@ -2615,6 +2677,9 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * case TSDB_DATA_TYPE_UBIGINT: blen += sprintf(buf + blen, "%d", rand() % 128); break; + case TSDB_DATA_TYPE_GEOMETRY: + blen += sprintf(buf + blen, "'geo%d'", rand() % 128); + break; default: printf("invalid col type:%d", colList[c]); exit(1); diff --git a/tests/script/api/demoapi.c b/tests/script/api/demoapi.c index 6c060a6325dccc03f18e295dd88329df7e81395c..18bd89ceee4f6f51a242493747fb255ae344ad8d 100644 --- a/tests/script/api/demoapi.c +++ b/tests/script/api/demoapi.c @@ -200,7 +200,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { for (int f = 0; f < num_fields; f++) { if ((fields[f].type != TSDB_DATA_TYPE_VARCHAR) && (fields[f].type != TSDB_DATA_TYPE_NCHAR) - && (fields[f].type != TSDB_DATA_TYPE_JSON)) { + && (fields[f].type != TSDB_DATA_TYPE_JSON) + && (fields[f].type != TSDB_DATA_TYPE_GEOMETRY)) { printf("col%d type is %d, no need get offset\n", f, fields[f].type); for (int64_t c = 0; c < rows; c++) { diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c index 1bf49876895987e08c96d7afed6236dcb626f678..d9cb2128efaf3ceaa9ebaf08cddb74346f6ea06e 100644 --- a/tests/script/api/passwdTest.c +++ b/tests/script/api/passwdTest.c @@ -32,9 +32,21 @@ #define nRoot 10 #define nUser 10 #define USER_LEN 24 +#define BUF_LEN 1024 + +typedef uint16_t VarDataLenT; + +#define TSDB_NCHAR_SIZE sizeof(int32_t) +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + +#define GET_FLOAT_VAL(x) (*(float *)(x)) +#define GET_DOUBLE_VAL(x) (*(double *)(x)) + +#define varDataLen(v) ((VarDataLenT *)(v))[0] void createUsers(TAOS *taos, const char *host, char *qstr); void passVerTestMulti(const char *host, char *qstr); +void sysInfoTest(TAOS *taos, const char *host, char *qstr); int nPassVerNotified = 0; TAOS *taosu[nRoot] = {0}; @@ -83,6 +95,95 @@ static void queryDB(TAOS *taos, char *command) { taos_free_result(pSql); } +int printRow(char *str, TAOS_ROW row, TAOS_FIELD *fields, int numFields) { + int len = 0; + char split = ' '; + + for (int i = 0; i < numFields; ++i) { + if (i > 0) { + str[len++] = split; + } + + if (row[i] == NULL) { + len += sprintf(str + len, "%s", "NULL"); + continue; + } + + switch (fields[i].type) { + case TSDB_DATA_TYPE_TINYINT: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + break; + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(str + len, "%u", *((uint8_t *)row[i])); + break; + case TSDB_DATA_TYPE_SMALLINT: + len += sprintf(str + len, "%d", *((int16_t *)row[i])); + break; + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(str + len, "%u", *((uint16_t *)row[i])); + break; + case TSDB_DATA_TYPE_INT: + len += sprintf(str + len, "%d", *((int32_t *)row[i])); + break; + case TSDB_DATA_TYPE_UINT: + len += sprintf(str + len, "%u", *((uint32_t *)row[i])); + break; + case TSDB_DATA_TYPE_BIGINT: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i])); + break; + case TSDB_DATA_TYPE_FLOAT: { + float fv = 0; + fv = GET_FLOAT_VAL(row[i]); + len += sprintf(str + len, "%f", fv); + } break; + case TSDB_DATA_TYPE_DOUBLE: { + double dv = 0; + dv = GET_DOUBLE_VAL(row[i]); + len += sprintf(str + len, "%lf", dv); + } break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: { + int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); + memcpy(str + len, row[i], charLen); + len += charLen; + } break; + case TSDB_DATA_TYPE_TIMESTAMP: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + case TSDB_DATA_TYPE_BOOL: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + default: + break; + } + } + return len; +} + +static int printResult(TAOS_RES *res, char *output) { + int numFields = taos_num_fields(res); + TAOS_FIELD *fields = taos_fetch_fields(res); + char header[BUF_LEN] = {0}; + int len = 0; + for (int i = 0; i < numFields; ++i) { + len += sprintf(header + len, "%s ", fields[i].name); + } + puts(header); + if (output) { + strncpy(output, header, BUF_LEN); + } + + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(res))) { + char temp[BUF_LEN] = {0}; + printRow(temp, row, fields, numFields); + puts(temp); + } +} + int main(int argc, char *argv[]) { char qstr[1024]; @@ -99,6 +200,7 @@ int main(int argc, char *argv[]) { } createUsers(taos, argv[1], qstr); passVerTestMulti(argv[1], qstr); + sysInfoTest(taos, argv[1], qstr); taos_close(taos); taos_cleanup(); @@ -167,6 +269,8 @@ void passVerTestMulti(const char *host, char *qstr) { int nConn = nRoot + nUser; for (int i = 0; i < 15; ++i) { + printf("%s:%d [%d] second(s) elasped, passVer notification received:%d, total:%d\n", __func__, __LINE__, i, + nPassVerNotified, nConn); if (nPassVerNotified >= nConn) break; sleep(1); } @@ -175,19 +279,100 @@ void passVerTestMulti(const char *host, char *qstr) { for (int i = 0; i < nRoot; ++i) { taos_close(taos[i]); printf("%s:%d close taos[%d]\n", __func__, __LINE__, i); - sleep(1); + // sleep(1); } for (int i = 0; i < nUser; ++i) { taos_close(taosu[i]); printf("%s:%d close taosu[%d]\n", __func__, __LINE__, i); - sleep(1); + // sleep(1); } + fprintf(stderr, "######## %s #########\n", __func__); if (nPassVerNotified >= nConn) { - fprintf(stderr, "succeed to get passVer notification since nNotify %d >= nConn %d\n", nPassVerNotified, nConn); + fprintf(stderr, ">>> succeed to get passVer notification since nNotify %d >= nConn %d\n", nPassVerNotified, + nConn); } else { - fprintf(stderr, "failed to get passVer notification since nNotify %d < nConn %d\n", nPassVerNotified, nConn); + fprintf(stderr, ">>> failed to get passVer notification since nNotify %d < nConn %d\n", nPassVerNotified, nConn); } + fprintf(stderr, "######## %s #########\n", __func__); // sleep(300); +} + +void sysInfoTest(TAOS *taosRoot, const char *host, char *qstr) { + TAOS *taos[nRoot] = {0}; + char userName[USER_LEN] = "user0"; + + for (int i = 0; i < nRoot; ++i) { + taos[i] = taos_connect(host, "user0", "taos", NULL, 0); + if (taos[i] == NULL) { + fprintf(stderr, "failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); + exit(1); + } + } + + queryDB(taosRoot, "create database if not exists demo11 vgroups 1 minrows 10"); + queryDB(taosRoot, "create database if not exists demo12 vgroups 1 minrows 10"); + queryDB(taosRoot, "create database if not exists demo13 vgroups 1 minrows 10"); + + queryDB(taosRoot, "create table demo11.stb (ts timestamp, c1 int) tags(t1 int)"); + queryDB(taosRoot, "create table demo12.stb (ts timestamp, c1 int) tags(t1 int)"); + queryDB(taosRoot, "create table demo13.stb (ts timestamp, c1 int) tags(t1 int)"); + + sprintf(qstr, "show grants"); + char output[BUF_LEN]; + TAOS_RES *res = NULL; + int32_t nRep = 0; + +_REP: + fprintf(stderr, "######## %s loop:%d #########\n", __func__, nRep); + res = taos_query(taos[0], qstr); + if (taos_errno(res) != 0) { + fprintf(stderr, "%s:%d failed to execute: %s since %s\n", __func__, __LINE__, qstr, taos_errstr(res)); + taos_free_result(res); + exit(EXIT_FAILURE); + } + printResult(res, output); + taos_free_result(res); + if (!strstr(output, "timeseries")) { + fprintf(stderr, "%s:%d expected output: 'timeseries' not occur\n", __func__, __LINE__); + exit(EXIT_FAILURE); + } + + queryDB(taosRoot, "alter user user0 sysinfo 0"); + + fprintf(stderr, "%s:%d sleep 2 seconds to wait HB take effect\n", __func__, __LINE__); + for (int i = 1; i <= 2; ++i) { + sleep(1); + } + + res = taos_query(taos[0], qstr); + if (taos_errno(res) != 0) { + if (!strstr(taos_errstr(res), "Permission denied")) { + fprintf(stderr, "%s:%d expected error: 'Permission denied' not occur\n", __func__, __LINE__); + taos_free_result(res); + exit(EXIT_FAILURE); + } + } + taos_free_result(res); + + queryDB(taosRoot, "alter user user0 sysinfo 1"); + fprintf(stderr, "%s:%d sleep 2 seconds to wait HB take effect\n", __func__, __LINE__); + for (int i = 1; i <= 2; ++i) { + sleep(1); + } + + if(++nRep < 5) { + goto _REP; + } + + // close the taos_conn + for (int i = 0; i < nRoot; ++i) { + taos_close(taos[i]); + fprintf(stderr, "%s:%d close taos[%d]\n", __func__, __LINE__, i); + } + + fprintf(stderr, "######## %s #########\n", __func__); + fprintf(stderr, ">>> succeed to run sysInfoTest\n"); + fprintf(stderr, "######## %s #########\n", __func__); } \ No newline at end of file diff --git a/tests/script/api/stmt_function.c b/tests/script/api/stmt_function.c index 62f8fab29f1cf964bc9c9813a74191e080908e3e..55ab3f5631aca6dc44839db0a470a981486382f3 100644 --- a/tests/script/api/stmt_function.c +++ b/tests/script/api/stmt_function.c @@ -400,6 +400,7 @@ void taos_stmt_use_result_query(void *taos, char *col, int type) { params->length = ¶ms->buffer_length; break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_GEOMETRY: params->buffer_length = sizeof(v.c6); params->buffer = &v.c6; params->length = ¶ms->buffer_length; diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 7da8da09bfcf3810b692ff25f37333c93ae87497..5b1773e66418e96902c887f455032083bc0ddd2c 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -118,7 +118,7 @@ echo "statusInterval 1" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG -echo "tmrDebugFlag 131" >> $TAOS_CFG +echo "tmrDebugFlag 143" >> $TAOS_CFG echo "uDebugFlag 143" >> $TAOS_CFG echo "rpcDebugFlag 143" >> $TAOS_CFG echo "jniDebugFlag 143" >> $TAOS_CFG diff --git a/tests/script/sh/pycumsum.py b/tests/script/sh/pycumsum.py new file mode 100644 index 0000000000000000000000000000000000000000..27d575aec4f997874172d3823295cd3621375b3f --- /dev/null +++ b/tests/script/sh/pycumsum.py @@ -0,0 +1,29 @@ +import pickle +import numpy as np + +def init(): + pass + +def destroy(): + pass + +def start(): + return pickle.dumps(0.0) + +def finish(buf): + return pickle.loads(buf) + +def reduce(datablock, buf): + (rows, cols) = datablock.shape() + state = pickle.loads(buf) + row = [] + for i in range(rows): + for j in range(cols): + cell = datablock.data(i, j) + if cell is not None: + row.append(datablock.data(i, j)) + if len(row) > 1: + new_state = np.cumsum(row)[-1] + else: + new_state = state + return pickle.dumps(new_state) diff --git a/tests/script/sh/stop_dnodes.bat b/tests/script/sh/stop_dnodes.bat index ab7af2ca92023745b8b712cb78e9b168a6c00598..65aee26ed4f575ce446ba02055713f69f6082191 100644 --- a/tests/script/sh/stop_dnodes.bat +++ b/tests/script/sh/stop_dnodes.bat @@ -3,4 +3,6 @@ rem echo taskkill /F /IM taosd.exe wmic process where "name='taosd.exe'" call terminate > NUL 2>&1 -taskkill /F /IM taosd.exe > NUL 2>&1 \ No newline at end of file +taskkill /F /IM taosd.exe > NUL 2>&1 + +rem echo taskkill /F /IM taosd.exe finished \ No newline at end of file diff --git a/tests/script/tsim/alter/table.sim b/tests/script/tsim/alter/table.sim index db2a22205f9002c715f516fe9cdc31ea8082d1ee..0cf291523a7b4b6ca2cbf321a797c65656dfd59f 100644 --- a/tests/script/tsim/alter/table.sim +++ b/tests/script/tsim/alter/table.sim @@ -657,36 +657,33 @@ if $data20 != null then return -1 endi -#print =============== error for normal table -#sql create table tb2023(ts timestamp, f int); -#sql_error alter table tb2023 add column v varchar(65535); -#sql_error alter table tb2023 add column v varchar(65535); -#sql_error alter table tb2023 add column v varchar(65530); -#sql alter table tb2023 add column v varchar(16374); -#sql_error alter table tb2023 modify column v varchar(65536); -#sql desc tb2023 -#sql alter table tb2023 drop column v -#sql_error alter table tb2023 add column v nchar(16384); -#sql alter table tb2023 add column v nchar(4093); -#sql_error alter table tb2023 modify column v nchar(16384); -#sql_error alter table tb2023 add column v nchar(16384); -#sql alter table tb2023 drop column v -#sql alter table tb2023 add column v nchar(16374); -#sql desc tb2023 -# -#print =============== error for super table -#sql create table stb2023(ts timestamp, f int) tags(t1 int); -#sql_error alter table stb2023 add column v varchar(65535); -#sql_error alter table stb2023 add column v varchar(65536); -#sql_error alter table stb2023 add column v varchar(33100); -#sql alter table stb2023 add column v varchar(16374); -#sql_error alter table stb2023 modify column v varchar(16375); -#sql desc stb2023 -#sql alter table stb2023 drop column v -#sql_error alter table stb2023 add column v nchar(4094); -#sql alter table stb2023 add column v nchar(4093); -#sql_error alter table stb2023 modify column v nchar(4094); -#sql desc stb2023 +print =============== error for normal table +sql create table tb2023(ts timestamp, f int); +sql_error alter table tb2023 add column v varchar(65518); +sql_error alter table tb2023 add column v varchar(65531); +sql_error alter table tb2023 add column v varchar(65535); +sql alter table tb2023 add column v varchar(65517); +sql_error alter table tb2023 modify column v varchar(65518); +sql desc tb2023 +sql alter table tb2023 drop column v +sql_error alter table tb2023 add column v nchar(16380); +sql alter table tb2023 add column v nchar(16379); +sql_error alter table tb2023 modify column v nchar(16380); +sql desc tb2023 + +print =============== error for super table +sql create table stb2023(ts timestamp, f int) tags(t1 int); +sql_error alter table stb2023 add column v varchar(65518); +sql_error alter table stb2023 add column v varchar(65531); +sql_error alter table stb2023 add column v varchar(65535); +sql alter table stb2023 add column v varchar(65517); +sql_error alter table stb2023 modify column v varchar(65518); +sql desc stb2023 +sql alter table stb2023 drop column v +sql_error alter table stb2023 add column v nchar(16380); +sql alter table stb2023 add column v nchar(16379); +sql_error alter table stb2023 modify column v nchar(16380); +sql desc stb2023 print ======= over sql drop database d1 diff --git a/tests/script/tsim/compute/last_row.sim b/tests/script/tsim/compute/last_row.sim index 2e060dc28526dba7f0a692046b9d987be861cebb..8e62fbffb5101fc2c12c885565a2506bcb84a350 100644 --- a/tests/script/tsim/compute/last_row.sim +++ b/tests/script/tsim/compute/last_row.sim @@ -213,4 +213,54 @@ if $rows != 2 then return -1 endi +print =======================> regresss bug in last_row query +sql drop database if exists db; +sql create database if not exists db vgroups 1 cachemodel 'both'; +sql create table db.stb (ts timestamp, c0 bigint) tags(t1 int); +sql insert into db.stb_0 using db.stb tags(1) values ('2023-11-23 19:06:40.000', 491173569); +sql insert into db.stb_2 using db.stb tags(3) values ('2023-11-25 19:30:00.000', 2080726142); +sql insert into db.stb_3 using db.stb tags(4) values ('2023-11-26 06:48:20.000', 1907405128); +sql insert into db.stb_4 using db.stb tags(5) values ('2023-11-24 22:56:40.000', 220783803); + +sql create table db.stb_1 using db.stb tags(2); +sql insert into db.stb_1 (ts) values('2023-11-26 13:11:40.000'); +sql insert into db.stb_1 (ts, c0) values('2023-11-26 13:11:39.000', 11); + +sql select tbname,ts,last_row(c0) from db.stb; +if $rows != 1 then + return -1 +endi + +if $data00 != @stb_1@ then + return -1 +endi + +if $data01 != @23-11-26 13:11:40.000@ then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +sql alter database db cachemodel 'none'; +sql reset query cache; +sql select tbname,last_row(c0, ts) from db.stb; + +if $rows != 1 then + return -1 +endi + +if $data00 != @stb_1@ then + return -1 +endi + +if $data02 != @23-11-26 13:11:40.000@ then + return -1 +endi + +if $data01 != NULL then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index f20f861bd0d40df47ab9c6e1792651410678f515..6c98d43794d5bb9a692e9a52e6a341e7e53df9f9 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -111,18 +111,12 @@ endi if $data21_db != 1000 then # wal_level fsyncperiod return -1 endi -if $data22_db != 0 then # wal_retention_period +if $data22_db != 3600 then # wal_retention_period return -1 endi if $data23_db != 0 then # wal_retention_size return -1 endi -if $data24_db != 0 then # wal_roll_period - return -1 -endi -if $data25_db != 0 then # wal_segment_size - return -1 -endi #sql show db.vgroups #if $data[0][4] == leader then diff --git a/tests/script/tsim/dnode/drop_dnode_force.sim b/tests/script/tsim/dnode/drop_dnode_force.sim index bce51a98887dd4aa9bfe6a4bb444424fdf8b006d..dbe34ab52960f8694f5a0299944aa7f1c033b2ec 100644 --- a/tests/script/tsim/dnode/drop_dnode_force.sim +++ b/tests/script/tsim/dnode/drop_dnode_force.sim @@ -209,7 +209,8 @@ endi print =============== step5a: drop dnode 3 sql_error drop dnode 3 -sql drop dnode 3 force +sql_error drop dnode 3 force +sql drop dnode 3 unsafe print select * from information_schema.ins_dnodes; sql select * from information_schema.ins_dnodes; diff --git a/tests/script/tsim/dnode/drop_dnode_has_mnode.sim b/tests/script/tsim/dnode/drop_dnode_has_mnode.sim index 054f9786072f002ff8a7cce05104e879a9f3e552..8a7436772668bc4846180ccedd34e3ad31550119 100644 --- a/tests/script/tsim/dnode/drop_dnode_has_mnode.sim +++ b/tests/script/tsim/dnode/drop_dnode_has_mnode.sim @@ -35,7 +35,7 @@ endi print =============== step2 drop dnode 3 sql_error drop dnode 1 -sql drop dnode 3 +sql drop dnode 3 force sql select * from information_schema.ins_dnodes print ===> $data00 $data01 $data02 $data03 $data04 $data05 diff --git a/tests/script/tsim/dnode/offline_reason.sim b/tests/script/tsim/dnode/offline_reason.sim index 8c4d8b47f765f22f07e8f5114797c94dc89c67ca..23f015392b943f56d1c93bceb57d0700ee9e945c 100644 --- a/tests/script/tsim/dnode/offline_reason.sim +++ b/tests/script/tsim/dnode/offline_reason.sim @@ -57,7 +57,7 @@ if $data(2)[7] != @status msg timeout@ then endi print ========== step4 -sql drop dnode 2 +sql drop dnode 2 force sql select * from information_schema.ins_dnodes if $rows != 1 then return -1 diff --git a/tests/script/tsim/parser/alter_column.sim b/tests/script/tsim/parser/alter_column.sim index f89211573596ae02f0a484840cc7679fb1d58a34..2bf369b91054c3953406fc5116a88c94f508aafb 100644 --- a/tests/script/tsim/parser/alter_column.sim +++ b/tests/script/tsim/parser/alter_column.sim @@ -48,7 +48,7 @@ sql_error alter table tb modify column c2 binary(10); sql_error alter table tb modify column c2 binary(9); sql_error alter table tb modify column c2 binary(-9); sql_error alter table tb modify column c2 binary(0); -sql_error alter table tb modify column c2 binary(65600); +sql_error alter table tb modify column c2 binary(65436); sql_error alter table tb modify column c2 nchar(30); sql_error alter table tb modify column c3 double; sql_error alter table tb modify column c3 nchar(10); diff --git a/tests/script/tsim/parser/condition_scl.sim b/tests/script/tsim/parser/condition_scl.sim new file mode 100644 index 0000000000000000000000000000000000000000..f377988006d8136f95c6ea458eec3e19e9df4ba0 --- /dev/null +++ b/tests/script/tsim/parser/condition_scl.sim @@ -0,0 +1,136 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c filterScalarMode -v 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists cdb +sql create database if not exists cdb +sql use cdb +sql create table stb1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb1 using stb1 tags(1,'1',1.0) +sql create table tb2 using stb1 tags(2,'2',2.0) +sql create table tb3 using stb1 tags(3,'3',3.0) +sql create table tb4 using stb1 tags(4,'4',4.0) +sql create table tb5 using stb1 tags(5,'5',5.0) +sql create table tb6 using stb1 tags(6,'6',6.0) + +sql insert into tb1 values ('2021-05-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1') +sql insert into tb1 values ('2021-05-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2') +sql insert into tb1 values ('2021-05-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3') +sql insert into tb1 values ('2021-05-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4') +sql insert into tb1 values ('2021-05-05 18:19:04',11,11.0,11,11,11,11.0,true ,'11','11') +sql insert into tb1 values ('2021-05-05 18:19:05',12,12.0,12,12,12,12.0,true ,'12','12') +sql insert into tb1 values ('2021-05-05 18:19:06',13,13.0,13,13,13,13.0,false,'13','13') +sql insert into tb1 values ('2021-05-05 18:19:07',14,14.0,14,14,14,14.0,false,'14','14') +sql insert into tb2 values ('2021-05-05 18:19:08',21,21.0,21,21,21,21.0,true ,'21','21') +sql insert into tb2 values ('2021-05-05 18:19:09',22,22.0,22,22,22,22.0,true ,'22','22') +sql insert into tb2 values ('2021-05-05 18:19:10',23,23.0,23,23,23,23.0,false,'23','23') +sql insert into tb2 values ('2021-05-05 18:19:11',24,24.0,24,24,24,24.0,false,'24','24') +sql insert into tb3 values ('2021-05-05 18:19:12',31,31.0,31,31,31,31.0,true ,'31','31') +sql insert into tb3 values ('2021-05-05 18:19:13',32,32.0,32,32,32,32.0,true ,'32','32') +sql insert into tb3 values ('2021-05-05 18:19:14',33,33.0,33,33,33,33.0,false,'33','33') +sql insert into tb3 values ('2021-05-05 18:19:15',34,34.0,34,34,34,34.0,false,'34','34') +sql insert into tb4 values ('2021-05-05 18:19:16',41,41.0,41,41,41,41.0,true ,'41','41') +sql insert into tb4 values ('2021-05-05 18:19:17',42,42.0,42,42,42,42.0,true ,'42','42') +sql insert into tb4 values ('2021-05-05 18:19:18',43,43.0,43,43,43,43.0,false,'43','43') +sql insert into tb4 values ('2021-05-05 18:19:19',44,44.0,44,44,44,44.0,false,'44','44') +sql insert into tb5 values ('2021-05-05 18:19:20',51,51.0,51,51,51,51.0,true ,'51','51') +sql insert into tb5 values ('2021-05-05 18:19:21',52,52.0,52,52,52,52.0,true ,'52','52') +sql insert into tb5 values ('2021-05-05 18:19:22',53,53.0,53,53,53,53.0,false,'53','53') +sql insert into tb5 values ('2021-05-05 18:19:23',54,54.0,54,54,54,54.0,false,'54','54') +sql insert into tb6 values ('2021-05-05 18:19:24',61,61.0,61,61,61,61.0,true ,'61','61') +sql insert into tb6 values ('2021-05-05 18:19:25',62,62.0,62,62,62,62.0,true ,'62','62') +sql insert into tb6 values ('2021-05-05 18:19:26',63,63.0,63,63,63,63.0,false,'63','63') +sql insert into tb6 values ('2021-05-05 18:19:27',64,64.0,64,64,64,64.0,false,'64','64') +sql insert into tb6 values ('2021-05-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql create table stb2 (ts timestamp, u1 int unsigned, u2 bigint unsigned, u3 smallint unsigned, u4 tinyint unsigned, ts2 timestamp) TAGS(t1 int unsigned, t2 bigint unsigned, t3 timestamp, t4 int) +sql create table tb2_1 using stb2 tags(1,1,'2021-05-05 18:38:38',1) +sql create table tb2_2 using stb2 tags(2,2,'2021-05-05 18:58:58',2) + +sql insert into tb2_1 values ('2021-05-05 18:19:00',1,2,3,4,'2021-05-05 18:28:01') +sql insert into tb2_1 values ('2021-05-05 18:19:01',5,6,7,8,'2021-05-05 18:28:02') +sql insert into tb2_1 values ('2021-05-05 18:19:02',2,2,3,4,'2021-05-05 18:28:03') +sql insert into tb2_1 values ('2021-05-05 18:19:03',5,6,7,8,'2021-05-05 18:28:04') +sql insert into tb2_1 values ('2021-05-05 18:19:04',3,2,3,4,'2021-05-05 18:28:05') +sql insert into tb2_1 values ('2021-05-05 18:19:05',5,6,7,8,'2021-05-05 18:28:06') +sql insert into tb2_1 values ('2021-05-05 18:19:06',4,2,3,4,'2021-05-05 18:28:07') +sql insert into tb2_1 values ('2021-05-05 18:19:07',5,6,7,8,'2021-05-05 18:28:08') +sql insert into tb2_1 values ('2021-05-05 18:19:08',5,2,3,4,'2021-05-05 18:28:09') +sql insert into tb2_1 values ('2021-05-05 18:19:09',5,6,7,8,'2021-05-05 18:28:10') +sql insert into tb2_1 values ('2021-05-05 18:19:10',6,2,3,4,'2021-05-05 18:28:11') +sql insert into tb2_2 values ('2021-05-05 18:19:11',5,6,7,8,'2021-05-05 18:28:12') +sql insert into tb2_2 values ('2021-05-05 18:19:12',7,2,3,4,'2021-05-05 18:28:13') +sql insert into tb2_2 values ('2021-05-05 18:19:13',5,6,7,8,'2021-05-05 18:28:14') +sql insert into tb2_2 values ('2021-05-05 18:19:14',8,2,3,4,'2021-05-05 18:28:15') +sql insert into tb2_2 values ('2021-05-05 18:19:15',5,6,7,8,'2021-05-05 18:28:16') + +sql create table stb3 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb3_1 using stb3 tags(1,'1',1.0) +sql create table tb3_2 using stb3 tags(2,'2',2.0) + +sql insert into tb3_1 values ('2021-01-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1') +sql insert into tb3_1 values ('2021-02-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2') +sql insert into tb3_1 values ('2021-03-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3') +sql insert into tb3_1 values ('2021-04-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4') +sql insert into tb3_1 values ('2021-05-05 18:19:28',5,NULL,5,NULL,5,NULL,true,NULL,'5') +sql insert into tb3_1 values ('2021-06-05 18:19:28',NULL,6.0,NULL,6,NULL,6.0,NULL,'6',NULL) +sql insert into tb3_1 values ('2021-07-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) +sql insert into tb3_2 values ('2021-01-06 18:19:00',11,11.0,11,11,11,11.0,true ,'11','11') +sql insert into tb3_2 values ('2021-02-06 18:19:01',12,12.0,12,12,12,12.0,true ,'12','12') +sql insert into tb3_2 values ('2021-03-06 18:19:02',13,13.0,13,13,13,13.0,false,'13','13') +sql insert into tb3_2 values ('2021-04-06 18:19:03',14,14.0,14,14,14,14.0,false,'14','14') +sql insert into tb3_2 values ('2021-05-06 18:19:28',15,NULL,15,NULL,15,NULL,true,NULL,'15') +sql insert into tb3_2 values ('2021-06-06 18:19:28',NULL,16.0,NULL,16,NULL,16.0,NULL,'16',NULL) +sql insert into tb3_2 values ('2021-07-06 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql create table stb4 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9),c10 binary(16300)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb4_0 using stb4 tags(0,'0',0.0) +sql create table tb4_1 using stb4 tags(1,'1',1.0) +sql create table tb4_2 using stb4 tags(2,'2',2.0) +sql create table tb4_3 using stb4 tags(3,'3',3.0) +sql create table tb4_4 using stb4 tags(4,'4',4.0) + +$i = 0 +$ts0 = 1625850000000 +$blockNum = 5 +$delta = 0 +$tbname0 = tb4_ +$a = 0 +$b = 200 +$c = 400 +while $i < $blockNum + $x = 0 + $rowNum = 1200 + while $x < $rowNum + $ts = $ts0 + $x + $a = $a + 1 + $b = $b + 1 + $c = $c + 1 + $d = $x / 10 + $tin = $rowNum + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + $tbname = 'tb4_ . $i + $tbname = $tbname . ' + sql insert into $tbname values ( $ts , $a , $b , $c , $d , $d , $c , true, $binary , $nchar , $binary ) + $x = $x + 1 + endw + + $i = $i + 1 + $ts0 = $ts0 + 259200000 +endw + +run tsim/parser/condition_query.sim + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql connect + +run tsim/parser/condition_query.sim + diff --git a/tests/script/tsim/parser/fill.sim b/tests/script/tsim/parser/fill.sim index 0510f80419413d893c5172e34b441dd25de0fb9b..0534aa5d5b762c24bb50bba9c52b6733a2d7b429 100644 --- a/tests/script/tsim/parser/fill.sim +++ b/tests/script/tsim/parser/fill.sim @@ -1143,4 +1143,185 @@ if $rows != 20026 then return -1 endi +print ===================== TD-25209 test fill prev/next/linear after data range +sql use $db + +sql select _wstart,_wend,count(*) from tm0 where ts >= '2020-01-01 01:03:06.000' and ts <= '2020-01-01 01:03:10.000' interval(1s) fill(prev); + +if $rows != 5 then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +if $data12 != 1 then + return -1 +endi + +if $data22 != 1 then + return -1 +endi + +if $data32 != 1 then + return -1 +endi + +if $data42 != 1 then + return -1 +endi + +sql select _wstart,_wend,count(*) from tm0 where ts >= '2020-01-01 01:03:06.000' and ts <= '2020-01-01 01:03:10.000' interval(1s) fill(next); + +if $rows != 5 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +if $data12 != 1 then + return -1 +endi + +if $data22 != 1 then + return -1 +endi + +if $data32 != 1 then + return -1 +endi + +if $data42 != NULL then + return -1 +endi + +sql select _wstart,_wend,count(*) from tm0 where ts >= '2020-01-01 01:03:06.000' and ts <= '2020-01-01 01:03:10.000' interval(1s) fill(linear); + +if $rows != 5 then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +if $data12 != 1 then + return -1 +endi + +if $data22 != 1 then + return -1 +endi + +if $data32 != 1 then + return -1 +endi + +if $data42 != NULL then + return -1 +endi + +print ===================== TD-3625 test fill value NULL +sql use $db + +sql select _wstart,_wend,count(*) from tm0 where ts >= '2020-01-01 01:03:06.000' and ts <= '2020-01-01 01:03:10.000' interval(1s) fill(value, NULL); + +if $rows != 5 then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +if $data12 != 1 then + return -1 +endi + +if $data22 != 1 then + return -1 +endi + +if $data32 != 1 then + return -1 +endi + +if $data42 != NULL then + return -1 +endi + +sql select _wstart,_wend,count(*),sum(k),avg(k) from tm0 where ts >= '2020-01-01 01:03:06.000' and ts <= '2020-01-01 01:03:10.000' interval(1s) fill(value, 1, NULL, 1); + +if $rows != 5 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +if $data12 != 1 then + return -1 +endi + +if $data22 != 1 then + return -1 +endi + +if $data32 != 1 then + return -1 +endi + +if $data42 != 1 then + return -1 +endi + + +if $data03 != NULL then + return -1 +endi + +if $data13 != 7 then + return -1 +endi + +if $data23 != 8 then + return -1 +endi + +if $data33 != 9 then + return -1 +endi + +if $data43 != NULL then + return -1 +endi + + +if $data04 != 1.000000000 then + return -1 +endi + +if $data14 != 7.000000000 then + return -1 +endi + +if $data24 != 8.000000000 then + return -1 +endi + +if $data34 != 9.000000000 then + return -1 +endi + +if $data44 != 1.000000000 then + return -1 +endi + + system sh/exec.sh -n dnode1 -s stop -x SIGINT + + diff --git a/tests/script/tsim/parser/function.sim b/tests/script/tsim/parser/function.sim index 7f69aa2d02adeaa2cd64eafbefaa2d2df3a79184..120c4b8148d9eef6e423f0ff0a06ac5af5e88559 100644 --- a/tests/script/tsim/parser/function.sim +++ b/tests/script/tsim/parser/function.sim @@ -954,11 +954,14 @@ endi print =========================>TD-5190 sql select _wstart, stddev(f1) from st1 where ts>'2021-07-01 1:1:1' and ts<'2021-07-30 00:00:00' interval(1d) fill(NULL); if $rows != 29 then + print expect 29, actual: $rows return -1 endi + if $data00 != @21-07-01 00:00:00.000@ then return -1 endi + if $data01 != NULL then return -1 endi diff --git a/tests/script/tsim/parser/join_manyblocks.sim b/tests/script/tsim/parser/join_manyblocks.sim index a40a75f50c54cb9ffdfed24edd2a7e03e0c13e5c..7fd0df21b31b598b54dbeaa3e7780a1e9c0ea010 100644 --- a/tests/script/tsim/parser/join_manyblocks.sim +++ b/tests/script/tsim/parser/join_manyblocks.sim @@ -6,8 +6,8 @@ sql connect $dbPrefix = join_m_db $tbPrefix = join_tb $mtPrefix = join_mt -$tbNum = 3 -$rowNum = 2000 +$tbNum = 20 +$rowNum = 200 $totalNum = $tbNum * $rowNum print =============== join_manyBlocks.sim @@ -78,8 +78,8 @@ print ==============> td-3313 sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1; print $row -if $row != 6000 then - print expect 6000, actual: $row +if $row != 4000 then + print expect 4000, actual: $row return -1 endi diff --git a/tests/script/tsim/parser/limit1_stb.sim b/tests/script/tsim/parser/limit1_stb.sim index 731a218de5d11f11b623a8348344597fb84f78b8..027a4f5c797133979a90a352f5d2b58fd67094ff 100644 --- a/tests/script/tsim/parser/limit1_stb.sim +++ b/tests/script/tsim/parser/limit1_stb.sim @@ -468,7 +468,7 @@ if $data01 != 1 then endi ## supertable aggregation + where + interval + group by order by tag + limit offset -sql select _wstart, max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9),t1 from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 5 and c1 > 0 and c2 < 9 and c3 > 1 and c4 < 7 and c5 > 4 partition by t1 interval(5m) order by t1 desc limit 2 offset 0 +sql select _wstart, max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9),t1 from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 5 and c1 > 0 and c2 < 9 and c3 > 1 and c4 < 7 and c5 > 4 partition by t1 interval(5m) order by t1 desc, max(c1) asc limit 2 offset 0 if $rows != 2 then return -1 endi diff --git a/tests/script/tsim/parser/limit_stb.sim b/tests/script/tsim/parser/limit_stb.sim index 6950df9ee1b41816feca5c8753efd14489fda063..46bd6260c3b8c726e24e8e07ab0d88f94872a5e9 100644 --- a/tests/script/tsim/parser/limit_stb.sim +++ b/tests/script/tsim/parser/limit_stb.sim @@ -508,7 +508,7 @@ endi ### supertable aggregation + where + interval + group by order by tag + limit offset ## TBASE-345 -sql select _wstart, max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9), t1 from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 5 and c1 > 0 and c2 < 9 and c3 > 1 and c4 < 7 and c5 > 4 partition by t1 interval(5m) order by t1 desc limit 3 offset 0 +sql select _wstart, max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9), t1 from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 5 and c1 > 0 and c2 < 9 and c3 > 1 and c4 < 7 and c5 > 4 partition by t1 interval(5m) order by t1 desc, max(c1) asc limit 3 offset 0 if $rows != 3 then return -1 endi @@ -554,7 +554,7 @@ if $data09 != 4 then return -1 endi -sql select _wstart, max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9), t1 from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 8 and c1 > 0 and c2 < 9 and c3 > 4 and c4 < 7 and c5 > 4 partition by t1 interval(5m) order by t1 desc limit 3 offset 0 +sql select _wstart, max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9), t1 from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 8 and c1 > 0 and c2 < 9 and c3 > 4 and c4 < 7 and c5 > 4 partition by t1 interval(5m) order by t1 desc, max(c1) asc limit 3 offset 0 if $rows != 3 then return -1 endi diff --git a/tests/script/tsim/parser/projection_limit_offset.sim b/tests/script/tsim/parser/projection_limit_offset.sim index 2d99b0a296621ca0b29f908dba8714afde47076c..cab46a93d3cb5600d4f6c9a0ec196874ded8e93c 100644 --- a/tests/script/tsim/parser/projection_limit_offset.sim +++ b/tests/script/tsim/parser/projection_limit_offset.sim @@ -380,10 +380,10 @@ if $row != 8 then endi sql select diff(k) from tm0 -if $row != 3 then +if $row != 4 then return -1 endi -if $data20 != -1 then +if $data20 != NULL then return -1 endi diff --git a/tests/script/tsim/parser/sliding.sim b/tests/script/tsim/parser/sliding.sim index 1cb4cb5340f993ee25b4f73e8b076375ea28a445..7aa69ce9a9c164b8741ba40e607387e8ba184b30 100644 --- a/tests/script/tsim/parser/sliding.sim +++ b/tests/script/tsim/parser/sliding.sim @@ -450,10 +450,11 @@ endi print ====================>check boundary check crash at client side sql select count(*) from sliding_mt0 where ts>now and ts < now-1h; +sql select sum(c1) from sliding_tb0 interval(1a) sliding(1a); + print ========================query on super table print ========================error case -sql_error select sum(c1) from sliding_tb0 interval(1a) sliding(1a); sql_error select sum(c1) from sliding_tb0 interval(10a) sliding(12a); sql_error select sum(c1) from sliding_tb0 sliding(1n) interval(1y); sql_error select sum(c1) from sliding_tb0 interval(-1y) sliding(1n); diff --git a/tests/script/tsim/parser/union.sim b/tests/script/tsim/parser/union.sim index dee5da96e8488a14cc615d7ab3944ccf6558f93e..f0c534ad11b336cade9d79e2f652742ed2dfbd86 100644 --- a/tests/script/tsim/parser/union.sim +++ b/tests/script/tsim/parser/union.sim @@ -126,7 +126,6 @@ endi if $data10 != 1 then return -1 endi - sql (select 'ab' as options from union_tb1 limit 1) union all (select 'dd' as options from union_tb0 limit 1) order by options; if $rows != 2 then return -1 diff --git a/tests/script/tsim/query/bug3398.sim b/tests/script/tsim/query/bug3398.sim new file mode 100644 index 0000000000000000000000000000000000000000..3ca88cf4596fcc8e732eb9ef94b3e1cdb8fc45de --- /dev/null +++ b/tests/script/tsim/query/bug3398.sim @@ -0,0 +1,30 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create database +sql create database test + +print =============== create super table and child table +sql use test + +sql CREATE STABLE st (day timestamp, c2 int) TAGS (vin binary(32)) + +sql insert into test.g using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) +sql insert into test.t using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) +sql insert into test.tg using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) + +sql select sum(case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end), sum(case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end) from test.t t, test.g g, test.tg tg where t.day = g.day and t.day = tg.day and t.day between '2021-05-03' and '2023-05-04' and t.vin = 'TAG1' and t.vin = g.vin and t.vin = tg.vin group by t.day; + +print $rows $data00 $data01 +if $rows != 1 then + return -1 +endi +if $data00 != 2.000000000 then + return -1 +endi + +if $data01 != 3.000000000 then + return -1 +endi diff --git a/tests/script/tsim/query/delete_and_query.sim b/tests/script/tsim/query/delete_and_query.sim new file mode 100644 index 0000000000000000000000000000000000000000..3004ababa1c79d1cbc90925da0af8a0f3b1fb427 --- /dev/null +++ b/tests/script/tsim/query/delete_and_query.sim @@ -0,0 +1,25 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database if not exists test +sql use test +sql create table t1 (ts timestamp, c2 int) +sql insert into t1 values(now, 1) + +sql delete from t1 where ts is null +sql delete from t1 where ts < now +sql select ts from t1 order by ts asc + +print ----------rows: $rows +if $rows != 0 then + return -1 +endi + +sql select ts from t1 order by ts desc +print ----------rows: $rows +if $rows != 0 then + return -1 +endi + diff --git a/tests/script/tsim/query/emptyTsRange_scl.sim b/tests/script/tsim/query/emptyTsRange_scl.sim new file mode 100644 index 0000000000000000000000000000000000000000..43734b047dd6aaec0ac5b3f8a048da30f12d26ad --- /dev/null +++ b/tests/script/tsim/query/emptyTsRange_scl.sim @@ -0,0 +1,21 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c filterScalarMode -v 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists db1; +sql create database if not exists db1; +sql use db1; +sql create stable sta (ts timestamp, f1 double, f2 binary(200)) tags(t1 int); +sql create table tba1 using sta tags(1); +sql insert into tba1 values ('2022-04-26 15:15:01', 1.0, "a"); +sql insert into tba1 values ('2022-04-26 15:15:02', 2.0, "b"); +sql insert into tba1 values ('2022-04-26 15:15:04', 4.0, "b"); +sql insert into tba1 values ('2022-04-26 15:15:05', 5.0, "b"); +sql select last_row(*) from sta where ts >= 1678901803783 and ts <= 1678901803783 and _c0 <= 1678901803782 interval(10d,8d) fill(linear) order by _wstart desc; +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/explain_tsorder.sim b/tests/script/tsim/query/explain_tsorder.sim new file mode 100644 index 0000000000000000000000000000000000000000..202f85bcf0745e883b6ea1e58094ea3dbed64a3c --- /dev/null +++ b/tests/script/tsim/query/explain_tsorder.sim @@ -0,0 +1,37 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database test +sql use test +sql CREATE STABLE `meters` (`ts` TIMESTAMP, `c2` INT) TAGS (`cc` VARCHAR(3)) + + +sql insert into d1 using meters tags("MY") values("2022-05-15 00:01:08.000 ",234) +sql insert into d1 using meters tags("MY") values("2022-05-16 00:01:08.000 ",136) +sql insert into d1 using meters tags("MY") values("2022-05-17 00:01:08.000 ", 59) +sql insert into d1 using meters tags("MY") values("2022-05-18 00:01:08.000 ", 58) +sql insert into d1 using meters tags("MY") values("2022-05-19 00:01:08.000 ",243) +sql insert into d1 using meters tags("MY") values("2022-05-20 00:01:08.000 ",120) +sql insert into d1 using meters tags("MY") values("2022-05-21 00:01:08.000 ", 11) +sql insert into d1 using meters tags("MY") values("2022-05-22 00:01:08.000 ",196) +sql insert into d1 using meters tags("MY") values("2022-05-23 00:01:08.000 ",116) +sql insert into d1 using meters tags("MY") values("2022-05-24 00:01:08.000 ",210) + +sql insert into d2 using meters tags("HT") values("2022-05-15 00:01:08.000", 234) +sql insert into d2 using meters tags("HT") values("2022-05-16 00:01:08.000", 136) +sql insert into d2 using meters tags("HT") values("2022-05-17 00:01:08.000", 59) +sql insert into d2 using meters tags("HT") values("2022-05-18 00:01:08.000", 58) +sql insert into d2 using meters tags("HT") values("2022-05-19 00:01:08.000", 243) +sql insert into d2 using meters tags("HT") values("2022-05-20 00:01:08.000", 120) +sql insert into d2 using meters tags("HT") values("2022-05-21 00:01:08.000", 11) +sql insert into d2 using meters tags("HT") values("2022-05-22 00:01:08.000", 196) +sql insert into d2 using meters tags("HT") values("2022-05-23 00:01:08.000", 116) +sql insert into d2 using meters tags("HT") values("2022-05-24 00:01:08.000", 210) + +#sleep 10000000 +system taos -P7100 -s 'source tsim/query/t/explain_tsorder.sql' | grep -v 'Query OK' | grep -v 'Client Version' > /tmp/explain_tsorder.result +system echo ----------------------diff start----------------------- +system git diff --exit-code --color tsim/query/r/explain_tsorder.result /tmp/explain_tsorder.result +system echo ----------------------diff succeed----------------------- diff --git a/tests/script/tsim/query/groupby_distinct.sim b/tests/script/tsim/query/groupby_distinct.sim new file mode 100644 index 0000000000000000000000000000000000000000..8b16bb1910e565d3bc586a6610cef6c0d7b25e4e --- /dev/null +++ b/tests/script/tsim/query/groupby_distinct.sim @@ -0,0 +1,30 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + + +sql drop database if exists db1; +sql create database db1; +sql use db1; + +sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create table tba1 using sta tags(1, 1, 1); +sql insert into tba1 values ('2022-04-26 15:15:08', 1, "a"); +sql insert into tba1 values ('2022-04-26 15:15:07', 1, "b"); +sql insert into tba1 values ('2022-04-26 15:15:06', 1, "a"); +sql insert into tba1 values ('2022-04-26 15:15:05', 1, "b"); +sql insert into tba1 values ('2022-04-26 15:15:04', 1, "c"); +sql insert into tba1 values ('2022-04-26 15:15:03', 1, "c"); +sql insert into tba1 values ('2022-04-26 15:15:02', 1, "d"); +sql insert into tba1 values ('2022-04-26 15:15:01', 1, "d"); +sql select distinct avg(f1) as avgv from sta group by f2; +if $rows != 1 then + return -1 +endi +sql select distinct avg(f1) as avgv from sta group by f2 limit 1,10; +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/interval.sim b/tests/script/tsim/query/interval.sim index 833da4a8ba2b3daf495167f06c99d222564a6bf3..e2b0d219cb0eaaace3e571d31a19fa329364c575 100644 --- a/tests/script/tsim/query/interval.sim +++ b/tests/script/tsim/query/interval.sim @@ -170,6 +170,42 @@ endi # return -1 #endi +print ================ step10 +print -------- create database and table +sql create database if not exists test +sql use test +sql create stable st (ts timestamp, c2 int) tags(tg int) +print -------- insert 300 rows data +$i = 0 +while $i < 300 + $t = 1577807983000 + $cc = $i * 1000 + $t = $t + $cc + sql select $i % 3 + if $data00 != 0.000000000 then + goto continue_while + endi + sql select $i % 4 + if $data00 == 0.000000000 then + goto continue_while + endi + sql insert into t1 using st tags(1) values ( $t , $i ) + continue_while: + $i = $i + 1 +endw + +$ms1 = 1577808120000 +$ms2 = 1577808000000 +sql select * from (select _wstart, last(ts) as ts, avg(c2) as av from t1 where ts <= $ms1 and ts >= $ms2 interval(10s) sliding(1s) fill(NULL)) order by ts asc +print ----> select asc rows: $rows +$asc_rows = $rows +sql select * from (select _wstart, last(ts) as ts, avg(c2) as av from t1 where ts <= $ms1 and ts >= $ms2 interval(10s) sliding(1s) fill(NULL)) order by ts desc +print ----> select desc rows: $rows +$desc_rows = $rows +if $desc_rows != $asc_rows then + return -1 +endi + print =============== clear #sql drop database $db #sql select * from information_schema.ins_databases diff --git a/tests/script/tsim/query/join_pk.sim b/tests/script/tsim/query/join_pk.sim new file mode 100644 index 0000000000000000000000000000000000000000..4bb015ded115fc423f4cf4ec6f2014096f384db8 --- /dev/null +++ b/tests/script/tsim/query/join_pk.sim @@ -0,0 +1,56 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database test; +sql use test; +sql create table st(ts timestamp, f int) tags(t int); +sql insert into ct1 using st tags(1) values(now, 0)(now+1s, 1) +sql insert into ct2 using st tags(2) values(now+2s, 2)(now+3s, 3) +sql select * from (select _wstart - 1s as ts, count(*) as num1 from st interval(1s)) as t1 inner join (select _wstart as ts, count(*) as num2 from st interval(1s)) as t2 on t1.ts = t2.ts + +if $rows != 3 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data11 != 1 then + return -1 +endi + +if $data21 != 1 then + return -1 +endi +if $data03 != 1 then + return -1 +endi + +if $data13 != 1 then + return -1 +endi +if $data23 != 1 then + return -1 +endi +sql select * from (select _wstart - 1d as ts, count(*) as num1 from st interval(1s)) as t1 inner join (select _wstart as ts, count(*) as num2 from st interval(1s)) as t2 on t1.ts = t2.ts + +sql select * from (select _wstart + 1a as ts, count(*) as num1 from st interval(1s)) as t1 inner join (select _wstart as ts, count(*) as num2 from st interval(1s)) as t2 on t1.ts = t2.ts + +sql_error select * from (select _wstart * 3 as ts, count(*) as num1 from st interval(1s)) as t1 inner join (select _wstart as ts, count(*) as num2 from st interval(1s)) as t2 on t1.ts = t2.ts + +sql create table sst(ts timestamp, ts2 timestamp, f int) tags(t int); +sql insert into sct1 using sst tags(1) values('2023-08-07 13:30:56', '2023-08-07 13:30:56', 0)('2023-08-07 13:30:57', '2023-08-07 13:30:57', 1) +sql insert into sct2 using sst tags(2) values('2023-08-07 13:30:58', '2023-08-07 13:30:58', 2)('2023-08-07 13:30:59', '2023-08-07 13:30:59', 3) +sql select * from (select ts - 1s as jts from sst) as t1 inner join (select ts-1s as jts from sst) as t2 on t1.jts = t2.jts +if $rows != 4 then + return -1 +endi +sql select * from (select ts - 1s as jts from sst) as t1 inner join (select ts as jts from sst) as t2 on t1.jts = t2.jts +if $rows != 3 then + return -1 +endi +sql_error select * from (select ts2 - 1s as jts from sst) as t1 inner join (select ts2 as jts from sst) as t2 on t1.jts = t2.jts + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT + diff --git a/tests/script/tsim/query/partitionby.sim b/tests/script/tsim/query/partitionby.sim index 4c221e02d39098ed078e25e5aa4739764f8c0fe9..76d4f87908590839cdec1e8689404be974d4e86d 100644 --- a/tests/script/tsim/query/partitionby.sim +++ b/tests/script/tsim/query/partitionby.sim @@ -24,18 +24,85 @@ sql use $db sql create table $mt1 (ts timestamp, f1 int) TAGS(tag1 int, tag2 binary(500)) sql create table tb0 using $mt1 tags(0, 'a'); sql create table tb1 using $mt1 tags(1, 'b'); -sql create table tb2 using $mt1 tags(1, 'a'); -sql create table tb3 using $mt1 tags(1, 'a'); -sql create table tb4 using $mt1 tags(3, 'b'); -sql create table tb5 using $mt1 tags(3, 'a'); -sql create table tb6 using $mt1 tags(3, 'b'); -sql create table tb7 using $mt1 tags(3, 'b'); +sql create table tb2 using $mt1 tags(2, 'a'); +sql create table tb3 using $mt1 tags(3, 'a'); +sql create table tb4 using $mt1 tags(4, 'b'); +sql create table tb5 using $mt1 tags(5, 'a'); +sql create table tb6 using $mt1 tags(6, 'b'); +sql create table tb7 using $mt1 tags(7, 'b'); sql select * from $mt1 partition by tag1,tag2 limit 1; if $rows != 0 then return -1 endi +sql insert into tb0 values ('2022-04-26 15:15:08', 1); +sql insert into tb1 values ('2022-04-26 15:15:07', 2); +sql insert into tb2 values ('2022-04-26 15:15:06', 3); +sql insert into tb3 values ('2022-04-26 15:15:05', 4); +sql insert into tb4 values ('2022-04-26 15:15:04', 5); +sql insert into tb5 values ('2022-04-26 15:15:03', 6); +sql insert into tb6 values ('2022-04-26 15:15:02', 7); +sql insert into tb7 values ('2022-04-26 15:15:01', 8); + +sql select _wstart as ts, count(*) from $mt1 partition by tag1 interval(1s) order by _wstart; +if $rows != 8 then + return -1 +endi +if $data00 != @22-04-26 15:15:01.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data10 != @22-04-26 15:15:02.000@ then + return -1 +endi +if $data11 != 1 then + return -1 +endi +if $data20 != @22-04-26 15:15:03.000@ then + return -1 +endi +if $data21 != 1 then + return -1 +endi +if $data30 != @22-04-26 15:15:04.000@ then + return -1 +endi +if $data31 != 1 then + return -1 +endi +if $data40 != @22-04-26 15:15:05.000@ then + return -1 +endi +if $data41 != 1 then + return -1 +endi +if $data50 != @22-04-26 15:15:06.000@ then + return -1 +endi +if $data51 != 1 then + return -1 +endi +if $data60 != @22-04-26 15:15:07.000@ then + return -1 +endi +if $data61 != 1 then + return -1 +endi +if $data70 != @22-04-26 15:15:08.000@ then + return -1 +endi +if $data71 != 1 then + return -1 +endi +sql select * from (select _wstart as ts, count(*) from $mt1 partition by tag1 interval(1s) order by _wstart) order by ts; +sql select _wstart as ts, count(*) from $mt1 interval(1s) order by _wstart; +sql select * from (select _wstart as ts, count(*) from $mt1 interval(1s) order by _wstart) order by ts; +sql select diff(a) from (select _wstart as ts, count(*) a from $mt1 interval(1s) order by _wstart); +sql select diff(a) from (select _wstart as ts, count(*) a from $mt1 partition by tag1 interval(1s) order by _wstart); + sql insert into tb0 values (now, 0); sql insert into tb1 values (now, 1); sql insert into tb2 values (now, 2); @@ -54,7 +121,7 @@ sql select count(*) from (select ts from $mt1 where ts is not null partition by if $rows != 1 then return -1 endi -if $data00 != 2 then +if $data00 != 4 then return -1 endi @@ -62,7 +129,7 @@ sql select count(*) from (select ts from $mt1 where ts is not null partition by if $rows != 1 then return -1 endi -if $data00 != 8 then +if $data00 != 16 then return -1 endi diff --git a/tests/script/tsim/query/r/explain_tsorder.result b/tests/script/tsim/query/r/explain_tsorder.result new file mode 100644 index 0000000000000000000000000000000000000000..b29a1b0a953b703d8b6b9493ea33e4f139516c6a --- /dev/null +++ b/tests/script/tsim/query/r/explain_tsorder.result @@ -0,0 +1,3960 @@ +Copyright (c) 2022 by TDengine, all rights reserved. + +taos> source tsim/query/t/explain_tsorder.sql +taos> use test; +Database changed. + +taos> explain verbose true select _wstart, last(ts), avg(c2) from meters interval(10s) order by _wstart desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=desc output_order=desc) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 4.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 5.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=desc output_order=desc) +*************************** 6.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 7.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 8.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, desc +*************************** 9.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 10.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 11.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=desc ) +*************************** 12.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 13.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 14.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 15.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 18.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 20.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=desc ) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 23.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 24.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 25.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 26.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, last(ts), avg(c2) from meters interval(10s) order by _wstart asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 4.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 5.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 6.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 7.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 8.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 9.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 10.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 11.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 12.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 13.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 14.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 15.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 18.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 20.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 23.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 24.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 25.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 26.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, first(ts), avg(c2) from meters interval(10s) order by _wstart asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 4.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 5.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 6.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 7.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 8.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 9.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 10.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 11.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=asc output_order=asc ) +*************************** 12.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 13.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 14.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 15.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 18.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 20.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=asc output_order=asc ) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 23.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 24.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 25.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 26.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, first(ts), avg(c2) from meters interval(10s) order by _wstart desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=desc output_order=desc) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 4.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 5.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=desc output_order=desc) +*************************** 6.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 7.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 8.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, desc +*************************** 9.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 10.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 11.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=asc output_order=desc ) +*************************** 12.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 13.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 14.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 15.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 18.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 20.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=asc output_order=desc ) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 23.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 24.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 25.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 26.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s)) order by d\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=asc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Aggragate (functions=1 width=8 input_order=asc ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 5.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 6.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=asc ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 10.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 13.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 14.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 16.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 17.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 18.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 20.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 23.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 24.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 25.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 26.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 27.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 28.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 29.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 31.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 32.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 33.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 34.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 35.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s)) order by d desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=desc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Aggragate (functions=1 width=8 input_order=asc ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 5.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 6.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=asc ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 10.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 13.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 14.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 16.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 17.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 18.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 20.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 23.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 24.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 25.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 26.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 27.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 28.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 29.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 31.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 32.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 33.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 34.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 35.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a) order by d\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=asc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Aggragate (functions=1 width=8 input_order=asc ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 5.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 6.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=asc ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 10.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 13.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 14.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 16.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 17.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 18.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 20.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 23.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 24.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 25.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 26.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 27.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 28.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 29.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 31.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 32.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 33.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 34.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 35.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) order by d\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=asc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Aggragate (functions=1 width=8 input_order=desc ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 5.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 6.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=desc ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 10.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=desc output_order=desc) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 13.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 14.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=desc output_order=desc) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 16.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 17.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, desc +*************************** 18.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 20.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=desc ) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 23.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 24.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 25.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 26.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 27.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 28.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 29.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=desc ) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 31.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 32.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 33.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 34.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 35.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a) order by d desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=desc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Aggragate (functions=1 width=8 input_order=asc ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 5.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 6.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=asc ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 10.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 13.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 14.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 16.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 17.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 18.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 20.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 23.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 24.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 25.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 26.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 27.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 28.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 29.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 31.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 32.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 33.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 34.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 35.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) order by d desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=desc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Aggragate (functions=1 width=8 input_order=desc ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 5.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 6.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=desc ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 10.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=desc output_order=desc) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 13.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 14.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=desc output_order=desc) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 16.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 17.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, desc +*************************** 18.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 20.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=desc ) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 23.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 24.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 25.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 26.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 27.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 28.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 29.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=desc ) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 31.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 32.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 33.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 34.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 35.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) order by d\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=asc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Aggragate (functions=1 width=8 input_order=unknown ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 5.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 6.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=unknown ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 10.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=3 width=24) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 14.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 15.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 16.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 17.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 18.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 19.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 20.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 23.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 24.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 25.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 26.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 27.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 28.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 29.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 31.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 32.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 33.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 34.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 35.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 36.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 37.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) order by d\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=asc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Aggragate (functions=1 width=8 input_order=unknown ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 5.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 6.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=unknown ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 10.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=3 width=24) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 14.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 15.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 16.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 17.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 18.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 19.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 20.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 23.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 24.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 25.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 26.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 27.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 28.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 29.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 31.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 32.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 33.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 34.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 35.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 36.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 37.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) order by d desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=desc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Aggragate (functions=1 width=8 input_order=unknown ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 5.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 6.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=unknown ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 10.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=3 width=24) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 14.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 15.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 16.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 17.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 18.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 19.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 20.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 23.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 24.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 25.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 26.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 27.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 28.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 29.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 31.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 32.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 33.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 34.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 35.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 36.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 37.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) order by d desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=desc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Aggragate (functions=1 width=8 input_order=unknown ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 5.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 6.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=unknown ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 10.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=3 width=24) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 14.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 15.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 16.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 17.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 18.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 19.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 20.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 23.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 24.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 25.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 26.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 27.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 28.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 29.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 31.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 32.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 33.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 34.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 35.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 36.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 37.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) group by c order by d\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=asc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Aggragate (functions=1 width=16 groups=1 input_order=unknown ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 5.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 6.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=unknown ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 10.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=3 width=24) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 14.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 15.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 16.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 17.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 18.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 19.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 20.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 23.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 24.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 25.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 26.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 27.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 28.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 29.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 31.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 32.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 33.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 34.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 35.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 36.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 37.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) group by c order by d\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=asc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Aggragate (functions=1 width=16 groups=1 input_order=unknown ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 5.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 6.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=unknown ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 10.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=3 width=24) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 14.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 15.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 16.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 17.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 18.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 19.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 20.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 23.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 24.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 25.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 26.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 27.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 28.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 29.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 31.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 32.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 33.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 34.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 35.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 36.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 37.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) group by c order by d desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=desc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Aggragate (functions=1 width=16 groups=1 input_order=unknown ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 5.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 6.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=unknown ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 10.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=3 width=24) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 14.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 15.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 16.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 17.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 18.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 19.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 20.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 23.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 24.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 25.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 26.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 27.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 28.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 29.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 31.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 32.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 33.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 34.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 35.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 36.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 37.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) group by c order by d desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=desc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Aggragate (functions=1 width=16 groups=1 input_order=unknown ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 5.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 6.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=unknown ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 10.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=3 width=24) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 14.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 15.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 16.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 17.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 18.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 19.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 20.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 22.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 23.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 24.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 25.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 26.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 27.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 28.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 29.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 31.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 32.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 33.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 34.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 35.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 36.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 37.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) where a > 10000 and a < 20000 interval(10s) fill(NULL) order by d\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=asc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Fill (mode=null width=24 input_order=unknown ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=2 width=16 +*************************** 5.row *************************** +QUERY_PLAN: Time Range: [10001, 19999] +*************************** 6.row *************************** +QUERY_PLAN: -> Interval on Column a (functions=2 width=16 input_order=asc output_order=asc ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=2 width=16 +*************************** 8.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 10.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=unknown ) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 13.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 14.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=3 width=24) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 16.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 17.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 18.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 19.row *************************** +QUERY_PLAN: Filter: ((a > 10000) AND (a < 20000)) +*************************** 20.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 21.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 22.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 23.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 24.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 25.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 26.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 27.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 28.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 29.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 30.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 31.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 32.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 33.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 34.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 35.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 36.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 37.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 38.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 39.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 40.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 41.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 42.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) where a > 10000 and a < 20000 interval(10s) fill(NULL) order by d\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=asc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Fill (mode=null width=24 input_order=asc ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=2 width=16 +*************************** 5.row *************************** +QUERY_PLAN: Time Range: [10001, 19999] +*************************** 6.row *************************** +QUERY_PLAN: -> Interval on Column a (functions=2 width=16 input_order=desc output_order=asc ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=2 width=16 +*************************** 8.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 10.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=desc ) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 13.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 14.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=desc output_order=desc) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 16.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 17.row *************************** +QUERY_PLAN: Filter: ((a > 10000) AND (a < 20000)) +*************************** 18.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 19.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=desc output_order=desc) +*************************** 20.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 21.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 22.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, desc +*************************** 23.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 24.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 25.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=desc ) +*************************** 26.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 27.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 28.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 29.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 31.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 32.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 33.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 34.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=desc ) +*************************** 35.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 36.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 37.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 38.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 39.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 40.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(b) as d from (select last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) where b > 10000 and b < 20000 interval(10s) fill(NULL) order by d\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=asc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Fill (mode=null width=24 input_order=asc ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=2 width=16 +*************************** 5.row *************************** +QUERY_PLAN: Time Range: [10001, 19999] +*************************** 6.row *************************** +QUERY_PLAN: -> Interval on Column b (functions=2 width=16 input_order=desc output_order=asc ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=2 width=16 +*************************** 8.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 10.row *************************** +QUERY_PLAN: -> Projection (columns=2 width=16 input_order=desc ) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=16 +*************************** 12.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 13.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 14.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=2 width=16) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=2 width=16 +*************************** 16.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=2 width=16 input_order=asc output_order=asc) +*************************** 17.row *************************** +QUERY_PLAN: Output: columns=2 width=16 +*************************** 18.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 19.row *************************** +QUERY_PLAN: Filter: ((b > 10000) AND (b < 20000)) +*************************** 20.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 21.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 22.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 23.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 24.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 25.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 26.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 27.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 28.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 29.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 30.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 31.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 32.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 33.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 34.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 35.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 36.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 37.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 38.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 39.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 40.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 41.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 42.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select last(b) as d from (select last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) where b > 10000 and b < 20000 interval(10s) fill(NULL) order by d desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=1 width=8) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 3.row *************************** +QUERY_PLAN: -> Fill (mode=null width=24 input_order=asc ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=2 width=16 +*************************** 5.row *************************** +QUERY_PLAN: Time Range: [10001, 19999] +*************************** 6.row *************************** +QUERY_PLAN: -> Interval on Column b (functions=2 width=16 input_order=desc output_order=asc ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=2 width=16 +*************************** 8.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 10.row *************************** +QUERY_PLAN: -> Projection (columns=2 width=16 input_order=desc ) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=16 +*************************** 12.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 13.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 14.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=2 width=16) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=2 width=16 +*************************** 16.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=2 width=16 input_order=asc output_order=asc) +*************************** 17.row *************************** +QUERY_PLAN: Output: columns=2 width=16 +*************************** 18.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 19.row *************************** +QUERY_PLAN: Filter: ((b > 10000) AND (b < 20000)) +*************************** 20.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 21.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 22.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 23.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 24.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 25.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 26.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 27.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 28.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 29.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 30.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 31.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 32.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 33.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 34.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 35.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 36.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 37.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 38.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 39.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 40.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 41.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 42.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> select _wstart, last(ts), avg(c2) from meters interval(10s) order by _wstart desc; + _wstart | last(ts) | avg(c2) | +================================================================================ + 2022-05-24 00:01:00.000 | 2022-05-24 00:01:08.000 | 210.000000000000000 | + 2022-05-23 00:01:00.000 | 2022-05-23 00:01:08.000 | 116.000000000000000 | + 2022-05-22 00:01:00.000 | 2022-05-22 00:01:08.000 | 196.000000000000000 | + 2022-05-21 00:01:00.000 | 2022-05-21 00:01:08.000 | 11.000000000000000 | + 2022-05-20 00:01:00.000 | 2022-05-20 00:01:08.000 | 120.000000000000000 | + 2022-05-19 00:01:00.000 | 2022-05-19 00:01:08.000 | 243.000000000000000 | + 2022-05-18 00:01:00.000 | 2022-05-18 00:01:08.000 | 58.000000000000000 | + 2022-05-17 00:01:00.000 | 2022-05-17 00:01:08.000 | 59.000000000000000 | + 2022-05-16 00:01:00.000 | 2022-05-16 00:01:08.000 | 136.000000000000000 | + 2022-05-15 00:01:00.000 | 2022-05-15 00:01:08.000 | 234.000000000000000 | + +taos> select _wstart, last(ts), avg(c2) from meters interval(10s) order by _wstart asc; + _wstart | last(ts) | avg(c2) | +================================================================================ + 2022-05-15 00:01:00.000 | 2022-05-15 00:01:08.000 | 234.000000000000000 | + 2022-05-16 00:01:00.000 | 2022-05-16 00:01:08.000 | 136.000000000000000 | + 2022-05-17 00:01:00.000 | 2022-05-17 00:01:08.000 | 59.000000000000000 | + 2022-05-18 00:01:00.000 | 2022-05-18 00:01:08.000 | 58.000000000000000 | + 2022-05-19 00:01:00.000 | 2022-05-19 00:01:08.000 | 243.000000000000000 | + 2022-05-20 00:01:00.000 | 2022-05-20 00:01:08.000 | 120.000000000000000 | + 2022-05-21 00:01:00.000 | 2022-05-21 00:01:08.000 | 11.000000000000000 | + 2022-05-22 00:01:00.000 | 2022-05-22 00:01:08.000 | 196.000000000000000 | + 2022-05-23 00:01:00.000 | 2022-05-23 00:01:08.000 | 116.000000000000000 | + 2022-05-24 00:01:00.000 | 2022-05-24 00:01:08.000 | 210.000000000000000 | + +taos> select _wstart, first(ts), avg(c2) from meters interval(10s) order by _wstart asc; + _wstart | first(ts) | avg(c2) | +================================================================================ + 2022-05-15 00:01:00.000 | 2022-05-15 00:01:08.000 | 234.000000000000000 | + 2022-05-16 00:01:00.000 | 2022-05-16 00:01:08.000 | 136.000000000000000 | + 2022-05-17 00:01:00.000 | 2022-05-17 00:01:08.000 | 59.000000000000000 | + 2022-05-18 00:01:00.000 | 2022-05-18 00:01:08.000 | 58.000000000000000 | + 2022-05-19 00:01:00.000 | 2022-05-19 00:01:08.000 | 243.000000000000000 | + 2022-05-20 00:01:00.000 | 2022-05-20 00:01:08.000 | 120.000000000000000 | + 2022-05-21 00:01:00.000 | 2022-05-21 00:01:08.000 | 11.000000000000000 | + 2022-05-22 00:01:00.000 | 2022-05-22 00:01:08.000 | 196.000000000000000 | + 2022-05-23 00:01:00.000 | 2022-05-23 00:01:08.000 | 116.000000000000000 | + 2022-05-24 00:01:00.000 | 2022-05-24 00:01:08.000 | 210.000000000000000 | + +taos> select _wstart, first(ts), avg(c2) from meters interval(10s) order by _wstart desc; + _wstart | first(ts) | avg(c2) | +================================================================================ + 2022-05-24 00:01:00.000 | 2022-05-24 00:01:08.000 | 210.000000000000000 | + 2022-05-23 00:01:00.000 | 2022-05-23 00:01:08.000 | 116.000000000000000 | + 2022-05-22 00:01:00.000 | 2022-05-22 00:01:08.000 | 196.000000000000000 | + 2022-05-21 00:01:00.000 | 2022-05-21 00:01:08.000 | 11.000000000000000 | + 2022-05-20 00:01:00.000 | 2022-05-20 00:01:08.000 | 120.000000000000000 | + 2022-05-19 00:01:00.000 | 2022-05-19 00:01:08.000 | 243.000000000000000 | + 2022-05-18 00:01:00.000 | 2022-05-18 00:01:08.000 | 58.000000000000000 | + 2022-05-17 00:01:00.000 | 2022-05-17 00:01:08.000 | 59.000000000000000 | + 2022-05-16 00:01:00.000 | 2022-05-16 00:01:08.000 | 136.000000000000000 | + 2022-05-15 00:01:00.000 | 2022-05-15 00:01:08.000 | 234.000000000000000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s)) order by d; + d | +========================== + 2022-05-24 00:01:00.000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s)) order by d desc; + d | +========================== + 2022-05-24 00:01:00.000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a) order by d; + d | +========================== + 2022-05-24 00:01:00.000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) order by d; + d | +========================== + 2022-05-24 00:01:00.000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a) order by d desc; + d | +========================== + 2022-05-24 00:01:00.000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) order by d desc; + d | +========================== + 2022-05-24 00:01:00.000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) order by d; + d | +========================== + 2022-05-24 00:01:00.000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) order by d; + d | +========================== + 2022-05-24 00:01:00.000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) order by d desc; + d | +========================== + 2022-05-24 00:01:00.000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) order by d desc; + d | +========================== + 2022-05-24 00:01:00.000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) group by c order by d + d | +========================== + 2022-05-15 00:01:00.000 | + 2022-05-16 00:01:00.000 | + 2022-05-17 00:01:00.000 | + 2022-05-18 00:01:00.000 | + 2022-05-19 00:01:00.000 | + 2022-05-20 00:01:00.000 | + 2022-05-21 00:01:00.000 | + 2022-05-22 00:01:00.000 | + 2022-05-23 00:01:00.000 | + 2022-05-24 00:01:00.000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) group by c order by d; + d | +========================== + 2022-05-15 00:01:00.000 | + 2022-05-16 00:01:00.000 | + 2022-05-17 00:01:00.000 | + 2022-05-18 00:01:00.000 | + 2022-05-19 00:01:00.000 | + 2022-05-20 00:01:00.000 | + 2022-05-21 00:01:00.000 | + 2022-05-22 00:01:00.000 | + 2022-05-23 00:01:00.000 | + 2022-05-24 00:01:00.000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) group by c order by d desc; + d | +========================== + 2022-05-24 00:01:00.000 | + 2022-05-23 00:01:00.000 | + 2022-05-22 00:01:00.000 | + 2022-05-21 00:01:00.000 | + 2022-05-20 00:01:00.000 | + 2022-05-19 00:01:00.000 | + 2022-05-18 00:01:00.000 | + 2022-05-17 00:01:00.000 | + 2022-05-16 00:01:00.000 | + 2022-05-15 00:01:00.000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) group by c order by d desc; + d | +========================== + 2022-05-24 00:01:00.000 | + 2022-05-23 00:01:00.000 | + 2022-05-22 00:01:00.000 | + 2022-05-21 00:01:00.000 | + 2022-05-20 00:01:00.000 | + 2022-05-19 00:01:00.000 | + 2022-05-18 00:01:00.000 | + 2022-05-17 00:01:00.000 | + 2022-05-16 00:01:00.000 | + 2022-05-15 00:01:00.000 | + +taos> select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) where a > '2022-05-15 00:01:00.000' and a < '2022-05-19 00:01:08.000' interval(10s) order by d; + d | +========================== + 2022-05-15 00:01:00.000 | + 2022-05-16 00:01:00.000 | + 2022-05-17 00:01:00.000 | + 2022-05-18 00:01:00.000 | + 2022-05-19 00:01:00.000 | + +taos> select last(b) as d from (select last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) where b > '2022-05-15 00:01:00.000' and b < '2022-05-19 00:01:08.000' interval(10s) order by d; + d | +========================== + 2022-05-15 00:01:08.000 | + 2022-05-16 00:01:08.000 | + 2022-05-17 00:01:08.000 | + 2022-05-18 00:01:08.000 | + 2022-05-24 00:01:08.000 | + +taos> select last(b) as d from (select last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) where b > '2022-05-15 00:01:00.000' and b < '2022-05-19 00:01:08.000' interval(10s) order by d desc; + d | +========================== + 2022-05-24 00:01:08.000 | + 2022-05-18 00:01:08.000 | + 2022-05-17 00:01:08.000 | + 2022-05-16 00:01:08.000 | + 2022-05-15 00:01:08.000 | + +taos> select _wstart, first(a) as d, avg(c) from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) where a > '2022-05-15 00:01:00.000' and a < '2022-05-21 00:01:08.000' interval(5h) fill(linear) order by d desc; + _wstart | d | avg(c) | +================================================================================ + 2022-05-20 20:00:00.000 | 2022-05-21 00:01:00.000 | 11.000000000000000 | + 2022-05-20 15:00:00.000 | 2022-05-20 18:01:00.000 | 38.250000000000000 | + 2022-05-20 10:00:00.000 | 2022-05-20 12:01:00.000 | 65.500000000000000 | + 2022-05-20 05:00:00.000 | 2022-05-20 06:01:00.000 | 92.750000000000000 | + 2022-05-20 00:00:00.000 | 2022-05-20 00:01:00.000 | 120.000000000000000 | + 2022-05-19 19:00:00.000 | 2022-05-19 19:13:00.000 | 144.599999999999994 | + 2022-05-19 14:00:00.000 | 2022-05-19 14:25:00.000 | 169.199999999999989 | + 2022-05-19 09:00:00.000 | 2022-05-19 09:37:00.000 | 193.800000000000011 | + 2022-05-19 04:00:00.000 | 2022-05-19 04:49:00.000 | 218.400000000000006 | + 2022-05-18 23:00:00.000 | 2022-05-19 00:01:00.000 | 243.000000000000000 | + 2022-05-18 18:00:00.000 | 2022-05-18 19:13:00.000 | 206.000000000000000 | + 2022-05-18 13:00:00.000 | 2022-05-18 14:25:00.000 | 169.000000000000000 | + 2022-05-18 08:00:00.000 | 2022-05-18 09:37:00.000 | 132.000000000000000 | + 2022-05-18 03:00:00.000 | 2022-05-18 04:49:00.000 | 95.000000000000000 | + 2022-05-17 22:00:00.000 | 2022-05-18 00:01:00.000 | 58.000000000000000 | + 2022-05-17 17:00:00.000 | 2022-05-17 19:13:00.000 | 58.200000000000003 | + 2022-05-17 12:00:00.000 | 2022-05-17 14:25:00.000 | 58.399999999999999 | + 2022-05-17 07:00:00.000 | 2022-05-17 09:37:00.000 | 58.600000000000001 | + 2022-05-17 02:00:00.000 | 2022-05-17 04:49:00.000 | 58.799999999999997 | + 2022-05-16 21:00:00.000 | 2022-05-17 00:01:00.000 | 59.000000000000000 | + 2022-05-16 16:00:00.000 | 2022-05-16 19:13:00.000 | 74.400000000000006 | + 2022-05-16 11:00:00.000 | 2022-05-16 14:25:00.000 | 89.799999999999997 | + 2022-05-16 06:00:00.000 | 2022-05-16 09:37:00.000 | 105.200000000000003 | + 2022-05-16 01:00:00.000 | 2022-05-16 04:49:00.000 | 120.599999999999994 | + 2022-05-15 20:00:00.000 | 2022-05-16 00:01:00.000 | 136.000000000000000 | + 2022-05-15 15:00:00.000 | 2022-05-15 18:01:00.000 | 160.500000000000000 | + 2022-05-15 10:00:00.000 | 2022-05-15 12:01:00.000 | 185.000000000000000 | + 2022-05-15 05:00:00.000 | 2022-05-15 06:01:00.000 | 209.500000000000000 | + 2022-05-15 00:00:00.000 | 2022-05-15 00:01:00.000 | 234.000000000000000 | + +taos> explain verbose true select _wstart, first(a) as d, avg(c) from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) where a > '2022-05-15 00:01:00.000' and a < '2022-05-21 00:01:08.000' interval(5h) fill(linear) order by d desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: -> Fill (mode=linear width=32 input_order=asc ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 5.row *************************** +QUERY_PLAN: Time Range: [1652544060001, 1653062467999] +*************************** 6.row *************************** +QUERY_PLAN: -> Interval on Column a (functions=4 width=32 input_order=desc output_order=asc ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 8.row *************************** +QUERY_PLAN: Time Window: interval=5h offset=0a sliding=5h +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 10.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=desc ) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 13.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 14.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=desc output_order=desc) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 16.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 17.row *************************** +QUERY_PLAN: Filter: ((a > 1652544060000) AND (a < 1653062468000)) +*************************** 18.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 19.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=desc output_order=desc) +*************************** 20.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 21.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 22.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, desc +*************************** 23.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 24.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 25.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=desc ) +*************************** 26.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 27.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 28.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 29.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 31.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 32.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 33.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 34.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=desc ) +*************************** 35.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 36.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 37.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 38.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 39.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 40.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, first(a) as d, avg(c) from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a asc) where a > '2022-05-15 00:01:00.000' and a < '2022-05-21 00:01:08.000' interval(5h) fill(linear) order by d desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: -> Fill (mode=linear width=32 input_order=asc ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 5.row *************************** +QUERY_PLAN: Time Range: [1652544060001, 1653062467999] +*************************** 6.row *************************** +QUERY_PLAN: -> Interval on Column a (functions=4 width=32 input_order=asc output_order=asc ) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 8.row *************************** +QUERY_PLAN: Time Window: interval=5h offset=0a sliding=5h +*************************** 9.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 10.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=asc ) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 12.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 13.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 14.row *************************** +QUERY_PLAN: -> Merge Aligned Interval on Column (functions=3 width=24 input_order=asc output_order=asc) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 16.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 17.row *************************** +QUERY_PLAN: Filter: ((a > 1652544060000) AND (a < 1653062468000)) +*************************** 18.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 19.row *************************** +QUERY_PLAN: -> SortMerge (columns=3 width=108 input_order=asc output_order=asc) +*************************** 20.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 21.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 22.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, asc +*************************** 23.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 24.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 25.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 26.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 27.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 28.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 29.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 30.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 31.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 32.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=108) +*************************** 33.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 34.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=108 input_order=desc output_order=asc ) +*************************** 35.row *************************** +QUERY_PLAN: Output: columns=3 width=108 +*************************** 36.row *************************** +QUERY_PLAN: Time Window: interval=10s offset=0a sliding=10s +*************************** 37.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 38.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 39.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 40.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select * from (select ts as a, c2 as b from meters order by c2 desc)\G; +*************************** 1.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 3.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 4.row *************************** +QUERY_PLAN: Merge Key: b desc +*************************** 5.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 6.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 7.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=2 width=12) +*************************** 8.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 9.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 10.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 11.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 12.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=2 width=12) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 16.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 17.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 18.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> select * from (select ts as a, c2 as b from meters order by c2 desc); + a | b | +======================================== + 2022-05-19 00:01:08.000 | 243 | + 2022-05-19 00:01:08.000 | 243 | + 2022-05-15 00:01:08.000 | 234 | + 2022-05-15 00:01:08.000 | 234 | + 2022-05-24 00:01:08.000 | 210 | + 2022-05-24 00:01:08.000 | 210 | + 2022-05-22 00:01:08.000 | 196 | + 2022-05-22 00:01:08.000 | 196 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-20 00:01:08.000 | 120 | + 2022-05-20 00:01:08.000 | 120 | + 2022-05-23 00:01:08.000 | 116 | + 2022-05-23 00:01:08.000 | 116 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-21 00:01:08.000 | 11 | + 2022-05-21 00:01:08.000 | 11 | + +taos> explain verbose true select * from (select ts as a, c2 as b from meters order by c2 desc) order by a desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=unknown output_order=desc (columns=2 width=12) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 3.row *************************** +QUERY_PLAN: -> Projection (columns=2 width=12 input_order=unknown ) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 5.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 6.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 7.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 8.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 9.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 10.row *************************** +QUERY_PLAN: Merge Key: b desc +*************************** 11.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 12.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 13.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=2 width=12) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 18.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 20.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=unknown (columns=2 width=12) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 22.row *************************** +QUERY_PLAN: -> Table Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 23.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 24.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> select * from (select ts as a, c2 as b from meters order by c2 desc) order by a desc; + a | b | +======================================== + 2022-05-24 00:01:08.000 | 210 | + 2022-05-24 00:01:08.000 | 210 | + 2022-05-23 00:01:08.000 | 116 | + 2022-05-23 00:01:08.000 | 116 | + 2022-05-22 00:01:08.000 | 196 | + 2022-05-22 00:01:08.000 | 196 | + 2022-05-21 00:01:08.000 | 11 | + 2022-05-21 00:01:08.000 | 11 | + 2022-05-20 00:01:08.000 | 120 | + 2022-05-20 00:01:08.000 | 120 | + 2022-05-19 00:01:08.000 | 243 | + 2022-05-19 00:01:08.000 | 243 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-15 00:01:08.000 | 234 | + 2022-05-15 00:01:08.000 | 234 | + +taos> explain verbose true select a.ts, a.c2, b.c2 from meters as a join meters as b on a.ts = b.ts\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=16 input_order=unknown ) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=16 +*************************** 3.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 4.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 5.row *************************** +QUERY_PLAN: -> Inner join (columns=4 width=24 input_order=asc ) +*************************** 6.row *************************** +QUERY_PLAN: Output: columns=4 width=24 +*************************** 7.row *************************** +QUERY_PLAN: Join Cond: (`test`.`a`.`ts` = `test`.`b`.`ts`) +*************************** 8.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 11.row *************************** +QUERY_PLAN: Merge Key: ts asc +*************************** 12.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 16.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 17.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 19.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 20.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 21.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 22.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 23.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 24.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 25.row *************************** +QUERY_PLAN: Merge Key: ts asc +*************************** 26.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 27.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 28.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 29.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 30.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 31.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 32.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 33.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 34.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 35.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select a.ts, a.c2, b.c2 from meters as a join meters as b on a.ts = b.ts order by a.ts\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=16 input_order=unknown ) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=16 +*************************** 3.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 4.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 5.row *************************** +QUERY_PLAN: -> Inner join (columns=4 width=24 input_order=asc ) +*************************** 6.row *************************** +QUERY_PLAN: Output: columns=4 width=24 +*************************** 7.row *************************** +QUERY_PLAN: Join Cond: (`test`.`a`.`ts` = `test`.`b`.`ts`) +*************************** 8.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=asc output_order=asc) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 11.row *************************** +QUERY_PLAN: Merge Key: ts asc +*************************** 12.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 16.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 17.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 19.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 20.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 21.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 22.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=asc output_order=asc) +*************************** 23.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 24.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 25.row *************************** +QUERY_PLAN: Merge Key: ts asc +*************************** 26.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 27.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 28.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 29.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 30.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 31.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 32.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 33.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 34.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 35.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> select a.ts, a.c2, b.c2 from meters as a join meters as b on a.ts = b.ts; + ts | c2 | c2 | +====================================================== + 2022-05-15 00:01:08.000 | 234 | 234 | + 2022-05-15 00:01:08.000 | 234 | 234 | + 2022-05-15 00:01:08.000 | 234 | 234 | + 2022-05-15 00:01:08.000 | 234 | 234 | + 2022-05-16 00:01:08.000 | 136 | 136 | + 2022-05-16 00:01:08.000 | 136 | 136 | + 2022-05-16 00:01:08.000 | 136 | 136 | + 2022-05-16 00:01:08.000 | 136 | 136 | + 2022-05-17 00:01:08.000 | 59 | 59 | + 2022-05-17 00:01:08.000 | 59 | 59 | + 2022-05-17 00:01:08.000 | 59 | 59 | + 2022-05-17 00:01:08.000 | 59 | 59 | + 2022-05-18 00:01:08.000 | 58 | 58 | + 2022-05-18 00:01:08.000 | 58 | 58 | + 2022-05-18 00:01:08.000 | 58 | 58 | + 2022-05-18 00:01:08.000 | 58 | 58 | + 2022-05-19 00:01:08.000 | 243 | 243 | + 2022-05-19 00:01:08.000 | 243 | 243 | + 2022-05-19 00:01:08.000 | 243 | 243 | + 2022-05-19 00:01:08.000 | 243 | 243 | + 2022-05-20 00:01:08.000 | 120 | 120 | + 2022-05-20 00:01:08.000 | 120 | 120 | + 2022-05-20 00:01:08.000 | 120 | 120 | + 2022-05-20 00:01:08.000 | 120 | 120 | + 2022-05-21 00:01:08.000 | 11 | 11 | + 2022-05-21 00:01:08.000 | 11 | 11 | + 2022-05-21 00:01:08.000 | 11 | 11 | + 2022-05-21 00:01:08.000 | 11 | 11 | + 2022-05-22 00:01:08.000 | 196 | 196 | + 2022-05-22 00:01:08.000 | 196 | 196 | + 2022-05-22 00:01:08.000 | 196 | 196 | + 2022-05-22 00:01:08.000 | 196 | 196 | + 2022-05-23 00:01:08.000 | 116 | 116 | + 2022-05-23 00:01:08.000 | 116 | 116 | + 2022-05-23 00:01:08.000 | 116 | 116 | + 2022-05-23 00:01:08.000 | 116 | 116 | + 2022-05-24 00:01:08.000 | 210 | 210 | + 2022-05-24 00:01:08.000 | 210 | 210 | + 2022-05-24 00:01:08.000 | 210 | 210 | + 2022-05-24 00:01:08.000 | 210 | 210 | + +taos> select a.ts, a.c2, b.c2 from meters as a join meters as b on a.ts = b.ts order by a.ts desc; + ts | c2 | c2 | +====================================================== + 2022-05-24 00:01:08.000 | 210 | 210 | + 2022-05-24 00:01:08.000 | 210 | 210 | + 2022-05-24 00:01:08.000 | 210 | 210 | + 2022-05-24 00:01:08.000 | 210 | 210 | + 2022-05-23 00:01:08.000 | 116 | 116 | + 2022-05-23 00:01:08.000 | 116 | 116 | + 2022-05-23 00:01:08.000 | 116 | 116 | + 2022-05-23 00:01:08.000 | 116 | 116 | + 2022-05-22 00:01:08.000 | 196 | 196 | + 2022-05-22 00:01:08.000 | 196 | 196 | + 2022-05-22 00:01:08.000 | 196 | 196 | + 2022-05-22 00:01:08.000 | 196 | 196 | + 2022-05-21 00:01:08.000 | 11 | 11 | + 2022-05-21 00:01:08.000 | 11 | 11 | + 2022-05-21 00:01:08.000 | 11 | 11 | + 2022-05-21 00:01:08.000 | 11 | 11 | + 2022-05-20 00:01:08.000 | 120 | 120 | + 2022-05-20 00:01:08.000 | 120 | 120 | + 2022-05-20 00:01:08.000 | 120 | 120 | + 2022-05-20 00:01:08.000 | 120 | 120 | + 2022-05-19 00:01:08.000 | 243 | 243 | + 2022-05-19 00:01:08.000 | 243 | 243 | + 2022-05-19 00:01:08.000 | 243 | 243 | + 2022-05-19 00:01:08.000 | 243 | 243 | + 2022-05-18 00:01:08.000 | 58 | 58 | + 2022-05-18 00:01:08.000 | 58 | 58 | + 2022-05-18 00:01:08.000 | 58 | 58 | + 2022-05-18 00:01:08.000 | 58 | 58 | + 2022-05-17 00:01:08.000 | 59 | 59 | + 2022-05-17 00:01:08.000 | 59 | 59 | + 2022-05-17 00:01:08.000 | 59 | 59 | + 2022-05-17 00:01:08.000 | 59 | 59 | + 2022-05-16 00:01:08.000 | 136 | 136 | + 2022-05-16 00:01:08.000 | 136 | 136 | + 2022-05-16 00:01:08.000 | 136 | 136 | + 2022-05-16 00:01:08.000 | 136 | 136 | + 2022-05-15 00:01:08.000 | 234 | 234 | + 2022-05-15 00:01:08.000 | 234 | 234 | + 2022-05-15 00:01:08.000 | 234 | 234 | + 2022-05-15 00:01:08.000 | 234 | 234 | + +taos> explain verbose true select a.ts, a.c2, b.c2 from meters as a join (select ts, c2 from meters order by ts desc) b on a.ts = b.ts order by a.ts desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=16 input_order=unknown ) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=16 +*************************** 3.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 4.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 5.row *************************** +QUERY_PLAN: -> Inner join (columns=4 width=24 input_order=desc ) +*************************** 6.row *************************** +QUERY_PLAN: Output: columns=4 width=24 +*************************** 7.row *************************** +QUERY_PLAN: Join Cond: (`test`.`a`.`ts` = `b`.`ts`) +*************************** 8.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=desc output_order=desc) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 11.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts desc +*************************** 12.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 16.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 17.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 19.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 20.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 21.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 22.row *************************** +QUERY_PLAN: -> Projection (columns=2 width=12 input_order=desc ) +*************************** 23.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 24.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 25.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 26.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=desc output_order=desc) +*************************** 27.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 28.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 29.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts desc +*************************** 30.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 31.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 32.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 33.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 34.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 35.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 36.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 37.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|0 desc|1]) +*************************** 38.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 39.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select a.ts, a.c2, b.c2 from meters as a join (select ts, c2 from meters order by ts desc) b on a.ts = b.ts order by a.ts asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=16 input_order=unknown ) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=16 +*************************** 3.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 4.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 5.row *************************** +QUERY_PLAN: -> Inner join (columns=4 width=24 input_order=asc ) +*************************** 6.row *************************** +QUERY_PLAN: Output: columns=4 width=24 +*************************** 7.row *************************** +QUERY_PLAN: Join Cond: (`test`.`a`.`ts` = `b`.`ts`) +*************************** 8.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=asc output_order=asc) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 11.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 12.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 16.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 17.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 19.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 20.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 21.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 22.row *************************** +QUERY_PLAN: -> Projection (columns=2 width=12 input_order=asc ) +*************************** 23.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 24.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 25.row *************************** +QUERY_PLAN: Merge ResBlocks: False +*************************** 26.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=asc output_order=asc) +*************************** 27.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 28.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 29.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 30.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 31.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 32.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 33.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 34.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 35.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 36.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 37.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 38.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 39.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> select a.ts, a.c2, b.c2 from meters as a join (select * from meters order by ts desc) b on a.ts = b.ts order by a.ts asc; + ts | c2 | c2 | +====================================================== + 2022-05-15 00:01:08.000 | 234 | 234 | + 2022-05-15 00:01:08.000 | 234 | 234 | + 2022-05-15 00:01:08.000 | 234 | 234 | + 2022-05-15 00:01:08.000 | 234 | 234 | + 2022-05-16 00:01:08.000 | 136 | 136 | + 2022-05-16 00:01:08.000 | 136 | 136 | + 2022-05-16 00:01:08.000 | 136 | 136 | + 2022-05-16 00:01:08.000 | 136 | 136 | + 2022-05-17 00:01:08.000 | 59 | 59 | + 2022-05-17 00:01:08.000 | 59 | 59 | + 2022-05-17 00:01:08.000 | 59 | 59 | + 2022-05-17 00:01:08.000 | 59 | 59 | + 2022-05-18 00:01:08.000 | 58 | 58 | + 2022-05-18 00:01:08.000 | 58 | 58 | + 2022-05-18 00:01:08.000 | 58 | 58 | + 2022-05-18 00:01:08.000 | 58 | 58 | + 2022-05-19 00:01:08.000 | 243 | 243 | + 2022-05-19 00:01:08.000 | 243 | 243 | + 2022-05-19 00:01:08.000 | 243 | 243 | + 2022-05-19 00:01:08.000 | 243 | 243 | + 2022-05-20 00:01:08.000 | 120 | 120 | + 2022-05-20 00:01:08.000 | 120 | 120 | + 2022-05-20 00:01:08.000 | 120 | 120 | + 2022-05-20 00:01:08.000 | 120 | 120 | + 2022-05-21 00:01:08.000 | 11 | 11 | + 2022-05-21 00:01:08.000 | 11 | 11 | + 2022-05-21 00:01:08.000 | 11 | 11 | + 2022-05-21 00:01:08.000 | 11 | 11 | + 2022-05-22 00:01:08.000 | 196 | 196 | + 2022-05-22 00:01:08.000 | 196 | 196 | + 2022-05-22 00:01:08.000 | 196 | 196 | + 2022-05-22 00:01:08.000 | 196 | 196 | + 2022-05-23 00:01:08.000 | 116 | 116 | + 2022-05-23 00:01:08.000 | 116 | 116 | + 2022-05-23 00:01:08.000 | 116 | 116 | + 2022-05-23 00:01:08.000 | 116 | 116 | + 2022-05-24 00:01:08.000 | 210 | 210 | + 2022-05-24 00:01:08.000 | 210 | 210 | + 2022-05-24 00:01:08.000 | 210 | 210 | + 2022-05-24 00:01:08.000 | 210 | 210 | + +taos> select ts, c2 from meters order by c2; + ts | c2 | +======================================== + 2022-05-21 00:01:08.000 | 11 | + 2022-05-21 00:01:08.000 | 11 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-23 00:01:08.000 | 116 | + 2022-05-23 00:01:08.000 | 116 | + 2022-05-20 00:01:08.000 | 120 | + 2022-05-20 00:01:08.000 | 120 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-22 00:01:08.000 | 196 | + 2022-05-22 00:01:08.000 | 196 | + 2022-05-24 00:01:08.000 | 210 | + 2022-05-24 00:01:08.000 | 210 | + 2022-05-15 00:01:08.000 | 234 | + 2022-05-15 00:01:08.000 | 234 | + 2022-05-19 00:01:08.000 | 243 | + 2022-05-19 00:01:08.000 | 243 | + +taos> select ts, c2 from meters order by c2 limit 4; + ts | c2 | +======================================== + 2022-05-21 00:01:08.000 | 11 | + 2022-05-21 00:01:08.000 | 11 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-18 00:01:08.000 | 58 | + +taos> select ts, c2 from meters order by c2 limit 2,2; + ts | c2 | +======================================== + 2022-05-18 00:01:08.000 | 58 | + 2022-05-18 00:01:08.000 | 58 | + +taos> select ts, c2 from meters order by ts asc, c2 desc limit 10; + ts | c2 | +======================================== + 2022-05-15 00:01:08.000 | 234 | + 2022-05-15 00:01:08.000 | 234 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-19 00:01:08.000 | 243 | + 2022-05-19 00:01:08.000 | 243 | + +taos> select ts, c2 from meters order by ts asc, c2 desc limit 5,5; + ts | c2 | +======================================== + 2022-05-17 00:01:08.000 | 59 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-19 00:01:08.000 | 243 | + 2022-05-19 00:01:08.000 | 243 | + +taos> select ts, c2 from d1 order by c2; + ts | c2 | +======================================== + 2022-05-21 00:01:08.000 | 11 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-23 00:01:08.000 | 116 | + 2022-05-20 00:01:08.000 | 120 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-22 00:01:08.000 | 196 | + 2022-05-24 00:01:08.000 | 210 | + 2022-05-15 00:01:08.000 | 234 | + 2022-05-19 00:01:08.000 | 243 | + +taos> select ts, c2 from d1 order by c2 limit 4; + ts | c2 | +======================================== + 2022-05-21 00:01:08.000 | 11 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-23 00:01:08.000 | 116 | + +taos> select ts, c2 from d1 order by c2 limit 2,2; + ts | c2 | +======================================== + 2022-05-17 00:01:08.000 | 59 | + 2022-05-23 00:01:08.000 | 116 | + +taos> select ts, c2 from d1 order by ts asc, c2 desc limit 10; + ts | c2 | +======================================== + 2022-05-15 00:01:08.000 | 234 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-19 00:01:08.000 | 243 | + 2022-05-20 00:01:08.000 | 120 | + 2022-05-21 00:01:08.000 | 11 | + 2022-05-22 00:01:08.000 | 196 | + 2022-05-23 00:01:08.000 | 116 | + 2022-05-24 00:01:08.000 | 210 | + +taos> select ts, c2 from d1 order by ts asc, c2 desc limit 5,5; + ts | c2 | +======================================== + 2022-05-20 00:01:08.000 | 120 | + 2022-05-21 00:01:08.000 | 11 | + 2022-05-22 00:01:08.000 | 196 | + 2022-05-23 00:01:08.000 | 116 | + 2022-05-24 00:01:08.000 | 210 | + +taos> select _wstart, first(a) as d, avg(c) from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) where a > '2022-05-15 00:01:00.000' and a < '2022-05-21 00:01:08.000' interval(5h) fill(linear) order by avg(c) desc; + _wstart | d | avg(c) | +================================================================================ + 2022-05-18 23:00:00.000 | 2022-05-19 00:01:00.000 | 243.000000000000000 | + 2022-05-15 00:00:00.000 | 2022-05-15 00:01:00.000 | 234.000000000000000 | + 2022-05-19 04:00:00.000 | 2022-05-19 04:49:00.000 | 218.400000000000006 | + 2022-05-15 05:00:00.000 | 2022-05-15 06:01:00.000 | 209.500000000000000 | + 2022-05-18 18:00:00.000 | 2022-05-18 19:13:00.000 | 206.000000000000000 | + 2022-05-19 09:00:00.000 | 2022-05-19 09:37:00.000 | 193.800000000000011 | + 2022-05-15 10:00:00.000 | 2022-05-15 12:01:00.000 | 185.000000000000000 | + 2022-05-19 14:00:00.000 | 2022-05-19 14:25:00.000 | 169.199999999999989 | + 2022-05-18 13:00:00.000 | 2022-05-18 14:25:00.000 | 169.000000000000000 | + 2022-05-15 15:00:00.000 | 2022-05-15 18:01:00.000 | 160.500000000000000 | + 2022-05-19 19:00:00.000 | 2022-05-19 19:13:00.000 | 144.599999999999994 | + 2022-05-15 20:00:00.000 | 2022-05-16 00:01:00.000 | 136.000000000000000 | + 2022-05-18 08:00:00.000 | 2022-05-18 09:37:00.000 | 132.000000000000000 | + 2022-05-16 01:00:00.000 | 2022-05-16 04:49:00.000 | 120.599999999999994 | + 2022-05-20 00:00:00.000 | 2022-05-20 00:01:00.000 | 120.000000000000000 | + 2022-05-16 06:00:00.000 | 2022-05-16 09:37:00.000 | 105.200000000000003 | + 2022-05-18 03:00:00.000 | 2022-05-18 04:49:00.000 | 95.000000000000000 | + 2022-05-20 05:00:00.000 | 2022-05-20 06:01:00.000 | 92.750000000000000 | + 2022-05-16 11:00:00.000 | 2022-05-16 14:25:00.000 | 89.799999999999997 | + 2022-05-16 16:00:00.000 | 2022-05-16 19:13:00.000 | 74.400000000000006 | + 2022-05-20 10:00:00.000 | 2022-05-20 12:01:00.000 | 65.500000000000000 | + 2022-05-16 21:00:00.000 | 2022-05-17 00:01:00.000 | 59.000000000000000 | + 2022-05-17 02:00:00.000 | 2022-05-17 04:49:00.000 | 58.799999999999997 | + 2022-05-17 07:00:00.000 | 2022-05-17 09:37:00.000 | 58.600000000000001 | + 2022-05-17 12:00:00.000 | 2022-05-17 14:25:00.000 | 58.399999999999999 | + 2022-05-17 17:00:00.000 | 2022-05-17 19:13:00.000 | 58.200000000000003 | + 2022-05-17 22:00:00.000 | 2022-05-18 00:01:00.000 | 58.000000000000000 | + 2022-05-20 15:00:00.000 | 2022-05-20 18:01:00.000 | 38.250000000000000 | + 2022-05-20 20:00:00.000 | 2022-05-21 00:01:00.000 | 11.000000000000000 | + +taos> select _wstart, first(a) as d, avg(c) from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) where a > '2022-05-15 00:01:00.000' and a < '2022-05-21 00:01:08.000' interval(5h) fill(linear) order by avg(c) desc limit 2; + _wstart | d | avg(c) | +================================================================================ + 2022-05-18 23:00:00.000 | 2022-05-19 00:01:00.000 | 243.000000000000000 | + 2022-05-15 00:00:00.000 | 2022-05-15 00:01:00.000 | 234.000000000000000 | + +taos> select _wstart, first(a) as d, avg(c) from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) where a > '2022-05-15 00:01:00.000' and a < '2022-05-21 00:01:08.000' interval(5h) fill(linear) order by avg(c) desc limit 2,6; + _wstart | d | avg(c) | +================================================================================ + 2022-05-19 04:00:00.000 | 2022-05-19 04:49:00.000 | 218.400000000000006 | + 2022-05-15 05:00:00.000 | 2022-05-15 06:01:00.000 | 209.500000000000000 | + 2022-05-18 18:00:00.000 | 2022-05-18 19:13:00.000 | 206.000000000000000 | + 2022-05-19 09:00:00.000 | 2022-05-19 09:37:00.000 | 193.800000000000011 | + 2022-05-15 10:00:00.000 | 2022-05-15 12:01:00.000 | 185.000000000000000 | + 2022-05-19 14:00:00.000 | 2022-05-19 14:25:00.000 | 169.199999999999989 | + +taos> select last(ts), c2 as d from d1 group by c2 order by c2 desc limit 10; + last(ts) | d | +======================================== + 2022-05-19 00:01:08.000 | 243 | + 2022-05-15 00:01:08.000 | 234 | + 2022-05-24 00:01:08.000 | 210 | + 2022-05-22 00:01:08.000 | 196 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-20 00:01:08.000 | 120 | + 2022-05-23 00:01:08.000 | 116 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-21 00:01:08.000 | 11 | + +taos> select last(ts), c2 as d from d1 group by c2 order by c2 desc limit 2,8; + last(ts) | d | +======================================== + 2022-05-24 00:01:08.000 | 210 | + 2022-05-22 00:01:08.000 | 196 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-20 00:01:08.000 | 120 | + 2022-05-23 00:01:08.000 | 116 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-21 00:01:08.000 | 11 | + +taos> select last(ts), c2 as d from d1 group by c2 order by c2 desc limit 9,1; + last(ts) | d | +======================================== + 2022-05-21 00:01:08.000 | 11 | + +taos> select last(ts), c2 as d from d1 group by c2 order by c2 asc limit 2,8; + last(ts) | d | +======================================== + 2022-05-17 00:01:08.000 | 59 | + 2022-05-23 00:01:08.000 | 116 | + 2022-05-20 00:01:08.000 | 120 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-22 00:01:08.000 | 196 | + 2022-05-24 00:01:08.000 | 210 | + 2022-05-15 00:01:08.000 | 234 | + 2022-05-19 00:01:08.000 | 243 | + +taos> select last(ts), c2 as d from d1 group by c2 order by c2 asc limit 9,1; + last(ts) | d | +======================================== + 2022-05-19 00:01:08.000 | 243 | + +taos> select last(ts) as ts, c2 as d from d1 group by c2 order by ts desc, c2 asc limit 10; + ts | d | +======================================== + 2022-05-24 00:01:08.000 | 210 | + 2022-05-23 00:01:08.000 | 116 | + 2022-05-22 00:01:08.000 | 196 | + 2022-05-21 00:01:08.000 | 11 | + 2022-05-20 00:01:08.000 | 120 | + 2022-05-19 00:01:08.000 | 243 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-15 00:01:08.000 | 234 | + +taos> select last(ts) as ts, c2 as d from d1 group by c2 order by ts desc, c2 asc limit 2,8; + ts | d | +======================================== + 2022-05-22 00:01:08.000 | 196 | + 2022-05-21 00:01:08.000 | 11 | + 2022-05-20 00:01:08.000 | 120 | + 2022-05-19 00:01:08.000 | 243 | + 2022-05-18 00:01:08.000 | 58 | + 2022-05-17 00:01:08.000 | 59 | + 2022-05-16 00:01:08.000 | 136 | + 2022-05-15 00:01:08.000 | 234 | + +taos> select last(ts) as ts, c2 as d from d1 group by c2 order by ts desc, c2 asc limit 9,1; + ts | d | +======================================== + 2022-05-15 00:01:08.000 | 234 | + +taos> explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Event (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Start Cond: (`test`.`meters`.`c2` > 0) +*************************** 3.row *************************** +QUERY_PLAN: End Cond: (`test`.`meters`.`c2` < 100) +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: -> Event (functions=3 width=24) +*************************** 4.row *************************** +QUERY_PLAN: Start Cond: (`test`.`meters`.`c2` > 0) +*************************** 5.row *************************** +QUERY_PLAN: End Cond: (`test`.`meters`.`c2` < 100) +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Event (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Start Cond: (`test`.`meters`.`c2` > 0) +*************************** 3.row *************************** +QUERY_PLAN: End Cond: (`test`.`meters`.`c2` < 100) +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: -> Event (functions=3 width=24) +*************************** 4.row *************************** +QUERY_PLAN: Start Cond: (`test`.`meters`.`c2` > 0) +*************************** 5.row *************************** +QUERY_PLAN: End Cond: (`test`.`meters`.`c2` < 100) +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Event (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Start Cond: (`test`.`meters`.`c2` > 0) +*************************** 3.row *************************** +QUERY_PLAN: End Cond: (`test`.`meters`.`c2` < 100) +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-17 00:01:08.000 | 5 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-19 00:01:08.000 | 2022-05-21 00:01:08.000 | 5 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 1 | + +taos> select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart desc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 1 | + 2022-05-19 00:01:08.000 | 2022-05-21 00:01:08.000 | 5 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 1 | + 2022-05-15 00:01:08.000 | 2022-05-17 00:01:08.000 | 5 | + +taos> select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart asc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-17 00:01:08.000 | 5 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-19 00:01:08.000 | 2022-05-21 00:01:08.000 | 5 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 1 | + +taos> select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend desc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-19 00:01:08.000 | 2022-05-21 00:01:08.000 | 5 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-15 00:01:08.000 | 2022-05-17 00:01:08.000 | 5 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 1 | + +taos> select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend asc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-17 00:01:08.000 | 5 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-19 00:01:08.000 | 2022-05-21 00:01:08.000 | 5 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 1 | + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h)\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 5.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 5.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h); + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h)\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 5.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 5.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h); + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + +taos> explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2)\G; +*************************** 1.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=4 width=36) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 3.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=4 width=32) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 3.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=4 width=36) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=4 width=36) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 3.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=4 width=32) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 3.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=4 width=36) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=4 width=36) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 3.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> select _wstart, _wend, count(*), last(ts) from meters state_window(c2); + _wstart | _wend | count(*) | last(ts) | +====================================================================================================== + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | 2022-05-15 00:01:08.000 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | 2022-05-16 00:01:08.000 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | 2022-05-17 00:01:08.000 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | 2022-05-18 00:01:08.000 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | 2022-05-19 00:01:08.000 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | 2022-05-20 00:01:08.000 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | 2022-05-21 00:01:08.000 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | 2022-05-22 00:01:08.000 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | 2022-05-23 00:01:08.000 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | 2022-05-24 00:01:08.000 | + +taos> select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart desc; + _wstart | _wend | count(*) | last(ts) | +====================================================================================================== + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | 2022-05-24 00:01:08.000 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | 2022-05-23 00:01:08.000 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | 2022-05-22 00:01:08.000 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | 2022-05-21 00:01:08.000 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | 2022-05-20 00:01:08.000 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | 2022-05-19 00:01:08.000 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | 2022-05-18 00:01:08.000 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | 2022-05-17 00:01:08.000 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | 2022-05-16 00:01:08.000 | + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | 2022-05-15 00:01:08.000 | + +taos> select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart asc; + _wstart | _wend | count(*) | last(ts) | +====================================================================================================== + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | 2022-05-15 00:01:08.000 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | 2022-05-16 00:01:08.000 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | 2022-05-17 00:01:08.000 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | 2022-05-18 00:01:08.000 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | 2022-05-19 00:01:08.000 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | 2022-05-20 00:01:08.000 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | 2022-05-21 00:01:08.000 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | 2022-05-22 00:01:08.000 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | 2022-05-23 00:01:08.000 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | 2022-05-24 00:01:08.000 | + +taos> select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend desc; + _wstart | _wend | count(*) | last(ts) | +====================================================================================================== + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | 2022-05-24 00:01:08.000 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | 2022-05-23 00:01:08.000 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | 2022-05-22 00:01:08.000 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | 2022-05-21 00:01:08.000 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | 2022-05-20 00:01:08.000 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | 2022-05-19 00:01:08.000 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | 2022-05-18 00:01:08.000 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | 2022-05-17 00:01:08.000 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | 2022-05-16 00:01:08.000 | + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | 2022-05-15 00:01:08.000 | + +taos> select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc; + _wstart | _wend | count(*) | last(ts) | +====================================================================================================== + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | 2022-05-15 00:01:08.000 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | 2022-05-16 00:01:08.000 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | 2022-05-17 00:01:08.000 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | 2022-05-18 00:01:08.000 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | 2022-05-19 00:01:08.000 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | 2022-05-20 00:01:08.000 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | 2022-05-21 00:01:08.000 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | 2022-05-22 00:01:08.000 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | 2022-05-23 00:01:08.000 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | 2022-05-24 00:01:08.000 | + +taos> explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc, count(*) desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=asc (columns=4 width=32) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 3.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=5 width=44) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=5 width=40 +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=5 width=40 +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, last(ts) from (select _wstart as ts, _wend, count(*), last(ts) from meters state_window(c2) order by _wend desc) interval(1h) order by _wstart desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=24 input_order=desc output_order=desc ) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Time Window: interval=1h offset=0a sliding=1h +*************************** 4.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 5.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=desc ) +*************************** 6.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 7.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 8.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 9.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=3 width=24) +*************************** 10.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 11.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=4 width=36) +*************************** 12.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 14.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 16.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 17.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 18.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 20.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 22.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 23.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 24.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 25.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 26.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 27.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, last(ts) from (select _wstart as ts, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc) interval(1h) order by _wstart desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=24 input_order=asc output_order=desc ) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Time Window: interval=1h offset=0a sliding=1h +*************************** 4.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 5.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=asc ) +*************************** 6.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 7.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 8.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 9.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=4 width=36) +*************************** 10.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 12.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 15.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 16.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 17.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 18.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 20.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 21.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 22.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 23.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 24.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 25.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + diff --git a/tests/script/tsim/query/sys_tbname.sim b/tests/script/tsim/query/sys_tbname.sim index 849aeb2ac5dabc46e6770de956e4e885e988e2ab..f49a8e0a7d7e4c0780e77bb3ad3f1ae8ef5a3b65 100644 --- a/tests/script/tsim/query/sys_tbname.sim +++ b/tests/script/tsim/query/sys_tbname.sim @@ -131,4 +131,8 @@ print $rows if $rows != 9 then return -1 endi + +print =========================== td-24781 +sql select DISTINCT (`precision`) from `information_schema`.`ins_databases` PARTITION BY `precision` + #system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/t/explain_tsorder.sql b/tests/script/tsim/query/t/explain_tsorder.sql new file mode 100644 index 0000000000000000000000000000000000000000..53bfb9a597b47a66d84b88bdcea35c071a6ec439 --- /dev/null +++ b/tests/script/tsim/query/t/explain_tsorder.sql @@ -0,0 +1,162 @@ +use test; +explain verbose true select _wstart, last(ts), avg(c2) from meters interval(10s) order by _wstart desc\G; +explain verbose true select _wstart, last(ts), avg(c2) from meters interval(10s) order by _wstart asc\G; +explain verbose true select _wstart, first(ts), avg(c2) from meters interval(10s) order by _wstart asc\G; +explain verbose true select _wstart, first(ts), avg(c2) from meters interval(10s) order by _wstart desc\G; +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s)) order by d\G; +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s)) order by d desc\G; +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a) order by d\G; +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) order by d\G; +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a) order by d desc\G; +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) order by d desc\G; + + +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) order by d\G; +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) order by d\G; +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) order by d desc\G; +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) order by d desc\G; + +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) group by c order by d\G; +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) group by c order by d\G; +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) group by c order by d desc\G; +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) group by c order by d desc\G; + + +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) where a > 10000 and a < 20000 interval(10s) fill(NULL) order by d\G; +explain verbose true select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) where a > 10000 and a < 20000 interval(10s) fill(NULL) order by d\G; +explain verbose true select last(b) as d from (select last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) where b > 10000 and b < 20000 interval(10s) fill(NULL) order by d\G; +explain verbose true select last(b) as d from (select last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) where b > 10000 and b < 20000 interval(10s) fill(NULL) order by d desc\G; + + +select _wstart, last(ts), avg(c2) from meters interval(10s) order by _wstart desc; +select _wstart, last(ts), avg(c2) from meters interval(10s) order by _wstart asc; +select _wstart, first(ts), avg(c2) from meters interval(10s) order by _wstart asc; +select _wstart, first(ts), avg(c2) from meters interval(10s) order by _wstart desc; +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s)) order by d; +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s)) order by d desc; +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a) order by d; +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) order by d; +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a) order by d desc; +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) order by d desc; + + +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) order by d; +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) order by d; +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) order by d desc; +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) order by d desc; + +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) group by c order by d +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) group by c order by d; +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b) group by c order by d desc; +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) group by c order by d desc; + +select last(a) as d from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) where a > '2022-05-15 00:01:00.000' and a < '2022-05-19 00:01:08.000' interval(10s) order by d; +select last(b) as d from (select last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) where b > '2022-05-15 00:01:00.000' and b < '2022-05-19 00:01:08.000' interval(10s) order by d; +select last(b) as d from (select last(ts) as b, avg(c2) as c from meters interval(10s) order by b desc) where b > '2022-05-15 00:01:00.000' and b < '2022-05-19 00:01:08.000' interval(10s) order by d desc; +select _wstart, first(a) as d, avg(c) from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) where a > '2022-05-15 00:01:00.000' and a < '2022-05-21 00:01:08.000' interval(5h) fill(linear) order by d desc; + +explain verbose true select _wstart, first(a) as d, avg(c) from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) where a > '2022-05-15 00:01:00.000' and a < '2022-05-21 00:01:08.000' interval(5h) fill(linear) order by d desc\G; +explain verbose true select _wstart, first(a) as d, avg(c) from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a asc) where a > '2022-05-15 00:01:00.000' and a < '2022-05-21 00:01:08.000' interval(5h) fill(linear) order by d desc\G; + +explain verbose true select * from (select ts as a, c2 as b from meters order by c2 desc)\G; +select * from (select ts as a, c2 as b from meters order by c2 desc); + +explain verbose true select * from (select ts as a, c2 as b from meters order by c2 desc) order by a desc\G; +select * from (select ts as a, c2 as b from meters order by c2 desc) order by a desc; + +explain verbose true select a.ts, a.c2, b.c2 from meters as a join meters as b on a.ts = b.ts\G; +explain verbose true select a.ts, a.c2, b.c2 from meters as a join meters as b on a.ts = b.ts order by a.ts\G; +select a.ts, a.c2, b.c2 from meters as a join meters as b on a.ts = b.ts; +select a.ts, a.c2, b.c2 from meters as a join meters as b on a.ts = b.ts order by a.ts desc; +explain verbose true select a.ts, a.c2, b.c2 from meters as a join (select ts, c2 from meters order by ts desc) b on a.ts = b.ts order by a.ts desc\G; +explain verbose true select a.ts, a.c2, b.c2 from meters as a join (select ts, c2 from meters order by ts desc) b on a.ts = b.ts order by a.ts asc\G; +select a.ts, a.c2, b.c2 from meters as a join (select * from meters order by ts desc) b on a.ts = b.ts order by a.ts asc; + +select ts, c2 from meters order by c2; +select ts, c2 from meters order by c2 limit 4; +select ts, c2 from meters order by c2 limit 2,2; + +select ts, c2 from meters order by ts asc, c2 desc limit 10; +select ts, c2 from meters order by ts asc, c2 desc limit 5,5; + +select ts, c2 from d1 order by c2; +select ts, c2 from d1 order by c2 limit 4; +select ts, c2 from d1 order by c2 limit 2,2; + +select ts, c2 from d1 order by ts asc, c2 desc limit 10; +select ts, c2 from d1 order by ts asc, c2 desc limit 5,5; + +select _wstart, first(a) as d, avg(c) from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) where a > '2022-05-15 00:01:00.000' and a < '2022-05-21 00:01:08.000' interval(5h) fill(linear) order by avg(c) desc; +select _wstart, first(a) as d, avg(c) from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) where a > '2022-05-15 00:01:00.000' and a < '2022-05-21 00:01:08.000' interval(5h) fill(linear) order by avg(c) desc limit 2; +select _wstart, first(a) as d, avg(c) from (select _wstart as a, last(ts) as b, avg(c2) as c from meters interval(10s) order by a desc) where a > '2022-05-15 00:01:00.000' and a < '2022-05-21 00:01:08.000' interval(5h) fill(linear) order by avg(c) desc limit 2,6; + +select last(ts), c2 as d from d1 group by c2 order by c2 desc limit 10; +select last(ts), c2 as d from d1 group by c2 order by c2 desc limit 2,8; +select last(ts), c2 as d from d1 group by c2 order by c2 desc limit 9,1; +select last(ts), c2 as d from d1 group by c2 order by c2 asc limit 2,8; +select last(ts), c2 as d from d1 group by c2 order by c2 asc limit 9,1; +select last(ts) as ts, c2 as d from d1 group by c2 order by ts desc, c2 asc limit 10; +select last(ts) as ts, c2 as d from d1 group by c2 order by ts desc, c2 asc limit 2,8; +select last(ts) as ts, c2 as d from d1 group by c2 order by ts desc, c2 asc limit 9,1; + +explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100\G; +explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart desc\G; +explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart asc\G; + +explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend desc\G; +explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend asc\G; + +select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100; +select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart desc; +select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart asc; + +select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend desc; +select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend asc; + +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h)\G; +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc\G; +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc\G; + +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc\G; +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc\G; + +select _wstart, _wend, count(*) from meters session(ts, 1h); +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc; +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc; + +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc; +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc; + + +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h)\G; +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc\G; +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc\G; + +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc\G; +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc\G; + +select _wstart, _wend, count(*) from meters session(ts, 1h); +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc; +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc; + +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc; +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc; + +explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2)\G; +explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart desc\G; +explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart asc\G; + +explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend desc\G; +explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc\G; + +select _wstart, _wend, count(*), last(ts) from meters state_window(c2); +select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart desc; +select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart asc; + +select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend desc; +select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc; + +explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc, count(*) desc\G; + +explain verbose true select _wstart, _wend, last(ts) from (select _wstart as ts, _wend, count(*), last(ts) from meters state_window(c2) order by _wend desc) interval(1h) order by _wstart desc\G; +explain verbose true select _wstart, _wend, last(ts) from (select _wstart as ts, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc) interval(1h) order by _wstart desc\G; diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index e539f115317abcd2b64279ec476f8f3464a559d5..fbf9d50c2568bb571349ae1b5874fed7c217f0e1 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -8,6 +8,9 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c udf -v 1 system sh/exec.sh -n dnode1 -s start sql connect +sql alter user root pass 'taosdata2' +system sh/exec.sh -n dnode1 -s stop +system sh/exec.sh -n dnode1 -s start print ======== step1 udf system sh/compile_udf.sh diff --git a/tests/script/tsim/query/udfpy.sim b/tests/script/tsim/query/udfpy.sim index 9e0492ffd95e3baed20a77ddf3423a1e8872c1d0..2a4daedd5d6afba79b1aba63769ab584f7d8e644 100644 --- a/tests/script/tsim/query/udfpy.sim +++ b/tests/script/tsim/query/udfpy.sim @@ -15,6 +15,7 @@ system sh/prepare_pyudf.sh system mkdir -p /tmp/pyudf system cp sh/pybitand.py /tmp/pyudf/ system cp sh/pyl2norm.py /tmp/pyudf/ +system cp sh/pycumsum.py /tmp/pyudf/ system ls /tmp/pyudf sql create database udf vgroups 3; @@ -280,6 +281,18 @@ if $data20 != 8.000000000 then return -1 endi +#sql create aggregate function pycumsum as '/tmp/pyudf/pycumsum.py' outputtype double bufSize 128 language 'python'; +#sql select pycumsum(f2) from udf.t2 +#print ======= pycumsum +#print $rows $data00 +#if $rows != 1 then +# return -1 +#endi +#if $data00 != 20.000000000 then +# return -1 +#endi +#sql drop function pycumsum + sql create or replace function bit_and as '/tmp/udf/libbitand.so' outputtype int sql select func_version from information_schema.ins_functions where name='bit_and' if $data00 != 1 then diff --git a/tests/script/tsim/query/unionall_as_table.sim b/tests/script/tsim/query/unionall_as_table.sim index 4d8f99071829050a5da6de2c6b1ea2feb41d27bb..f8145d4e9763621a37719fb5e6c0f44728610aea 100644 --- a/tests/script/tsim/query/unionall_as_table.sim +++ b/tests/script/tsim/query/unionall_as_table.sim @@ -42,4 +42,12 @@ endi if $data00 != 4 then return -1 endi + +sql create table ctcount(ts timestamp, f int); +sql insert into ctcount(ts) values(now)(now+1s); +sql select count(*) from (select f from ctcount); +print $data00 +if $data00 != 2 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/sma/drop_sma.sim b/tests/script/tsim/sma/drop_sma.sim index 0d2712f8db9f65ae051466d00f5f39ab9d27093f..8fd8ebdcfd28849d64a85fe23df0727d232287ab 100644 --- a/tests/script/tsim/sma/drop_sma.sim +++ b/tests/script/tsim/sma/drop_sma.sim @@ -129,6 +129,7 @@ sql DROP INDEX sma_index_3 ; print ========== step8 sql drop database if exists db; +sleep 2000 sql create database db duration 300; sql use db; sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); diff --git a/tests/script/tsim/sma/rsmaCreateInsertQuery.sim b/tests/script/tsim/sma/rsmaCreateInsertQuery.sim index 7932cb68ac3251e78ea356796e20d6bfd1cffbce..b3144e4e0dd217319a0d58bf3222360fcd5fa355 100644 --- a/tests/script/tsim/sma/rsmaCreateInsertQuery.sim +++ b/tests/script/tsim/sma/rsmaCreateInsertQuery.sim @@ -4,9 +4,6 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect -#todo xukaili sma should use rocksdb. -return 1 - print =============== create database with retentions sql create database d0 retentions 5s:7d,10s:21d,15s:365d; sql use d0 diff --git a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim index 75969b1d0eddf9f122655d5483ec9c827b5ab550..0b3938d77342d1d572196ceabf10cc39b7bdd558 100644 --- a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim +++ b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim @@ -4,7 +4,7 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect -#todo xukaili sma should use rocksdb. +#todo wait for streamState checkpoint return 1 print =============== create database with retentions @@ -13,17 +13,18 @@ sql use d0 print =============== create super table and register rsma sql create table if not exists stb (ts timestamp, c1 int, c2 float) tags (city binary(20),district binary(20)) rollup(max) max_delay 5s,5s watermark 2s,3s; +sql create table if not exists stb1 (ts timestamp, c1 int, c2 float) tags (city binary(20),district binary(20)) rollup(max) max_delay 5s,5s watermark 2s,3s; sql show stables -if $rows != 1 then +if $rows != 2 then return -1 endi print =============== create child table -sql create table ct1 using stb tags("BeiJing", "ChaoYang"); +sql create table ct1 using stb tags("BeiJing", "ChaoYang") ct_1 using stb1 tags("BeiJing", "ChaoYang"); sql show tables -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -31,6 +32,9 @@ print =============== insert data and trigger rollup sql insert into ct1 values(now, 10, 10.0); sql insert into ct1 values(now+1s, 1, 1.0); sql insert into ct1 values(now+2s, 100, 100.0); +sql insert into ct_1 values(now, 10, 10.0); +sql insert into ct_1 values(now+1s, 1, 1.0); +sql insert into ct_1 values(now+2s, 100, 100.0); print =============== wait maxdelay 5+2 seconds for results sleep 7000 @@ -44,6 +48,20 @@ if $rows > 2 then return -1 endi +if $data01 != 100 then + if $data01 != 10 then + print retention level 2 file result $data01 != 100 or 10 + return -1 + endi +endi + +sql select * from ct_1; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 2 file rows $rows > 2 + return -1 +endi if $data01 != 100 then if $data01 != 10 then @@ -68,6 +86,21 @@ if $data01 != 100 then endi endi +sql select * from ct_1 where ts > now-8d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 1 file rows $rows > 2 + return -1 +endi + +if $data01 != 100 then + if $data01 != 10 then + print retention level 1 file result $data01 != 100 or 10 + return -1 + endi +endi + print =============== select * from retention level 0 from memory sql select * from ct1 where ts > now-3d; print $data00 $data01 $data02 @@ -84,6 +117,21 @@ if $data01 != 10 then return -1 endi +sql select * from ct_1 where ts > now-3d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 + +if $rows < 1 then + print retention level 0 file rows $rows < 1 + return -1 +endi + +if $data01 != 10 then + print retention level 0 file result $data01 != 10 + return -1 +endi + #=================================================================== system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start @@ -100,6 +148,22 @@ if $rows > 2 then endi +if $data01 != 100 then + if $data01 != 10 then + print retention level 2 file result $data01 != 100 or 10 + return -1 + endi +endi + +sql select * from ct_1; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 2 file rows $rows > 2 + return -1 +endi + + if $data01 != 100 then if $data01 != 10 then print retention level 2 file result $data01 != 100 or 10 @@ -123,6 +187,21 @@ if $data01 != 100 then endi endi +sql select * from ct_1 where ts > now-8d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 1 file rows $rows > 2 + return -1 +endi + +if $data01 != 100 then + if $data01 != 10 then + print retention level 1 file result $data01 != 100 or 10 + return -1 + endi +endi + print =============== select * from retention level 0 from memory after reboot sql select * from ct1 where ts > now-3d; print $data00 $data01 $data02 @@ -139,6 +218,21 @@ if $data01 != 10 then return -1 endi +sql select * from ct_1 where ts > now-3d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 + +if $rows < 1 then + print retention level 0 file rows $rows < 1 + return -1 +endi + +if $data01 != 10 then + print retention level 0 file result $data01 != 10 + return -1 +endi + #==================== flush database to trigger commit data to file sql flush database d0; @@ -161,6 +255,21 @@ if $data01 != 100 then endi endi +sql select * from ct_1; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 2 file rows $rows > 2 + return -1 +endi + +if $data01 != 100 then + if $data01 != 10 then + print retention level 2 file result $data01 != 100 or 10 + return -1 + endi +endi + print =============== select * from retention level 1 from file sql select * from ct1 where ts > now-8d; print $data00 $data01 $data02 @@ -177,6 +286,21 @@ if $data01 != 100 then endi endi +sql select * from ct_1 where ts > now-8d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 1 file rows $rows > 2 + return -1 +endi + +if $data01 != 100 then + if $data01 != 10 then + print retention level 1 file result $data01 != 100 or 10 + return -1 + endi +endi + print =============== select * from retention level 0 from file sql select * from ct1 where ts > now-3d; print $data00 $data01 $data02 @@ -192,9 +316,25 @@ if $data01 != 10 then return -1 endi +sql select * from ct_1 where ts > now-3d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 +if $rows < 1 then + print retention level 0 file rows $rows < 1 + return -1 +endi + +if $data01 != 10 then + print retention level 0 file result $data01 != 10 + return -1 +endi + print =============== insert after rsma qtaskinfo recovery sql insert into ct1 values(now, 50, 500.0); sql insert into ct1 values(now+1s, 40, 40.0); +sql insert into ct_1 values(now, 50, 500.0); +sql insert into ct_1 values(now+1s, 40, 40.0); print =============== wait maxdelay 5+2 seconds for results sleep 7000 @@ -217,11 +357,37 @@ endi if $data02 != 500.00000 then if $data02 != 100.00000 then - print retention level 1 file/mem result $data02 != 500.00000 or 100.00000 + if $data02 != 10.00000 then + print retention level 1 file/mem result $data02 != 500.00000 or 100.00000 or 10.00000 + return -1 + endi + endi +endi + +sql select * from ct_1; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 2 file/mem rows $rows > 2 + return -1 +endi + +if $data01 != 100 then + if $data01 != 10 then + print retention level 2 file/mem result $data01 != 100 or 10 return -1 endi endi +if $data02 != 500.00000 then + if $data02 != 100.00000 then + if $data02 != 10.00000 then + print retention level 1 file/mem result $data02 != 500.00000 or 100.00000 or 10.00000 + return -1 + endi + endi +endi + print =============== select * from retention level 1 from file and memory after rsma qtaskinfo recovery sql select * from ct1 where ts > now-8d; print $data00 $data01 $data02 @@ -240,11 +406,37 @@ endi if $data02 != 500.00000 then if $data02 != 100.00000 then - print retention level 1 file/mem result $data02 != 500.00000 or 100.00000 + if $data02 != 10.00000 then + print retention level 1 file/mem result $data02 != 500.00000 or 100.00000 or 10.00000 + return -1 + endi + endi +endi + +sql select * from ct_1 where ts > now-8d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 1 file/mem rows $rows > 2 + return -1 +endi + +if $data01 != 100 then + if $data01 != 10 then + print retention level 1 file/mem result $data01 != 100 or 10 return -1 endi endi +if $data02 != 500.00000 then + if $data02 != 100.00000 then + if $data02 != 10.00000 then + print retention level 1 file/mem result $data02 != 500.00000 or 100.00000 or 10.00000 + return -1 + endi + endi +endi + print =============== select * from retention level 0 from file and memory after rsma qtaskinfo recovery sql select * from ct1 where ts > now-3d; @@ -295,6 +487,61 @@ if $data42 != 40.00000 then return -1 endi +sql select * from ct_1 where ts > now-3d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 +print $data30 $data31 $data32 +print $data40 $data41 $data42 + +if $rows < 1 then + print retention level 0 file/mem rows $rows < 1 + return -1 +endi + +if $data01 != 10 then + print retention level 0 file/mem result $data01 != 10 + return -1 +endi + +if $data11 != 1 then + print retention level 0 file/mem result $data11 != 1 + return -1 +endi + +if $data21 != 100 then + print retention level 0 file/mem result $data21 != 100 + return -1 +endi + +if $data31 != 50 then + print retention level 0 file/mem result $data31 != 50 + return -1 +endi + +if $data32 != 500.00000 then + print retention level 0 file/mem result $data32 != 500.00000 + return -1 +endi + + +if $data41 != 40 then + print retention level 0 file/mem result $data41 != 40 + return -1 +endi + +if $data42 != 40.00000 then + print retention level 0 file/mem result $data42 != 40.00000 + return -1 +endi + +print =============== drop stb1 +sql drop table stb1; +sql flush database d0; +print =============== select * from retention level 0 from file and memory after rsma qtaskinfo recovery +sql_error select * from ct_1 where ts > now-3d; +sql_error select * from ct_1 where ts > now-8d; +sql_error select * from ct_1; system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/sma/tsmaCreateInsertQuery.sim b/tests/script/tsim/sma/tsmaCreateInsertQuery.sim index 9b6fc96bc00fde8418bf1d29d4c8cd8fc7f02091..692212d511160033e909836a12e2f4747251f574 100644 --- a/tests/script/tsim/sma/tsmaCreateInsertQuery.sim +++ b/tests/script/tsim/sma/tsmaCreateInsertQuery.sim @@ -78,7 +78,7 @@ if $rows != 5 then return -1 endi -print =============== select * from stb from memory in designated vgroup +print =============== select with _wstart from stb from memory in designated vgroup sql select _wstart, _wend, min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m); print $data00 $data01 $data02 $data03 $data04 print $data10 $data11 $data12 $data13 $data14 @@ -102,6 +102,30 @@ if $data04 != 20 then return -1 endi +print =============== select without _wstart from stb from memory in designated vgroup +sql select min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m); +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data00 != -13 then + print data00 $data00 != -13 + return -1 +endi + +if $data01 != 20.00000 then + print data01 $data01 != 20.00000 + return -1 +endi + +if $data02 != 20 then + print data02 $data02 != 20 + return -1 +endi + print =============== select * from stb from memory in common vgroups sql select _wstart, _wend, min(c1),max(c2),max(c1),max(c3) from stb interval(5m,10s) sliding(5m); print $data00 $data01 $data02 $data03 $data04 $data05 @@ -148,7 +172,7 @@ if $rows != 5 then return -1 endi -print =============== select * from stb from memory in designated vgroup after reboot +print =============== select with _wstart from stb from memory in designated vgroup after reboot sql select _wstart, _wend, min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m); print $data00 $data01 $data02 $data03 $data04 print $data10 $data11 $data12 $data13 $data14 @@ -172,6 +196,30 @@ if $data04 != 20 then return -1 endi +print =============== select without _wstart from stb from memory in designated vgroup after reboot +sql select min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m); +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data00 != -13 then + print data00 $data00 != -13 + return -1 +endi + +if $data01 != 20.00000 then + print data01 $data01 != 20.00000 + return -1 +endi + +if $data02 != 20 then + print data02 $data02 != 20 + return -1 +endi + print =============== select * from stb from memory in common vgroups after reboot sql select _wstart, _wend, min(c1),max(c2),max(c1),max(c3) from stb interval(5m,10s) sliding(5m); print $data00 $data01 $data02 $data03 $data04 $data05 @@ -218,7 +266,7 @@ if $rows != 5 then return -1 endi -print =============== select * from stb from file in designated vgroup +print =============== select with _wstart from stb from file in designated vgroup sql select _wstart, _wend, min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m); print $data00 $data01 $data02 $data03 $data04 if $rows != 1 then @@ -241,6 +289,29 @@ if $data04 != 20 then return -1 endi +print =============== select without _wstart from stb from file in designated vgroup +sql select _wend, min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m); +print $data00 $data01 $data02 $data03 +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data01 != -13 then + print data01 $data01 != -13 + return -1 +endi + +if $data02 != 20.00000 then + print data02 $data02 != 20.00000 + return -1 +endi + +if $data03 != 20 then + print data03 $data03 != 20 + return -1 +endi + print =============== select * from stb from file in common vgroups sql select _wstart, _wend, min(c1),max(c2),max(c1),max(c3) from stb interval(5m,10s) sliding(5m); print $data00 $data01 $data02 $data03 $data04 $data05 @@ -269,6 +340,95 @@ if $data05 != 30.000000000 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT +print =============== select with _wstart/order by _wstart from stb from file in designated vgroup +sql select _wstart, _wend, min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m) order by _wstart; +print $data00 $data01 $data02 $data03 $data04 +if $rows != 1 then + print rows $rows != 1 + return -1 +endi +if $data02 != -13 then + print data02 $data02 != -13 + return -1 +endi + +if $data03 != 20.00000 then + print data03 $data03 != 20.00000 + return -1 +endi + +if $data04 != 20 then + print data04 $data04 != 20 + return -1 +endi + +print =============== select without _wstart/with order by _wstart from stb from file in designated vgroup +sql select _wend, min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m) order by _wstart; +print $data00 $data01 $data02 $data03 +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data01 != -13 then + print data01 $data01 != -13 + return -1 +endi + +if $data02 != 20.00000 then + print data02 $data02 != 20.00000 + return -1 +endi + +if $data03 != 20 then + print data03 $data03 != 20 + return -1 +endi + +print =============== select * from stb from file in common vgroups +sql select _wstart, _wend, min(c1),max(c2),max(c1),max(c3) from stb interval(5m,10s) sliding(5m) order by _wstart; +print $data00 $data01 $data02 $data03 $data04 $data05 +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data02 != -13 then + print data02 $data02 != -13 + return -1 +endi + +if $data03 != 20.00000 then + print data03 $data03 != 20.00000 + return -1 +endi + +if $data04 != 20 then + print data04 $data04 != 20 + return -1 +endi + +if $data05 != 30.000000000 then + print data05 $data05 != 30.000000000 + return -1 +endi + +sql delete from stb; + +print =============== query after delete in common vgroups +sql select _wstart, _wend, min(c1),max(c2),max(c1),max(c3) from stb interval(5m,10s) sliding(5m) order by _wstart; +if $rows != 0 then + print rows $rows != 0 + return -1 +endi + +sleep 2000 +print =============== query after delete in designated vgroups +sql select _wend, min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m) order by _wstart; +if $rows != 0 then + print rows $rows != 0 + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/stream/basic3.sim b/tests/script/tsim/stream/basic3.sim index 2df33541b461b1f2fe1c772cb80550e421fe56aa..f18061a6df012b5093beef6c8f1588a61b69f5b8 100644 --- a/tests/script/tsim/stream/basic3.sim +++ b/tests/script/tsim/stream/basic3.sim @@ -1,11 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c debugflag -v 131 system sh/cfg.sh -n dnode1 -c keepColumnName -v 1 system sh/exec.sh -n dnode1 -s start -sleep 5000 - +sleep 1000 sql connect print ========== interval\session\state window @@ -32,7 +30,6 @@ sql create stream streamd6 into streamt6 as select ca, _wstart,_wend, count(*), sql alter local 'keepColumnName' '1' - sql CREATE STABLE `meters_test_data` (`ts` TIMESTAMP, `close` FLOAT, `parttime` TIMESTAMP, `parttime_str` VARCHAR(32)) TAGS (`id` VARCHAR(32)); sql_error create stream realtime_meters fill_history 1 into realtime_meters as select last(parttime),first(close),last(close) from meters_test_data partition by tbname state_window(parttime_str); @@ -58,17 +55,13 @@ sql_error create stream streamd11 into streamd11 as select _wstart, _wend, count sql alter local 'keepColumnName' '0' sql create stream realtime_meters fill_history 1 into realtime_meters as select last(parttime),first(close),last(close) from meters_test_data partition by tbname state_window(parttime_str); - sql desc realtime_meters; - if $rows == 0 then return -1 endi -sql create stream streamd7 into streamt7 as select _wstart, _wend, count(*), first(ca), last(ca) from t1 interval(10s); - +sql create stream streamd7 into streamt7 as select _wstart t1, _wend t2, count(*), first(ca), last(ca) from t1 interval(10s); sql desc streamt7; - if $rows == 0 then return -1 endi @@ -76,12 +69,11 @@ endi sql create stream streamd71 into streamt71 as select _wstart, _wend, count(*) as ca, first(ca), last(ca) as c2 from t1 interval(10s); sql desc streamt71; - if $rows == 0 then return -1 endi -sleep 3000 +sleep 1000 sql drop stream if exists streamd1; sql drop stream if exists streamd2; @@ -93,23 +85,19 @@ sql drop stream if exists streamd6; sql create stream streamd10 into streamd10 as select _wstart, _wend, count(*), first(ca), last(cb) as c2 from t1 interval(10s); sql desc streamd10; - if $rows == 0 then return -1 endi sql_error create stream streamd11 into streamd11 as select _wstart, _wend, count(*), last(ca), last(ca) from t1 interval(10s); - sql create stream streamd12 into streamd12 as select _wstart, _wend, count(*), last(ca), last(cb) as c2 from t1 interval(10s); - sql desc streamd12; if $rows == 0 then return -1 endi - _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index 959b32fa5990ef72bc6d469ea6f24de59e5b1a0d..5bb03c8cbf0fcd03e6a401e60465248a91bfdfe5 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -1,36 +1,11 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 system sh/exec.sh -n dnode1 -s start sleep 50 sql connect -sql create dnode $hostname2 port 7200 - -system sh/exec.sh -n dnode2 -s start - -print ===== step1 -$x = 0 -step1: - $x = $x + 1 - sleep 1000 - if $x == 10 then - print ====> dnode not ready! - return -1 - endi -sql select * from information_schema.ins_dnodes -print ===> $data00 $data01 $data02 $data03 $data04 $data05 -print ===> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 2 then - return -1 -endi -if $data(1)[4] != ready then - goto step1 -endi -if $data(2)[4] != ready then - goto step1 -endi + print ===== step2 sql drop stream if exists stream_t1; @@ -248,10 +223,56 @@ sql insert into ts3 values(1648791223002,2,2,3,1.1); sql insert into ts4 values(1648791233003,3,2,3,2.1); sql insert into ts3 values(1648791243004,4,2,43,73.1); sql insert into ts4 values(1648791213002,24,22,23,4.1); + +$loop_count = 0 +loop032: + +$loop_count = $loop_count + 1 +if $loop_count == 30 then + return -1 +endi + +sleep 1000 +print 6-0 select * from streamtST1; +sql select * from streamtST1; + +if $rows != 4 then + print =====rows=$rows + goto loop032 +endi + +if $data01 != 8 then + print =6====data01=$data01 + goto loop032 +endi + sql insert into ts3 values(1648791243005,4,20,3,3.1); sql insert into ts4 values(1648791243006,4,2,3,3.1) (1648791243007,4,2,3,3.1) ; sql insert into ts3 values(1648791243008,4,2,30,3.1) (1648791243009,4,2,3,3.1) (1648791243010,4,2,3,3.1) ; sql insert into ts4 values(1648791243011,4,2,3,3.1) (1648791243012,34,32,33,3.1) (1648791243013,4,2,3,3.1) (1648791243014,4,2,13,3.1); + +$loop_count = 0 +loop033: + +$loop_count = $loop_count + 1 +if $loop_count == 30 then + return -1 +endi + +sleep 1000 +print 6-1 select * from streamtST1; +sql select * from streamtST1; + +if $rows != 4 then + print =====rows=$rows + goto loop033 +endi + +if $data01 != 8 then + print =6====data01=$data01 + goto loop033 +endi + sql insert into ts3 values(1648791243005,4,42,3,3.1) (1648791243003,4,2,33,3.1) (1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) ; $loop_count = 0 diff --git a/tests/script/tsim/stream/fillHistoryTransform.sim b/tests/script/tsim/stream/fillHistoryTransform.sim new file mode 100644 index 0000000000000000000000000000000000000000..fe58b76b78d6a2bd9614d05022b4f2694bf23db4 --- /dev/null +++ b/tests/script/tsim/stream/fillHistoryTransform.sim @@ -0,0 +1,405 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database test vgroups 1; +sql select * from information_schema.ins_databases +if $rows != 3 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use test; + +print =====step1 + +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql insert into t1 values(1648791213000,10,2,3,1.0); + +sql create stream stream0 trigger at_once fill_history 1 IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, sum(a) from t1 interval(10s); + +$loop_count = 0 +loop00: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop00 +endi + +if $data01 != 10 then + print =====data01=$data01 + goto loop00 +endi + +sql insert into t1 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 +loop0: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop0 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop0 +endi + +sql insert into t1 values(1648791213001,2,2,3,1.0); + +$loop_count = 0 +loop1: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop1 +endi + +if $data01 != 3 then + print ======$data01 + goto loop1 +endi + + +sql insert into t1 values(1648791223001,3,2,3,1.0); + +sql insert into t1 values(1648791223002,4,2,3,1.0); + +$loop_count = 0 +loop2: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt; + +if $rows != 2 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop2 +endi + +if $data01 != 3 then + print ======$data01 + goto loop2 +endi + +if $data11 != 7 then + print ======$data01 + goto loop2 +endi + +print =====step1 over + +print =====step2 + +sql create database test1 vgroups 4; + +sql use test1; + +sql create stable st(ts timestamp,a int,b int,c int,d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql insert into t1 values(1648791213000,10,2,3,1.0); + +sql create stream stream1 trigger at_once fill_history 1 IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, sum(a) from st interval(10s); + +$loop_count = 0 +loop00: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt1; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop00 +endi + +if $data01 != 10 then + print =====data01=$data01 + goto loop00 +endi + +sql insert into t1 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 +loop0: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt1; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop0 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop0 +endi + +sql insert into t1 values(1648791213001,2,2,3,1.0); + +$loop_count = 0 +loop1: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt1; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop1 +endi + +if $data01 != 3 then + print ======$data01 + goto loop1 +endi + + +sql insert into t1 values(1648791223001,3,2,3,1.0); + +sql insert into t1 values(1648791223002,4,2,3,1.0); + +$loop_count = 0 +loop2: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt1; + +if $rows != 2 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop2 +endi + +if $data01 != 3 then + print ======$data01 + goto loop2 +endi + +if $data11 != 7 then + print ======$data01 + goto loop2 +endi + +print =====step2 over + +print =====step3 + +sql create database test2 vgroups 4; + +sql use test2; + +sql create stable st(ts timestamp,a int,b int,c int,d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql insert into t1 values(1648791213000,10,2,3,1.0); + +sql create stream stream2 trigger at_once fill_history 1 IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, sum(a) from st partition by ta interval(10s); + +$loop_count = 0 +loop00: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt2; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop00 +endi + +if $data01 != 10 then + print =====data01=$data01 + goto loop00 +endi + +sql insert into t1 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 +loop0: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt2; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop0 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop0 +endi + +sql insert into t1 values(1648791213001,2,2,3,1.0); + +$loop_count = 0 +loop1: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt2; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop1 +endi + +if $data01 != 3 then + print ======$data01 + goto loop1 +endi + + +sql insert into t1 values(1648791223001,3,2,3,1.0); + +sql insert into t1 values(1648791223002,4,2,3,1.0); + +$loop_count = 0 +loop2: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt2; + +if $rows != 2 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop2 +endi + +if $data01 != 3 then + print ======$data01 + goto loop2 +endi + +if $data11 != 7 then + print ======$data01 + goto loop2 +endi + +print =====step3 over + +print =====over + + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/ignoreExpiredData.sim b/tests/script/tsim/stream/ignoreExpiredData.sim index 27920dd539826fb13715b7daf3136156c643f8ed..884b7cbb5f6db4e6a371b7f3eef1b6a8062240cd 100644 --- a/tests/script/tsim/stream/ignoreExpiredData.sim +++ b/tests/script/tsim/stream/ignoreExpiredData.sim @@ -132,12 +132,12 @@ if $loop_count == 10 then return -1 endi -if $data01 != 1 then +if $data01 != 2 then print =====data01=$data01 goto loop4 endi -if $data02 != 1 then +if $data02 != 2 then print =====data02=$data02 goto loop4 endi diff --git a/tests/script/tsim/stream/partitionby.sim b/tests/script/tsim/stream/partitionby.sim index df4b60314fc8b0b52fa2f7075262cea2cd0106ce..9a660741e7eab7242564225377ba2fc77691604a 100644 --- a/tests/script/tsim/stream/partitionby.sim +++ b/tests/script/tsim/stream/partitionby.sim @@ -14,6 +14,7 @@ sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test0.streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st partition by ta,tb,tc interval(10s); +sleep 500 sql insert into ts1 values(1648791213001,1,12,3,1.0); sql insert into ts2 values(1648791213001,1,12,3,1.0); diff --git a/tests/script/tsim/stream/pauseAndResume.sim b/tests/script/tsim/stream/pauseAndResume.sim index fa7be19310a3a673b0330b9fa7b0849ae5a6714d..402e0086f7d3797eb66a7da89b92b97885e147b5 100644 --- a/tests/script/tsim/stream/pauseAndResume.sim +++ b/tests/script/tsim/stream/pauseAndResume.sim @@ -307,4 +307,63 @@ sql resume stream IF EXISTS streams66666666; print ===== step 4 over +print ===== step5 +sql drop stream if exists streams6; +sql drop database if exists test6; +sql create database test6 vgroups 10; +sql use test6; +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create table ts3 using st tags(3,2,2); +sql create table ts4 using st tags(4,2,2); +sql create stream streams6 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt6 as select _wstart, count(*) c1 from st interval(10s); + +sql insert into ts1 values(1648791213001,1,12,3,1.0); +sql insert into ts2 values(1648791213001,1,12,3,1.0); + +sql insert into ts3 values(1648791213001,1,12,3,1.0); +sql insert into ts4 values(1648791213001,1,12,3,1.0); + +sleep 1000 + +sql pause stream streams6; + +sleep 1000 + + +sql insert into ts1 values(1648791223001,1,12,3,1.0); +sql insert into ts2 values(1648791233001,1,12,3,1.0); + +sql resume stream streams6; + +sql insert into ts3 values(1648791243001,1,12,3,1.0); +sql insert into ts4 values(1648791253001,1,12,3,1.0); + +$loop_count = 0 +loop6: + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sleep 500 + +print 2 select * from streamt6; +sql select * from streamt6; + +if $rows != 5 then + print =====rows=$rows + print $data00 $data01 $data02 + print $data10 $data11 $data12 + print $data20 $data21 $data22 + print $data30 $data31 $data32 + print $data40 $data41 $data42 + print $data50 $data51 $data52 + goto loop6 +endi + +print ===== step5 over + system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index 633b806f71f0e8a595e54b8ea3649f62f4a51001..18893245fa6b735168f388563566537818d4c88f 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -442,9 +442,8 @@ sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams21 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); -sql create stream streams22 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); -sql create stream streams23 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(20s) sliding(10s); +sql create stream streams21 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt21 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); +sql create stream streams22 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt22 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791223001,2,2,2,1.1); @@ -468,7 +467,7 @@ if $loop_count == 10 then return -1 endi -sql select * from streamt; +sql select * from streamt21; # row 0 if $data01 != 2 then @@ -526,7 +525,7 @@ if $loop_count == 10 then return -1 endi -sql select * from streamt2; +sql select * from streamt22; # row 0 if $data01 != 4 then @@ -573,6 +572,29 @@ if $data32 != 8 then endi +$loop_count = 0 + +print step 7 +sql create database test3 vgroups 6; +sql use test3; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams23 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt23 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(20s) sliding(10s); + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791223001,2,2,2,1.1); +sql insert into t1 values(1648791233002,3,3,3,2.1); +sql insert into t1 values(1648791243003,4,4,4,3.1); +sql insert into t1 values(1648791213004,4,5,5,4.1); + +sql insert into t2 values(1648791213000,1,6,6,1.0); +sql insert into t2 values(1648791223001,2,7,7,1.1); +sql insert into t2 values(1648791233002,3,8,8,2.1); +sql insert into t2 values(1648791243003,4,9,9,3.1); +sql insert into t2 values(1648791213004,4,10,10,4.1); + $loop_count = 0 print step 7 @@ -585,7 +607,7 @@ if $loop_count == 10 then return -1 endi -sql select * from streamt3; +sql select * from streamt23; # row 0 if $rows != 5 then @@ -629,7 +651,7 @@ if $loop_count == 10 then return -1 endi -sql select * from streamt3; +sql select * from streamt23; # row 0 if $rows != 7 then @@ -688,6 +710,8 @@ sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 in sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791243000,2,1,1,1.0); +sleep 1000 + sql insert into t2 values(1648791273000,3,1,1,1.0); sql insert into t2 values(1648791313000,4,1,1,1.0); diff --git a/tests/script/tsim/sync/oneReplica5VgElect.sim b/tests/script/tsim/sync/oneReplica5VgElect.sim index e42ef5fb821c8de4b91be71ed8a0d95a13d09b7d..225093ed8fd59782716b4d39decaecdc62e18416 100644 --- a/tests/script/tsim/sync/oneReplica5VgElect.sim +++ b/tests/script/tsim/sync/oneReplica5VgElect.sim @@ -355,7 +355,7 @@ system sh/exec.sh -n dnode4 -s start $loop_cnt = 0 check_dnode_ready_2: $loop_cnt = $loop_cnt + 1 - sleep 200 + sleep 1500 if $loop_cnt == 10 then print ====> dnode not ready! return -1 diff --git a/tests/script/tsim/table/hash.sim b/tests/script/tsim/table/hash.sim index 664f86713720e1ae1969027508e2931d23397f08..45ce689b5ac8dabf29066dd0b6ac98d9d78e8155 100644 --- a/tests/script/tsim/table/hash.sim +++ b/tests/script/tsim/table/hash.sim @@ -7,11 +7,11 @@ sql connect #sql create database d1 vgroups 2 sql create database d1 vgroups 2 table_prefix 3 table_suffix 2 sql select * from information_schema.ins_databases -print $data(d1)[27] $data(d1)[28] -if $data(d1)[27] != 3 then +print $data(d1)[25] $data(d1)[26] +if $data(d1)[25] != 3 then return -1 endi -if $data(d1)[28] != 2 then +if $data(d1)[26] != 2 then return -1 endi diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index 4551228f2f67e6da9beed93f984e93113a77055f..fe6ec04a205313f084b120bf7d7bdf0d6096d916 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -87,7 +86,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -162,7 +160,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -236,7 +233,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic1Of2Cons.sim b/tests/script/tsim/tmq/basic1Of2Cons.sim index 51d39e8d110f0be00e3a16cf7cdd47e6be226a78..c12351cbe81538ad87fdde313bd0e4f4c074875b 100644 --- a/tests/script/tsim/tmq/basic1Of2Cons.sim +++ b/tests/script/tsim/tmq/basic1Of2Cons.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -87,7 +86,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for stb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -193,7 +191,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -298,7 +295,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic2.sim b/tests/script/tsim/tmq/basic2.sim index 8356a60b672c54feb37663d5c3b8f3391c99a456..5c7528ea5dc749da644b83f0928f7b3cc35f8c7a 100644 --- a/tests/script/tsim/tmq/basic2.sim +++ b/tests/script/tsim/tmq/basic2.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -122,7 +121,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -182,7 +180,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic2Of2Cons.sim b/tests/script/tsim/tmq/basic2Of2Cons.sim index 63e7e2dcf4e600e340f6c3767ab91dfd19fc5338..23598c17a4bce516d2b0c888cb55c6405766f2d8 100644 --- a/tests/script/tsim/tmq/basic2Of2Cons.sim +++ b/tests/script/tsim/tmq/basic2Of2Cons.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -151,7 +150,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -241,7 +239,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim index cfdae059dc862d4177d12891ffd9499900ba3b41..1223a94fa7e666540d4e790440c2a9039ab2feab 100644 --- a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -172,7 +171,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -266,7 +264,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic3.sim b/tests/script/tsim/tmq/basic3.sim index a64dd6924d7c30dec12e315924439b37019af500..8bb34cefa2382fb89856cf477b9a1bd82daf800e 100644 --- a/tests/script/tsim/tmq/basic3.sim +++ b/tests/script/tsim/tmq/basic3.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -87,7 +86,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -161,7 +159,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -235,7 +232,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic3Of2Cons.sim b/tests/script/tsim/tmq/basic3Of2Cons.sim index 4e47e3dbf9a0eeaf4380e43ceb6caef81f3c8770..75d762c44b6b572f2c9dfd5633b77eb5ff53ec39 100644 --- a/tests/script/tsim/tmq/basic3Of2Cons.sim +++ b/tests/script/tsim/tmq/basic3Of2Cons.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -86,7 +85,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -204,7 +202,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -309,7 +306,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic4.sim b/tests/script/tsim/tmq/basic4.sim index 6b35342ad119431d1eece232486069c9792015d9..c72d8ff412cc8b8687f4388ed471951eb7c09a47 100644 --- a/tests/script/tsim/tmq/basic4.sim +++ b/tests/script/tsim/tmq/basic4.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -119,7 +118,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -179,7 +177,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic4Of2Cons.sim b/tests/script/tsim/tmq/basic4Of2Cons.sim index 122a91af3627fb12d00be61a503942e32dca604d..bb006a354c0f0af476d0365de4cd28f0dea2a7de 100644 --- a/tests/script/tsim/tmq/basic4Of2Cons.sim +++ b/tests/script/tsim/tmq/basic4Of2Cons.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -160,7 +159,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -251,7 +249,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/snapshot.sim b/tests/script/tsim/tmq/snapshot.sim index 81fff3522499ff99ccfa9a92be5483ea42d775ba..fbdaba7d28d266526bd02aaa86a5de3c85f41338 100644 --- a/tests/script/tsim/tmq/snapshot.sim +++ b/tests/script/tsim/tmq/snapshot.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -87,7 +86,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -159,7 +157,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -233,7 +230,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/snapshot1.sim b/tests/script/tsim/tmq/snapshot1.sim index c79892ae1d943a071c58fe2236eb90ef9a5e10d4..5349981cc719e41d6d91a9a0a6e18555a7b8fd86 100644 --- a/tests/script/tsim/tmq/snapshot1.sim +++ b/tests/script/tsim/tmq/snapshot1.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -151,7 +150,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -231,7 +229,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/topic.sim b/tests/script/tsim/tmq/topic.sim index 0bf0873e9f0513b08b6868bed220c5e3d74072f9..8d4b506b2492b4abbb178f3f8b99f25c1e88e9d9 100644 --- a/tests/script/tsim/tmq/topic.sim +++ b/tests/script/tsim/tmq/topic.sim @@ -40,7 +40,6 @@ endi sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create super table sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) @@ -108,4 +107,15 @@ if $rows != 6 then return -1 endi +sql create topic topic_stable_1 as stable stb where t1 > 0 +sql create topic topic_stable_2 as stable stb where t1 > 0 and t1 < 0 +sql create topic topic_stable_3 as stable stb where 1 > 0 +sql create topic topic_stable_4 as stable stb where abs(t1) > 0 +sql_error create topic topic_stable_5 as stable stb where last(t1) > 0 +sql_error create topic topic_stable_5 as stable stb where sum(t1) > 0 +sql create topic topic_stable_6 as stable stb where tbname is not null +sql create topic topic_stable_7 as stable stb where tbname > 'a' +sql_error create topic topic_stable_8 as stable stb where tbname > 0 and xx < 0 +sql_error create topic topic_stable_9 as stable stb where tbname > 0 and c1 < 0 + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/user/privilege_create_db.sim b/tests/script/tsim/user/privilege_create_db.sim new file mode 100644 index 0000000000000000000000000000000000000000..c81bd1b2581793a6005d2278190c3e416471b816 --- /dev/null +++ b/tests/script/tsim/user/privilege_create_db.sim @@ -0,0 +1,97 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ========================root user create user +sql create user u1 pass "taosdata" +sql create user u2 pass "taosdata" +sql create database test +sql select * from information_schema.ins_user_privileges where user_name == "root" +if $rows != 1 then + return -1 +endi + +print =============connect with u1 +sql connect u1 +sql create database u1_d1 +sql use u1_d1 +sql create table u1_d1.t1(ts timestamp, c2 int) +sql use information_schema +sql select * from ins_user_privileges where user_name == "u1" order by privilege +if $rows != 2 then + return -1 +endi +if $data01 != read then + return -1 +endi +if $data11 != write then + return -1 +endi +if $data02 != u1_d1 then + return -1 +endi +if $data12 != u1_d1 then + return -1 +endi + +sql_error grant all on *.* to u1 +sql_error grant all on test.* to u1 + +print =============connect with u2 +sql connect u2 +sql create database u2_d1 +sql use u2_d1 +sql create table u2_d1.t1(ts timestamp, c2 int) +sql use information_schema +sql select * from ins_user_privileges where user_name == "u2" order by privilege +if $rows != 2 then + return -1 +endi +if $data01 != read then + return -1 +endi +if $data11 != write then + return -1 +endi +if $data02 != u2_d1 then + return -1 +endi +if $data12 != u2_d1 then + return -1 +endi + +sql_error select * from u1_d1.t1 +sql_error revoke read on u2_d1.* from u2 + +print =============connect with root, revoke read from u1, all from u2 +sql connect +sql revoke read on u1_d1.* from u1 +sql revoke all on u2_d1.* from u2 +sleep 1000 + +print =============connect with u1 +sql connect u1 +sql insert into u1_d1.t1 values(now, 1) +sql_error select * from u1_d1.t1; + +print =============connect with u2 +sql connect u2 +sql_error select * from u2_d1.t1; +sql_error insert into u2_d1.t1 values(now, 1) + +print =============connect with root, grant read to u1, all to u2 +sql connect +sql grant read on u1_d1.* to u1 +sql grant all on u2_d1.* to u2 + +sleep 1000 +print =============connect with u1 +sql connect u1 +sql select * from u1_d1.t1; +sql insert into u1_d1.t1 values(now, 2) + +print =============connect with u2 +sql connect u2 +sql select * from u2_d1.t1; +sql insert into u2_d1.t1 values(now, 2) diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index 5f82d2d93569b36e8cef68accda6fbdb49756520..debe633f06078c5432d4a7af1c45cceb5878d55b 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -20,7 +20,7 @@ sql_error alter user u2 sysinfo 0 print =============== step2 create drop dnode sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 -sql drop dnode 3 +sql drop dnode 3 force sql alter dnode 1 'debugflag 131' print =============== step3: select * from information_schema.ins_dnodes diff --git a/tests/script/win-test-file b/tests/script/win-test-file index b7fbbed5c16e2190c78985867d80eeb57efee00b..dc3093e0ea3c35eb2e3ec26a5ca650a12bc19087 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -1,11 +1,34 @@ +./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim +./test.sh -f tsim/parser/where.sim +./test.sh -f tsim/parser/join_manyblocks.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim +./test.sh -f tsim/parser/limit1.sim +./test.sh -f tsim/parser/union.sim +./test.sh -f tsim/parser/commit.sim +./test.sh -f tsim/parser/nestquery.sim +./test.sh -f tsim/valgrind/checkError7.sim +./test.sh -f tsim/parser/groupby.sim +./test.sh -f tsim/parser/sliding.sim +./test.sh -f tsim/dnode/balance2.sim +./test.sh -f tsim/vnode/replica3_repeat.sim +./test.sh -f tsim/parser/col_arithmetic_operation.sim +./test.sh -f tsim/trans/create_db.sim +./test.sh -f tsim/dnode/balance3.sim +./test.sh -f tsim/vnode/replica3_many.sim +./test.sh -f tsim/stable/metrics_idx.sim +./test.sh -f tsim/db/alter_replica_13.sim +./test.sh -f tsim/sync/3Replica1VgElect.sim +./test.sh -f tsim/sync/3Replica5VgElect.sim +./test.sh -f tsim/valgrind/checkError6.sim ./test.sh -f tsim/user/basic.sim ./test.sh -f tsim/user/password.sim ./test.sh -f tsim/user/privilege_db.sim -./test.sh -f tsim/user/privilege_sysinfo.sim ./test.sh -f tsim/user/privilege_topic.sim ./test.sh -f tsim/user/privilege_table.sim ./test.sh -f tsim/db/alter_option.sim -rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/db/alter_replica_31.sim ./test.sh -f tsim/db/basic1.sim ./test.sh -f tsim/db/basic2.sim @@ -29,11 +52,10 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/db/show_create_table.sim ./test.sh -f tsim/db/tables.sim ./test.sh -f tsim/db/taosdlog.sim +./test.sh -f tsim/db/table_prefix_suffix.sim ./test.sh -f tsim/dnode/balance_replica1.sim ./test.sh -f tsim/dnode/balance_replica3.sim ./test.sh -f tsim/dnode/balance1.sim -./test.sh -f tsim/dnode/balance2.sim -./test.sh -f tsim/dnode/balance3.sim ./test.sh -f tsim/dnode/balancex.sim ./test.sh -f tsim/dnode/create_dnode.sim ./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim @@ -45,10 +67,6 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/dnode/drop_dnode_force.sim ./test.sh -f tsim/dnode/offline_reason.sim ./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim ./test.sh -f tsim/dnode/vnode_clean.sim ./test.sh -f tsim/dnode/use_dropped_dnode.sim ./test.sh -f tsim/dnode/split_vgroup_replica1.sim @@ -74,6 +92,7 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/insert/query_multi_file.sim ./test.sh -f tsim/insert/tcp.sim ./test.sh -f tsim/insert/update0.sim +./test.sh -f tsim/insert/delete0.sim ./test.sh -f tsim/insert/update1_sort_merge.sim ./test.sh -f tsim/insert/update2.sim ./test.sh -f tsim/parser/alter__for_community_version.sim @@ -85,7 +104,6 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/parser/auto_create_tb.sim ./test.sh -f tsim/parser/between_and.sim ./test.sh -f tsim/parser/binary_escapeCharacter.sim -./test.sh -f tsim/parser/col_arithmetic_operation.sim ./test.sh -f tsim/parser/columnValue_bigint.sim ./test.sh -f tsim/parser/columnValue_bool.sim ./test.sh -f tsim/parser/columnValue_double.sim @@ -94,8 +112,8 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/parser/columnValue_smallint.sim ./test.sh -f tsim/parser/columnValue_tinyint.sim ./test.sh -f tsim/parser/columnValue_unsign.sim -./test.sh -f tsim/parser/commit.sim ./test.sh -f tsim/parser/condition.sim +./test.sh -f tsim/parser/condition_scl.sim ./test.sh -f tsim/parser/constCol.sim ./test.sh -f tsim/parser/create_db.sim ./test.sh -f tsim/parser/create_mt.sim @@ -111,7 +129,6 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/parser/fourArithmetic-basic.sim ./test.sh -f tsim/parser/function.sim ./test.sh -f tsim/parser/groupby-basic.sim -./test.sh -f tsim/parser/groupby.sim ./test.sh -f tsim/parser/having_child.sim ./test.sh -f tsim/parser/having.sim ./test.sh -f tsim/parser/import_commit1.sim @@ -121,7 +138,6 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/parser/import.sim ./test.sh -f tsim/parser/insert_multiTbl.sim ./test.sh -f tsim/parser/insert_tb.sim -./test.sh -f tsim/parser/join_manyblocks.sim ./test.sh -f tsim/parser/join_multitables.sim ./test.sh -f tsim/parser/join_multivnode.sim ./test.sh -f tsim/parser/join.sim @@ -131,10 +147,8 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/parser/lastrow2.sim ./test.sh -f tsim/parser/like.sim ./test.sh -f tsim/parser/limit.sim -./test.sh -f tsim/parser/limit1.sim ./test.sh -f tsim/parser/mixed_blocks.sim ./test.sh -f tsim/parser/nchar.sim -./test.sh -f tsim/parser/nestquery.sim ./test.sh -f tsim/parser/null_char.sim ./test.sh -f tsim/parser/precision_ns.sim ./test.sh -f tsim/parser/projection_limit_offset.sim @@ -147,7 +161,6 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/parser/selectResNum.sim ./test.sh -f tsim/parser/set_tag_vals.sim ./test.sh -f tsim/parser/single_row_in_tb.sim -./test.sh -f tsim/parser/sliding.sim ./test.sh -f tsim/parser/slimit_alter_tags.sim ./test.sh -f tsim/parser/slimit.sim ./test.sh -f tsim/parser/slimit1.sim @@ -158,9 +171,9 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/parser/timestamp.sim ./test.sh -f tsim/parser/top_groupby.sim ./test.sh -f tsim/parser/topbot.sim -./test.sh -f tsim/parser/union.sim ./test.sh -f tsim/parser/union_sysinfo.sim -./test.sh -f tsim/parser/where.sim +./test.sh -f tsim/parser/slimit_limit.sim +./test.sh -f tsim/parser/table_merge_limit.sim ./test.sh -f tsim/query/tagLikeFilter.sim ./test.sh -f tsim/query/charScalarFunction.sim ./test.sh -f tsim/query/explain.sim @@ -169,12 +182,21 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/query/scalarFunction.sim ./test.sh -f tsim/query/scalarNull.sim ./test.sh -f tsim/query/session.sim +./test.sh -f tsim/query/join_interval.sim +./test.sh -f tsim/query/unionall_as_table.sim +./test.sh -f tsim/query/multi_order_by.sim ./test.sh -f tsim/query/sys_tbname.sim ./test.sh -f tsim/query/groupby.sim +./test.sh -f tsim/query/groupby_distinct.sim ./test.sh -f tsim/query/event.sim ./test.sh -f tsim/query/forceFill.sim ./test.sh -f tsim/query/emptyTsRange.sim +./test.sh -f tsim/query/emptyTsRange_scl.sim ./test.sh -f tsim/query/partitionby.sim +./test.sh -f tsim/query/tableCount.sim +./test.sh -f tsim/query/tag_scan.sim +./test.sh -f tsim/query/nullColSma.sim +./test.sh -f tsim/query/bug3398.sim ./test.sh -f tsim/qnode/basic1.sim ./test.sh -f tsim/snode/basic1.sim ./test.sh -f tsim/mnode/basic1.sim @@ -257,7 +279,6 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/stream/udTableAndTag2.sim ./test.sh -f tsim/stream/windowClose.sim ./test.sh -f tsim/trans/lossdata1.sim -./test.sh -f tsim/trans/create_db.sim ./test.sh -f tsim/tmq/basic1.sim ./test.sh -f tsim/tmq/basic2.sim ./test.sh -f tsim/tmq/basic3.sim @@ -266,7 +287,6 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/tmq/basic2Of2Cons.sim ./test.sh -f tsim/tmq/basic3Of2Cons.sim ./test.sh -f tsim/tmq/basic4Of2Cons.sim -./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim ./test.sh -f tsim/tmq/topic.sim ./test.sh -f tsim/tmq/snapshot.sim ./test.sh -f tsim/tmq/snapshot1.sim @@ -290,7 +310,6 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/stable/tag_rename.sim ./test.sh -f tsim/stable/values.sim ./test.sh -f tsim/stable/vnode3.sim -./test.sh -f tsim/stable/metrics_idx.sim ./test.sh -f tsim/sma/drop_sma.sim ./test.sh -f tsim/sma/sma_leak.sim ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim @@ -301,13 +320,9 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/valgrind/checkError3.sim ./test.sh -f tsim/valgrind/checkError4.sim ./test.sh -f tsim/valgrind/checkError5.sim -./test.sh -f tsim/valgrind/checkError6.sim -./test.sh -f tsim/valgrind/checkError7.sim ./test.sh -f tsim/valgrind/checkError8.sim ./test.sh -f tsim/vnode/replica3_basic.sim -./test.sh -f tsim/vnode/replica3_repeat.sim ./test.sh -f tsim/vnode/replica3_vgroup.sim -./test.sh -f tsim/vnode/replica3_many.sim ./test.sh -f tsim/vnode/replica3_import.sim ./test.sh -f tsim/vnode/stable_balance_replica1.sim ./test.sh -f tsim/vnode/stable_dnode2_stop.sim @@ -315,8 +330,6 @@ rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/vnode/stable_dnode3.sim ./test.sh -f tsim/vnode/stable_replica3_dnode6.sim ./test.sh -f tsim/vnode/stable_replica3_vnode3.sim -./test.sh -f tsim/sync/3Replica1VgElect.sim -./test.sh -f tsim/sync/3Replica5VgElect.sim ./test.sh -f tsim/sync/oneReplica1VgElect.sim ./test.sh -f tsim/sync/oneReplica5VgElect.sim ./test.sh -f tsim/catalog/alterInCurrent.sim diff --git a/tests/system-test/0-others/backquote_check.py b/tests/system-test/0-others/backquote_check.py index be8590f913f110e0156d71fa81560fc99fa39132..7c91fd9e8cb9ef732b51efc32452df5601510555 100644 --- a/tests/system-test/0-others/backquote_check.py +++ b/tests/system-test/0-others/backquote_check.py @@ -22,7 +22,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) self.dbname = 'db' self.setsql = TDSetSql() self.stbname = 'stb' diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 22e319fdaf076190f3fc31390f5ca88d9e5561b1..cb804aad0cb17f2fd726f5cb93f754c3b1791da6 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -30,7 +30,15 @@ class TDTestCase: self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - + self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300;use deldata; + create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int); + create table deldata.ct1 using deldata.stb1 tags ( 1 ); + insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); + select avg(c1) from deldata.ct1; + delete from deldata.stb1; + flush database deldata; + insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); + delete from deldata.ct1;''' def checkProcessPid(self,processName): i=0 while i<60: @@ -138,9 +146,11 @@ class TDTestCase: tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{BASEVERSION}") tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") - os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ") - os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ') - os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show streams;" ') + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database test '") + + # os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ") + # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ') + # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show streams;" ') os.system(f"sed -i 's/\/etc\/taos/{cPath}/' 0-others/tmqBasic.json ") # os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/tmqBasic.json -y ") os.system('LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists tmq_test_topic as select current,voltage,phase from test.meters where voltage <= 106 and current <= 5;" ') @@ -151,7 +161,12 @@ class TDTestCase: os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") os.system("LD_LIBRARY_PATH=/usr/lib taos -f 0-others/TS-3131.tsql") + # add deleted data + os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "{self.deletedDataSql}" ') + + cmd = f" LD_LIBRARY_PATH={bPath}/build/lib {bPath}/build/bin/taos -h localhost ;" + tdLog.info(f"new client version connect to old version taosd, commad return value:{cmd}") if os.system(cmd) == 0: raise Exception("failed to execute system command. cmd: %s" % cmd) @@ -184,15 +199,22 @@ class TDTestCase: # tdsql.query("show streams;") # tdsql.query(f"select count(*) from {stb}") # tdsql.checkData(0,0,tableNumbers*recordNumbers2) - tdsql.query(f"select count(*) from db4096.stb0") + + # checkout db4096 + tdsql.query("select count(*) from db4096.stb0") tdsql.checkData(0,0,50000) + + # checkout deleted data + tdsql.execute("insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a );") + tdsql.execute("flush database deldata;") + tdsql.query("select avg(c1) from deldata.ct1;") + tdsql=tdCom.newTdSql() - tdLog.printNoPrefix(f"==========step4:verify backticks in taos Sql-TD18542") + tdLog.printNoPrefix("==========step4:verify backticks in taos Sql-TD18542") tdsql.execute("drop database if exists db") tdsql.execute("create database db") tdsql.execute("use db") - tdsql.execute("alter database db wal_retention_period 3600") tdsql.execute("create stable db.stb1 (ts timestamp, c1 int) tags (t1 int);") tdsql.execute("insert into db.ct1 using db.stb1 TAGS(1) values(now(),11);") tdsql.error(" insert into `db.ct2` using db.stb1 TAGS(9) values(now(),11);") @@ -203,7 +225,8 @@ class TDTestCase: tdsql.execute("insert into db.`ct4` using db.stb1 TAGS(4) values(now(),14);") tdsql.query("select * from db.ct4") tdsql.checkData(0,1,14) - print(1) + + #check retentions tdsql=tdCom.newTdSql() tdsql.query("describe information_schema.ins_databases;") qRows=tdsql.queryRows @@ -223,8 +246,12 @@ class TDTestCase: caller = inspect.getframeinfo(inspect.stack()[0][0]) args = (caller.filename, caller.lineno) tdLog.exit("%s(%d) failed" % args) + + # check stream tdsql.query("show streams;") - tdsql.checkRows(2) + tdsql.checkRows(0) + + #check TS-3131 tdsql.query("select *,tbname from d0.almlog where mcid='m0103';") tdsql.checkRows(6) expectList = [0,3003,20031,20032,20033,30031] @@ -239,6 +266,8 @@ class TDTestCase: tdsql.execute("insert into test.d80 values (now+1s, 11, 103, 0.21);") tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);") + + # check tmq conn = taos.connect() consumer = Consumer( @@ -266,6 +295,8 @@ class TDTestCase: print(block.fetchall()) tdsql.query("show topics;") tdsql.checkRows(1) + + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/0-others/deletedData.sql b/tests/system-test/0-others/deletedData.sql new file mode 100644 index 0000000000000000000000000000000000000000..781b9562cfebf498374092ed93cbde402b77ba9c --- /dev/null +++ b/tests/system-test/0-others/deletedData.sql @@ -0,0 +1,11 @@ +drop database if exists deldata; +create database deldata duration 300; +use deldata; +create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int); +create table deldata.ct1 using deldata.stb1 tags ( 1 ); +insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); +select avg(c1) from deldata.ct1; +delete from deldata.stb1; +flush database deldata; +insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); +delete from deldata.ct1; diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 3c4a71c3e4e47baf7c6503f77ea9b588eaebec83..762361f0511d0a19fcadeffca40de45687694f39 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -132,8 +132,61 @@ class TDTestCase: else: tdSql.checkEqual(result[i][0],f'stb_{i-1}') tdSql.checkEqual(result[i][1],ctbnum) + + def ins_stable_check2(self): + tdSql.execute('drop database if exists restful_test') + tdSql.execute('drop database if exists log') + tdSql.execute('drop database if exists d0') + tdSql.execute('drop database if exists d1') + tdSql.execute('create database restful_test vgroups 4 replica 1') + tdSql.execute('create database log vgroups 2 replica 1') + tdSql.execute('create database d0 vgroups 4 replica 1') + tdSql.execute('create database d1 vgroups 4 replica 1') + log_stb_num = 5 + rest_stb_num = 51 + for i in range(rest_stb_num): + tdSql.execute(f'create stable restful_test._stb_{i} (ts timestamp,c0 int) tags(t0 int);') + tdSql.execute(f'create stable d0._stb_{i} (ts timestamp,c0 int, c1 int) tags(t0 int,t1 int);') + tdSql.execute(f'create stable d1._stb_{i} (ts timestamp,c0 int, c1 int, c2 int) tags(t0 int,t1 int, t2 int);') + tdSql.execute(f'CREATE STABLE log.`taosadapter_restful_http_request_summary_milliseconds` (`_ts` TIMESTAMP, `sum` DOUBLE) TAGS (`request_uri` NCHAR(128));') + tdSql.execute(f'CREATE STABLE log.`taosadapter_system_cpu_percent` (`_ts` TIMESTAMP, `gauge` DOUBLE) TAGS (`endpoint` NCHAR(45));') + tdSql.execute(f'CREATE STABLE log.`taosadapter_restful_http_request_total` (`_ts` TIMESTAMP, `gauge` DOUBLE) TAGS (`client_ip` NCHAR(40));') + tdSql.execute(f'CREATE STABLE log.`taosadapter_system_mem_percent` (`_ts` TIMESTAMP, `gauge` DOUBLE) TAGS (`endpoint` NCHAR(45));') + tdSql.execute(f'CREATE STABLE log.`taosadapter_restful_http_request_fail` (`_ts` TIMESTAMP, `gauge` DOUBLE) TAGS (`request_uri` NCHAR(128), `status_code` NCHAR(4));') + + tdSql.query(f'select * from information_schema.ins_stables where db_name="restful_test" limit 0,25;') # condition 1 + result = tdSql.queryResult + tdSql.checkEqual(len(result),25) + for i in range(len(result)): + tdSql.checkEqual(result[i][0][0:5],f'_stb_') # stable_name + tdSql.checkEqual(result[i][1],f'restful_test') # db_name + tdSql.checkEqual(result[i][5]>=result[i][2],True) # last_update >= create_time + tdSql.checkEqual(result[i][3]>1,True) # columns + tdSql.checkEqual(result[i][4]>0,True) # tags + tdSql.checkEqual(result[i][6],None) # table_comment + tdSql.checkEqual(result[i][7],f'5000a,5000a') # watermark + tdSql.checkEqual(result[i][8],f'-1a,-1a') # max_delay + tdSql.checkEqual(result[i][9],f'') # rollup + tdSql.query(f'select create_time from information_schema.ins_stables where db_name="restful_test" order by create_time asc limit 10,1') + result = tdSql.queryResult + tdSql.checkEqual(len(result),1) + _create_time=result[0][0] + tdSql.query("select * from information_schema.ins_stables where db_name='restful_test' and create_time > '%s' limit 10,30" % (_create_time)) # condition 2 + result = tdSql.queryResult + tdSql.checkEqual(len(result),30) + for i in range(len(result)): + tdSql.checkEqual(result[i][0][0:5],f'_stb_') # stable_name + tdSql.checkEqual(result[i][1],f'restful_test') # db_name + tdSql.checkEqual(result[i][5]>=result[i][2],True) # last_update >= create_time + tdSql.checkEqual(result[i][2]>_create_time,True) # create_time + tdSql.checkEqual(result[i][3]>1,True) # columns + tdSql.checkEqual(result[i][4]>0,True) # tags + tdSql.checkEqual(result[i][6],None) # table_comment + tdSql.checkEqual(result[i][7],f'5000a,5000a') # watermark + tdSql.checkEqual(result[i][8],f'-1a,-1a') # max_delay + tdSql.checkEqual(result[i][9],f'') # rollup def ins_columns_check(self): tdSql.execute('drop database if exists db2') @@ -162,7 +215,53 @@ class TDTestCase: for t in range (2): tdSql.query(f'select * from information_schema.ins_columns where db_name="db2" and table_type=="NORMAL_TABLE"') tdSql.checkEqual(20470,len(tdSql.queryResult)) - + + def ins_dnodes_check(self): + tdSql.execute('drop database if exists db2') + tdSql.execute('create database if not exists db2 vgroups 1 replica 1') + tdSql.query(f'select * from information_schema.ins_dnodes') + result = tdSql.queryResult + tdSql.checkEqual(result[0][0],1) + tdSql.checkEqual(result[0][8],"") + tdSql.checkEqual(result[0][9],"") + self.str107 = 'Hc7VCc+' + for t in range (10): + self.str107 += 'tP+2soIXpP' + self.str108 = self.str107 + '=' + self.str109 = self.str108 + '+' + self.str254 = self.str108 + self.str108 + '01234567890123456789012345678901234567' + self.str255 = self.str254 + '=' + self.str256 = self.str254 + '=(' + self.str257 = self.str254 + '=()' + self.str510 = self.str255 + self.str255 + tdSql.error('alter dnode 1 "activeCode" "a"') + tdSql.error('alter dnode 1 "activeCode" "' + self.str107 + '"') + tdSql.execute('alter all dnodes "activeCode" "' + self.str108 + '"') + tdSql.error('alter dnode 1 "activeCode" "' + self.str109 + '"') + tdSql.error('alter all dnodes "activeCode" "' + self.str510 + '"') + tdSql.query(f'select * from information_schema.ins_dnodes') + tdSql.checkEqual(tdSql.queryResult[0][8],self.str108) + tdSql.execute('alter dnode 1 "activeCode" ""') + tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') + tdSql.checkEqual(tdSql.queryResult[0][0],"") + tdSql.checkEqual(tdSql.queryResult[0][1],'') + tdSql.error('alter dnode 1 "cActiveCode" "a"') + tdSql.error('alter dnode 1 "cActiveCode" "' + self.str107 + '"') + tdSql.error('alter dnode 1 "cActiveCode" "' + self.str256 + '"') + tdSql.error('alter all dnodes "cActiveCode" "' + self.str255 + '"') + tdSql.error('alter all dnodes "cActiveCode" "' + self.str256 + '"') + tdSql.error('alter all dnodes "cActiveCode" "' + self.str257 + '"') + tdSql.execute('alter all dnodes "cActiveCode" "' + self.str254 + '"') + tdSql.error('alter dnode 1 "cActiveCode" "' + self.str510 + '"') + tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') + tdSql.checkEqual(tdSql.queryResult[0][0],"") + tdSql.checkEqual(tdSql.queryResult[0][1],self.str254) + tdSql.execute('alter dnode 1 "cActiveCode" "' + self.str109 + '"') + tdSql.query(f'show dnodes') + tdSql.checkEqual(tdSql.queryResult[0][9],self.str109) + tdSql.execute('alter all dnodes "cActiveCode" ""') + tdSql.query(f'select c_active_code from information_schema.ins_dnodes') + tdSql.checkEqual(tdSql.queryResult[0][0],'') def run(self): self.prepare_data() @@ -170,6 +269,8 @@ class TDTestCase: self.ins_columns_check() # self.ins_col_check_4096() self.ins_stable_check() + self.ins_stable_check2() + self.ins_dnodes_check() def stop(self): @@ -177,4 +278,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/multilevel.py b/tests/system-test/0-others/multilevel.py index 7ad4eba6453877149e2be6a01f76d6cf50b906bc..f086dcb735e73037866e2a382322c9c4046b00bb 100644 --- a/tests/system-test/0-others/multilevel.py +++ b/tests/system-test/0-others/multilevel.py @@ -116,7 +116,7 @@ class TDTestCase: tdSql.checkRows(1000) tdLog.info("================= step3") tdSql.execute('drop database test') - for i in range(50): + for i in range(10): tdSql.execute("create database test%d duration 1" %(i)) tdSql.execute("use test%d" %(i)) tdSql.execute("create table tb (ts timestamp,i int)") diff --git a/tests/system-test/0-others/performance_schema.py b/tests/system-test/0-others/performance_schema.py index 9d2a362254714a2daefc98787a3c1ec236c3e80c..b4a16794bb0f36b907c21ff4dd657b9b920321a1 100755 --- a/tests/system-test/0-others/performance_schema.py +++ b/tests/system-test/0-others/performance_schema.py @@ -75,7 +75,6 @@ class TDTestCase: def prepare_data(self): tdSql.execute(f"create database if not exists {self.dbname} vgroups 2") #1 query tdSql.execute(f'use {self.dbname}') #1 query - tdsql.execute(f"alter database {self.dbname} wal_retention_period 3600") tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) #1 query for i in range(self.tbnum): #self.tbnum query diff --git a/tests/system-test/0-others/show.py b/tests/system-test/0-others/show.py index b284605a0e8dc619c5ab004d4a65d022b0eb8946..9d26b3a2aeb679195246e0989605b48f12b91a77 100644 --- a/tests/system-test/0-others/show.py +++ b/tests/system-test/0-others/show.py @@ -45,8 +45,6 @@ class TDTestCase: "replica":1, "wal_level":1, "wal_fsync_period":6000, - "wal_roll_period":0, - "wal_segment_size":1024, "vgroups":self.vgroups, "stt_trigger":1, "tsdb_pagesize":16 @@ -83,12 +81,37 @@ class TDTestCase: tag_sql += f"{k} {v}, " create_stb_sql = f'create stable {stbname} ({column_sql[:-2]}) tags ({tag_sql[:-2]})' return create_stb_sql - + def set_create_database_sql(self,sql_dict): create_sql = 'create' for key,value in sql_dict.items(): create_sql += f' {key} {value}' return create_sql + + def show_create_sysdb_sql(self): + sysdb_list = {'information_schema', 'performance_schema'} + for db in sysdb_list: + tdSql.query(f'show create database {db}') + tdSql.checkEqual(f'{db}',tdSql.queryResult[0][0]) + tdSql.checkEqual(f'CREATE DATABASE `{db}`',tdSql.queryResult[0][1]) + + def show_create_systb_sql(self): + for param in self.ins_param_list: + tdSql.query(f'show create table information_schema.ins_{param}') + tdSql.checkEqual(f'ins_{param}',tdSql.queryResult[0][0]) + + tdSql.execute(f'use information_schema') + tdSql.query(f'show create table ins_{param}') + tdSql.checkEqual(f'ins_{param}',tdSql.queryResult[0][0]) + + for param in self.perf_param_list: + tdSql.query(f'show create table performance_schema.perf_{param}') + tdSql.checkEqual(f'perf_{param}',tdSql.queryResult[0][0]) + + tdSql.execute(f'use performance_schema') + tdSql.query(f'show create table perf_{param}') + tdSql.checkEqual(f'perf_{param}',tdSql.queryResult[0][0]) + def show_create_sql(self): create_db_sql = self.set_create_database_sql(self.db_param) print(create_db_sql) @@ -108,7 +131,7 @@ class TDTestCase: tdSql.query('show vnodes 1') tdSql.checkRows(self.vgroups) tdSql.execute(f'use {self.dbname}') - + column_dict = { '`ts`': 'timestamp', '`col1`': 'tinyint', @@ -124,7 +147,7 @@ class TDTestCase: '`col11`': 'bool', '`col12`': 'varchar(20)', '`col13`': 'nchar(20)' - + } tag_dict = { '`t1`': 'tinyint', @@ -141,7 +164,7 @@ class TDTestCase: '`t12`': 'varchar(20)', '`t13`': 'nchar(20)', '`t14`': 'timestamp' - + } create_table_sql = self.set_stb_sql(self.stbname,column_dict,tag_dict) tdSql.execute(create_table_sql) @@ -152,7 +175,7 @@ class TDTestCase: tag_sql = '(' for tag_keys in tag_dict.keys(): tag_sql += f'{tag_keys}, ' - tags = f'{tag_sql[:-2]})' + tags = f'{tag_sql[:-2]})' sql = f'create table {self.tbname} using {self.stbname} {tags} tags (1, 1, 1, 1, 1, 1, 1, 1, 1.000000e+00, 1.000000e+00, true, "abc", "abc123", 0)' tdSql.query(f'show create table {self.tbname}') query_result = tdSql.queryResult @@ -175,7 +198,7 @@ class TDTestCase: taosd_info = os.popen('taosd -V').read() taosd_gitinfo = re.findall("^gitinfo.*",taosd_info,re.M) tdSql.checkEqual(taosd_gitinfo_sql,taosd_gitinfo[0]) - + def show_base(self): for sql in ['dnodes','mnodes','cluster']: tdSql.query(f'show {sql}') @@ -193,6 +216,8 @@ class TDTestCase: self.ins_check() self.perf_check() self.show_create_sql() + self.show_create_sysdb_sql() + self.show_create_systb_sql() def stop(self): tdSql.close() diff --git a/tests/system-test/0-others/show_tag_index.py b/tests/system-test/0-others/show_tag_index.py new file mode 100644 index 0000000000000000000000000000000000000000..663426b7ffdb6fcee2394b9913984fd0a980cb31 --- /dev/null +++ b/tests/system-test/0-others/show_tag_index.py @@ -0,0 +1,184 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import re +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def check_tags(self): + tdSql.checkRows(2) + tdSql.checkCols(6) + tdSql.checkData(0, 0, 'ctb1') + tdSql.checkData(0, 1, 'db') + tdSql.checkData(0, 2, 'stb') + tdSql.checkData(0, 3, 't0') + tdSql.checkData(0, 4, 'INT') + tdSql.checkData(0, 5, 1) + tdSql.checkData(1, 0, 'ctb1') + tdSql.checkData(1, 1, 'db') + tdSql.checkData(1, 2, 'stb') + tdSql.checkData(1, 3, 't1') + tdSql.checkData(1, 4, 'INT') + tdSql.checkData(1, 5, 1) + + def check_table_tags(self, is_super_table): + + if is_super_table == False: + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 'ctb1') + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + else: + tdSql.checkRows(2) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 'ctb1') + tdSql.checkData(1, 0, 'ctb2') + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 2) + + def check_indexes(self): + tdSql.checkRows(1) + tdSql.checkCols(7) + tdSql.checkData(0, 0, 'idx1') + tdSql.checkData(0, 1, 'db') + tdSql.checkData(0, 2, 'stb') + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 5, 't1') + tdSql.checkData(0, 6, 'tag_index') + + def run(self): + tdSql.execute(f'create database db') + tdSql.execute(f'use db') + tdSql.execute(f'create table stb (ts timestamp, c0 int) tags (t0 int, t1 int)') + tdSql.execute(f'create table ctb1 using stb tags (1, 1)') + tdSql.execute(f'create table ctb2 using stb tags (2, 2)') + tdSql.execute(f'insert into ctb1 values (now, 1)') + tdSql.execute(f'insert into ctb2 values (now, 2)') + + # show tags + tdSql.query(f'show tags from stb') + tdSql.checkRows(0) + tdSql.query(f'show tags from stb') + tdSql.checkRows(0); + tdSql.query(f'show tags from `stb`') + tdSql.checkRows(0); + tdSql.query(f'show tags from stb from db') + tdSql.checkRows(0); + tdSql.query(f'show tags from `stb` from `db`') + tdSql.checkRows(0); + tdSql.query(f'show tags from db.stb') + tdSql.checkRows(0); + tdSql.query(f'show tags from `db`.`stb`') + tdSql.checkRows(0); + tdSql.query(f'show tags from ctb1') + self.check_tags(); + tdSql.query(f'show tags from `ctb1`') + self.check_tags(); + tdSql.query(f'show tags from ctb1 from db') + self.check_tags(); + tdSql.query(f'show tags from `ctb1` from `db`') + self.check_tags(); + tdSql.query(f'show tags from db.ctb1') + self.check_tags(); + tdSql.query(f'show tags from `db`.`ctb1`') + self.check_tags(); + + tdSql.error(f'show tags from db.stb from db') + tdSql.error(f'show tags from `db`.`stb` from db') + tdSql.error(f'show tags from db.ctb1 from db') + tdSql.error(f'show tags from `db`.`ctb1` from db') + + # show table tags + tdSql.query(f'show table tags from stb') + self.check_table_tags(True); + tdSql.query(f'show table tags from `stb`') + self.check_table_tags(True); + tdSql.query(f'show table tags from stb from db') + self.check_table_tags(True); + tdSql.query(f'show table tags from `stb` from `db`') + self.check_table_tags(True); + tdSql.query(f'show table tags from db.stb') + self.check_table_tags(True); + tdSql.query(f'show table tags from `db`.`stb`') + self.check_table_tags(True); + + tdSql.query(f'show table tags from ctb1') + self.check_table_tags(False); + tdSql.query(f'show table tags from `ctb1`') + self.check_table_tags(False); + tdSql.query(f'show table tags from ctb1 from db') + self.check_table_tags(False); + tdSql.query(f'show table tags from `ctb1` from `db`') + self.check_table_tags(False); + tdSql.query(f'show table tags from db.ctb1') + self.check_table_tags(False); + tdSql.query(f'show table tags from `db`.`ctb1`') + self.check_table_tags(False); + + tdSql.error(f'show table tags from db.stb from db') + tdSql.error(f'show table tags from `db`.`stb` from db') + tdSql.error(f'show table tags from db.ctb1 from db') + tdSql.error(f'show table tags from `db`.`ctb1` from db') + + # show indexes + tdSql.execute(f'create index idx1 on stb (t1)') + + tdSql.query(f'show indexes from stb') + self.check_indexes(); + tdSql.query(f'show indexes from `stb`') + self.check_indexes(); + tdSql.query(f'show indexes from stb from db') + self.check_indexes(); + tdSql.query(f'show indexes from `stb` from `db`') + self.check_indexes(); + tdSql.query(f'show indexes from db.stb') + self.check_indexes(); + tdSql.query(f'show indexes from `db`.`stb`') + self.check_indexes(); + + tdSql.query(f'show indexes from ctb1') + tdSql.checkRows(0) + tdSql.query(f'show indexes from `ctb1`') + tdSql.checkRows(0) + tdSql.query(f'show indexes from ctb1 from db') + tdSql.checkRows(0) + tdSql.query(f'show indexes from `ctb1` from `db`') + tdSql.checkRows(0) + tdSql.query(f'show indexes from db.ctb1') + tdSql.checkRows(0) + tdSql.query(f'show indexes from `db`.`ctb1`') + tdSql.checkRows(0) + + tdSql.error(f'show indexes from db.stb from db') + tdSql.error(f'show indexes from `db`.`stb` from db') + tdSql.error(f'show indexes from db.ctb1 from db') + tdSql.error(f'show indexes from `db`.`ctb1` from db') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/sma_index.py b/tests/system-test/0-others/sma_index.py new file mode 100644 index 0000000000000000000000000000000000000000..488342b6039a7beac13d7ecf9c5401cc8a69a145 --- /dev/null +++ b/tests/system-test/0-others/sma_index.py @@ -0,0 +1,48 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def create_databases(self): + tdSql.execute("create database db_ms precision 'ms'") + tdSql.execute("create database db_us precision 'us'") + tdSql.execute("create database db_ns precision 'ns'") + + def create_stables(self): + tdSql.execute("CREATE STABLE db_ms.`meters` (`ts` TIMESTAMP, `c0` INT, `c1` TINYINT, `c2` DOUBLE, `c3` VARCHAR(64), `c4` NCHAR(64)) TAGS (`cc` VARCHAR(16))") + tdSql.execute("CREATE STABLE db_us.`meters` (`ts` TIMESTAMP, `c0` INT, `c1` TINYINT, `c2` DOUBLE, `c3` VARCHAR(64), `c4` NCHAR(64)) TAGS (`cc` VARCHAR(16))") + tdSql.execute("CREATE STABLE db_ns.`meters` (`ts` TIMESTAMP, `c0` INT, `c1` TINYINT, `c2` DOUBLE, `c3` VARCHAR(64), `c4` NCHAR(64)) TAGS (`cc` VARCHAR(16))") + + def create_sma_index(self): + tdSql.execute("create sma index sma_index_ms on db_ms.meters function(max(c1), max(c2), min(c1)) interval(6m, 10s) sliding(6m)" ) + tdSql.execute("create sma index sma_index_us on db_us.meters function(max(c1), max(c2), min(c1)) interval(6m, 10s) sliding(6m)" ) + tdSql.execute("create sma index sma_index_ns on db_ns.meters function(max(c1), max(c2), min(c1)) interval(6m, 10s) sliding(6m)" ) + + def run(self): + tdSql.prepare() + self.create_databases() + self.create_stables() + self.create_sma_index() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/splitVGroup.py b/tests/system-test/0-others/splitVGroup.py new file mode 100644 index 0000000000000000000000000000000000000000..9fd00892e45ade46da43e4fb4a38d532109e1fb4 --- /dev/null +++ b/tests/system-test/0-others/splitVGroup.py @@ -0,0 +1,428 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time +import copy +import string + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # random string + def random_string(self, count): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(count)) + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null," + else: + value += f"{c2}," + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + # c3 same with ts + value += f"{self.ts})" + + # move next + self.ts += 1 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into @db_name.t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + self.exeDouble(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + self.exeDouble(sql) + values = "" + + # insert nomal talbe + for i in range(20): + self.ts += 1000 + name = self.random_string(20) + sql = f"insert into @db_name.ta values({self.ts}, {i}, {self.ts%100000}, '{name}', false)" + self.exeDouble(sql) + + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + def exeDouble(self, sql): + # dbname replace + sql1 = sql.replace("@db_name", self.db1) + + if len(sql1) > 100: + tdLog.info(sql1[:100]) + else: + tdLog.info(sql1) + tdSql.execute(sql1) + + sql2 = sql.replace("@db_name", self.db2) + if len(sql1) > 100: + tdLog.info(sql1[:100]) + else: + tdLog.info(sql1) + tdSql.execute(sql2) + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1680000000000 + self.childCnt = 10 + self.childRow = 10000 + self.batchSize = 5000 + self.vgroups1 = 4 + self.vgroups2 = 4 + self.db1 = "db1" + self.db2 = "db2" + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # create database db + sql = f"create database @db_name vgroups {self.vgroups1} replica 3" + self.exeDouble(sql) + + # create super talbe st + sql = f"create table @db_name.st(ts timestamp, c1 int, c2 bigint, ts1 timestamp) tags(area int)" + self.exeDouble(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table @db_name.t{i} using @db_name.st tags({i}) " + self.exeDouble(sql) + + # create normal table + sql = f"create table @db_name.ta(ts timestamp, c1 int, c2 bigint, c3 binary(32), c4 bool)" + self.exeDouble(sql) + + # insert data + self.insertData() + + # check data correct + def checkExpect(self, sql, expectVal): + tdSql.query(sql) + rowCnt = tdSql.getRows() + for i in range(rowCnt): + val = tdSql.getData(i,0) + if val != expectVal: + tdLog.exit(f"Not expect . query={val} expect={expectVal} i={i} sql={sql}") + return False + + tdLog.info(f"check expect ok. sql={sql} expect ={expectVal} rowCnt={rowCnt}") + return True + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.clock_gettime(time.CLOCK_REALTIME) + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # check query result same + def queryDouble(self, sql): + # sql + sql1 = sql.replace('@db_name', self.db1) + tdLog.info(sql1) + start1 = time.time() + rows1 = tdSql.query(sql1) + spend1 = time.time() - start1 + res1 = copy.copy(tdSql.queryResult) + + sql2 = sql.replace('@db_name', self.db2) + tdLog.info(sql2) + start2 = time.time() + tdSql.query(sql2) + spend2 = time.time() - start2 + res2 = tdSql.queryResult + + rowlen1 = len(res1) + rowlen2 = len(res2) + + if rowlen1 != rowlen2: + tdLog.exit(f"rowlen1={rowlen1} rowlen2={rowlen2} both not equal.") + return False + + for i in range(rowlen1): + row1 = res1[i] + row2 = res2[i] + collen1 = len(row1) + collen2 = len(row2) + if collen1 != collen2: + tdLog.exit(f"collen1={collen1} collen2={collen2} both not equal.") + return False + for j in range(collen1): + if row1[j] != row2[j]: + tdLog.exit(f"col={j} col1={row1[j]} col2={row2[j]} both col not equal.") + return False + + # warning performance + diff = (spend2 - spend1)*100/spend1 + tdLog.info("spend1=%.6fs spend2=%.6fs diff=%.1f%%"%(spend1, spend2, diff)) + if spend2 > spend1 and diff > 20: + tdLog.info("warning: the diff for performance after spliting is over 20%") + + return True + + + # check result + def checkResult(self): + # check vgroupid + sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{self.db2}'" + tdSql.query(sql) + tdSql.checkRows(self.vgroups2) + + # check child table count same + sql = "select table_name from information_schema.ins_tables where db_name='@db_name' order by table_name" + self.queryDouble(sql) + + # check row value is ok + sql = "select * from @db_name.st order by ts" + self.queryDouble(sql) + + # where + sql = "select *,tbname from @db_name.st where c1 < 1000 order by ts" + self.queryDouble(sql) + + # max + sql = "select max(c1) from @db_name.st" + self.queryDouble(sql) + + # min + sql = "select min(c2) from @db_name.st" + self.queryDouble(sql) + + # sum + sql = "select sum(c1) from @db_name.st" + self.queryDouble(sql) + + # normal table + + # count + sql = "select count(*) from @db_name.ta" + self.queryDouble(sql) + + # all rows + sql = "select * from @db_name.ta" + self.queryDouble(sql) + + # sum + sql = "select sum(c1) from @db_name.ta" + self.queryDouble(sql) + + + # get vgroup list + def getVGroup(self, db_name): + vgidList = [] + sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{db_name}'" + res = tdSql.getResult(sql) + rows = len(res) + for i in range(rows): + vgidList.append(res[i][0]) + + return vgidList; + + # split vgroup on db2 + def splitVGroup(self, db_name): + vgids = self.getVGroup(db_name) + selid = random.choice(vgids) + sql = f"split vgroup {selid}" + tdLog.info(sql) + tdSql.execute(sql) + + # wait end + seconds = 300 + for i in range(seconds): + sql ="show transactions;" + rows = tdSql.query(sql) + if rows == 0: + tdLog.info("split vgroup finished.") + return True + #tdLog.info(f"i={i} wait split vgroup ...") + time.sleep(1) + + tdLog.exit(f"split vgroup transaction is not finished after executing {seconds}s") + return False + + # split error + def expectSplitError(self, dbName): + vgids = self.getVGroup(dbName) + selid = random.choice(vgids) + sql = f"split vgroup {selid}" + tdLog.info(sql) + tdSql.error(sql) + + # expect split ok + def expectSplitOk(self, dbName): + # split vgroup + vgList1 = self.getVGroup(dbName) + self.splitVGroup(dbName) + vgList2 = self.getVGroup(dbName) + vgNum1 = len(vgList1) + 1 + vgNum2 = len(vgList2) + if vgNum1 != vgNum2: + tdLog.exit(f" vglist len={vgNum1} is not same for expect {vgNum2}") + return + + # split empty database + def splitEmptyDB(self): + dbName = "emptydb" + vgNum = 2 + # create database + sql = f"create database {dbName} vgroups {vgNum}" + tdLog.info(sql) + tdSql.execute(sql) + + # split vgroup + self.expectSplitOk(dbName) + + + # forbid + def checkForbid(self): + # stream + tdLog.info("check forbid split having stream...") + tdSql.execute("create database streamdb;") + tdSql.execute("use streamdb;") + tdSql.execute("create table ta(ts timestamp, age int);") + tdSql.execute("create stream ma into sta as select count(*) from ta interval(1s);") + self.expectSplitError("streamdb") + tdSql.execute("drop stream ma;") + self.expectSplitOk("streamdb") + + # topic + tdLog.info("check forbid split having topic...") + tdSql.execute("create database topicdb wal_retention_period 10;") + tdSql.execute("use topicdb;") + tdSql.execute("create table ta(ts timestamp, age int);") + tdSql.execute("create topic toa as select * from ta;") + self.expectSplitError("topicdb") + tdSql.execute("drop topic toa;") + self.expectSplitOk("topicdb") + + # compact and check db2 + def compactAndCheck(self): + tdLog.info("compact db2 and check result ...") + # compact + tdSql.execute(f"compact database {self.db2};") + # check result + self.checkResult() + + # run + def run(self): + # prepare env + self.prepareEnv() + + for i in range(5): + # split vgroup on db2 + start = time.time() + self.splitVGroup(self.db2) + end = time.time() + self.vgroups2 += 1 + + # check two db query result same + self.checkResult() + spend = "%.3f"%(end-start) + tdLog.info(f"split vgroup i={i} passed. spend = {spend}s") + + # split empty db + self.splitEmptyDB() + + # check topic and stream forib + self.checkForbid() + + # compact database + self.compactAndCheck() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index 8094c4e0f555dcd7a5a7340c439a0a34948b212d..a07d7f411e2528033711972addf300fbcd909de4 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -185,28 +185,14 @@ class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): if "total" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["total"] <= 0: tdLog.exit("total is null!") - # log_infos ==================================== if "log_infos" not in infoDict or infoDict["log_infos"]== None: tdLog.exit("log_infos is null!") - if "logs" not in infoDict["log_infos"] or len(infoDict["log_infos"]["logs"]) < 8:#!= 10: - tdLog.exit("logs is null!") - - if "ts" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 10: - tdLog.exit("ts is null!") - - if "level" not in infoDict["log_infos"]["logs"][0] or infoDict["log_infos"]["logs"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]: - tdLog.exit("level is null!") - - if "content" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 1: - tdLog.exit("content is null!") - if "summary" not in infoDict["log_infos"] or len(infoDict["log_infos"]["summary"])!= 4: tdLog.exit("summary is null!") - if "total" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["total"] < 0 : tdLog.exit("total is null!") @@ -315,4 +301,3 @@ class TDTestCase: tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) - diff --git a/tests/system-test/0-others/timeRangeWise.py b/tests/system-test/0-others/timeRangeWise.py new file mode 100644 index 0000000000000000000000000000000000000000..a7dc18aa820caec9166e2f636c11c5b81e4e201c --- /dev/null +++ b/tests/system-test/0-others/timeRangeWise.py @@ -0,0 +1,309 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time +import copy +import string + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # random string + def random_string(self, count): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(count)) + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null," + else: + value += f"{c2}," + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + # c3 same with ts + value += f"{self.ts})" + + # move next 1s interval + self.ts += 1 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into @db_name.t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + self.exeDouble(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + self.exeDouble(sql) + values = "" + + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + def exeDouble(self, sql): + # dbname replace + sql1 = sql.replace("@db_name", self.db1) + + if len(sql1) > 100: + tdLog.info(sql1[:100]) + else: + tdLog.info(sql1) + tdSql.execute(sql1) + + sql2 = sql.replace("@db_name", self.db2) + if len(sql2) > 100: + tdLog.info(sql2[:100]) + else: + tdLog.info(sql2) + tdSql.execute(sql2) + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1680000000000 + self.childCnt = 2 + self.childRow = 100000 + self.batchSize = 5000 + self.vgroups1 = 4 + self.vgroups2 = 4 + self.db1 = "db1" # no sma + self.db2 = "db2" # have sma + self.smaClause = "interval(10s)" + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # alter local optimization to treu + sql = "alter local 'querysmaoptimize 1'" + tdSql.execute(sql, 5, True) + + # check forbid mulit-replic on create sma index + sql = f"create database db vgroups {self.vgroups1} replica 3" + tdSql.execute(sql, 5, True) + sql = f"create table db.st(ts timestamp, c1 int, c2 bigint, ts1 timestamp) tags(area int)" + tdSql.execute(sql, 5, True) + + sql = f"create sma index sma_test on db.st function(max(c1),max(c2),min(c1),min(c2)) {self.smaClause};" + tdLog.info(sql) + tdSql.error(sql) + + + # create database db + sql = f"create database @db_name vgroups {self.vgroups1} replica 1" + self.exeDouble(sql) + + # create super talbe st + sql = f"create table @db_name.st(ts timestamp, c1 int, c2 bigint, ts1 timestamp) tags(area int)" + self.exeDouble(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table @db_name.t{i} using @db_name.st tags({i}) " + self.exeDouble(sql) + + # create sma index on db2 + sql = f"use {self.db2}" + tdSql.execute(sql) + sql = f"create sma index sma_index_maxmin on {self.db2}.st function(max(c1),max(c2),min(c1),min(c2)) {self.smaClause};" + tdLog.info(sql) + tdSql.execute(sql) + + # insert data + self.insertData() + + # check data correct + def checkExpect(self, sql, expectVal): + tdSql.query(sql) + rowCnt = tdSql.getRows() + for i in range(rowCnt): + val = tdSql.getData(i,0) + if val != expectVal: + tdLog.exit(f"Not expect . query={val} expect={expectVal} i={i} sql={sql}") + return False + + tdLog.info(f"check expect ok. sql={sql} expect ={expectVal} rowCnt={rowCnt}") + return True + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.clock_gettime(time.CLOCK_REALTIME) + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # check query result same + def queryDoubleImpl(self, sql): + # sql + sql1 = sql.replace('@db_name', self.db1) + tdLog.info(sql1) + start1 = time.time() + rows1 = tdSql.query(sql1) + spend1 = time.time() - start1 + res1 = copy.copy(tdSql.queryResult) + + sql2 = sql.replace('@db_name', self.db2) + tdLog.info(sql2) + start2 = time.time() + tdSql.query(sql2) + spend2 = time.time() - start2 + res2 = tdSql.queryResult + + rowlen1 = len(res1) + rowlen2 = len(res2) + + if rowlen1 != rowlen2: + tdLog.info(f"check error. rowlen1={rowlen1} rowlen2={rowlen2} both not equal.") + return False + + for i in range(rowlen1): + row1 = res1[i] + row2 = res2[i] + collen1 = len(row1) + collen2 = len(row2) + if collen1 != collen2: + tdLog.info(f"checkerror. collen1={collen1} collen2={collen2} both not equal.") + return False + for j in range(collen1): + if row1[j] != row2[j]: + tdLog.exit(f"col={j} col1={row1[j]} col2={row2[j]} both col not equal.") + return False + + # warning performance + multiple = spend1/spend2 + tdLog.info("spend1=%.6fs spend2=%.6fs multiple=%.1f"%(spend1, spend2, multiple)) + if spend2 > spend1 and multiple < 4: + tdLog.info(f"performace not reached: multiple(spend1/spend)={multiple} require is >=4 ") + return False + + return True + + # check query result same + def queryDouble(self, sql, tryCount=60, gap=1): + for i in range(tryCount): + if self.queryDoubleImpl(sql): + return True + # error + tdLog.info(f"queryDouble return false, try loop={i}") + time.sleep(gap) + + tdLog.exit(f"queryDouble try {tryCount} times, but all failed.") + return False + + # check result + def checkResult(self): + + # max + sql = f"select max(c1) from @db_name.st {self.smaClause}" + self.queryDouble(sql) + + # min + sql = f"select max(c2) from @db_name.st {self.smaClause}" + self.queryDouble(sql) + + # mix + sql = f"select max(c1),max(c2),min(c1),min(c2) from @db_name.st {self.smaClause}" + self.queryDouble(sql) + + + # run + def run(self): + # prepare env + self.prepareEnv() + + # check two db query result same + tdLog.info(f"check have sma(db1) and no sma(db2) performace...") + self.checkResult() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index 78020cb9586e6c59f7c0f84e5aeacdbb596b421c..88d0d420f73f9c06e633ebcf2dbc14a454f6e878 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -234,6 +234,11 @@ class TDTestCase: tdSql.checkData(20,6,88) tdSql.checkData(20,7,1) + tdSql.query("select udf1(1) from (select 1)") + tdSql.checkData(0,0,1) + + tdSql.query("select udf1(n) from (select 1 n)") + tdSql.checkData(0,0,1) # aggregate functions tdSql.query("select udf2(num1) ,udf2(num2), udf2(num3) from tb") diff --git a/tests/system-test/0-others/user_manage.py b/tests/system-test/0-others/user_manage.py index 1e33d4bb1c628bd43569b68d8f7fc07e68f2bec8..6f90a2873afa4e08c1ec4b72457b6cba9f232d48 100644 --- a/tests/system-test/0-others/user_manage.py +++ b/tests/system-test/0-others/user_manage.py @@ -115,7 +115,6 @@ class TDTestCase: jiacy0_read_conn = taos.connect(user='jiacy0_read', password='123') jiacy0_write_conn = taos.connect(user='jiacy0_write', password='123') jiacy0_none_conn = taos.connect(user='jiacy0_none', password='123') - tdSql.execute('alter database db wal_retention_period 3600') tdSql.execute('create topic root_db as select * from db.stb') for user in [jiacy1_all_conn, jiacy1_read_conn, jiacy0_all_conn, jiacy0_read_conn]: user.execute(f'create topic db_jiacy as select * from db.stb') diff --git a/tests/system-test/0-others/user_privilege_all.py b/tests/system-test/0-others/user_privilege_all.py new file mode 100644 index 0000000000000000000000000000000000000000..846b76317e8af9bdc7660816c58fcec5a9ac18c3 --- /dev/null +++ b/tests/system-test/0-others/user_privilege_all.py @@ -0,0 +1,469 @@ +from itertools import product +import taos +import time +from taos.tmq import * +from util.cases import * +from util.common import * +from util.log import * +from util.sql import * +from util.sqlset import * + + +class TDTestCase: + """This test case is used to veirfy the user privilege for insert and select operation on + stable、child table and table + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + # init the tdsql + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + # user info + self.username = 'test' + self.password = 'test' + # db info + self.dbname = "user_privilege_all_db" + self.stbname = 'stb' + self.common_tbname = "tb" + self.ctbname_list = ["ct1", "ct2"] + self.common_table_dict = { + 'ts':'timestamp', + 'col1':'float', + 'col2':'int' + } + self.stable_column_dict = { + 'ts': 'timestamp', + 'col1': 'float', + 'col2': 'int', + } + self.tag_dict = { + 'ctbname': 'binary(10)' + } + + # case list + self.cases = { + "test_db_table_both_no_permission": { + "db_privilege": "none", + "stable_priviege": "none", + "child_table_ct1_privilege": "none", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "none", + "sql": ["insert into ct1 using stb tags('ct1') values(now, 1.1, 1)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 3.3, 3);", + "select * from tb;"], + "res": [False, False, False, False, False, False] + }, + "test_db_no_permission_table_read": { + "db_privilege": "none", + "stable_priviege": "none", + "child_table_ct1_privilege": "none", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "read", + "sql": ["insert into ct1 using stb tags('ct1') values(now, 1.1, 1)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 3.3, 3);", + "select * from tb;"], + "res": [False, False, False, False, False, True] + }, + "test_db_no_permission_childtable_read": { + "db_privilege": "none", + "stable_priviege": "none", + "child_table_ct1_privilege": "read", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "none", + "sql": ["insert into ct1 using stb tags('ct1') values(now, 1.1, 1)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 3.3, 3);", + "select * from tb;"], + "res": [False, True, True, False, False, False] + }, + "test_db_no_permission_table_write": { + "db_privilege": "none", + "stable_priviege": "none", + "child_table_ct1_privilege": "none", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "write", + "sql": ["insert into ct1 using stb tags('ct1') values(now, 1.1, 1)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 3.3, 3);", + "select * from tb;"], + "res": [False, False, False, False, True, False] + }, + "test_db_no_permission_childtable_write": { + "db_privilege": "none", + "stable_priviege": "none", + "child_table_ct1_privilege": "none", + "child_table_ct2_privilege": "write", + "table_tb_privilege": "none", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 1.1, 1)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 3.3, 3);", + "select * from tb;"], + "res": [True, False, False, False, False, False] + }, + "test_db_read_table_no_permission": { + "db_privilege": "read", + "stable_priviege": "none", + "child_table_ct1_privilege": "none", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "none", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 1.1, 1)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 3.3, 3);", + "select * from tb;"], + "res": [False, True, True, True, False, True] + }, + "test_db_read_table_read": { + "db_privilege": "read", + "stable_priviege": "none", + "child_table_ct1_privilege": "none", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "read", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 1.1, 1)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 3.3, 3);", + "select * from tb;"], + "res": [False, True, True, True, False, True] + }, + "test_db_read_childtable_read": { + "db_privilege": "read", + "stable_priviege": "none", + "child_table_ct1_privilege": "read", + "child_table_ct2_privilege": "read", + "table_tb_privilege": "none", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 1.1, 1)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 3.3, 3);", + "select * from tb;"], + "res": [False, True, True, True, False, True] + }, + "test_db_read_table_write": { + "db_privilege": "read", + "stable_priviege": "none", + "child_table_ct1_privilege": "none", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "write", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 1.1, 1)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 4.4, 4);", + "select * from tb;"], + "res": [False, True, True, True, True, True] + }, + "test_db_read_childtable_write": { + "db_privilege": "read", + "stable_priviege": "none", + "child_table_ct1_privilege": "write", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "none", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 1.1, 1)", + "insert into ct1 using stb tags('ct1') values(now, 5.5, 5)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 4.4, 4);", + "select * from tb;"], + "res": [False, True, True, True, True, False, True] + }, + "test_db_write_table_no_permission": { + "db_privilege": "write", + "stable_priviege": "none", + "child_table_ct1_privilege": "none", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "none", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 6.6, 6)", + "insert into ct1 using stb tags('ct1') values(now, 7.7, 7)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 8.8, 8);", + "select * from tb;"], + "res": [True, True, False, False, False, True, False] + }, + "test_db_write_table_write": { + "db_privilege": "write", + "stable_priviege": "none", + "child_table_ct1_privilege": "none", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "none", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 9.9, 9)", + "insert into ct1 using stb tags('ct1') values(now, 10.0, 10)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 11.1, 11);", + "select * from tb;"], + "res": [True, True, False, False, False, True, False] + }, + "test_db_write_childtable_write": { + "db_privilege": "write", + "stable_priviege": "none", + "child_table_ct1_privilege": "none", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "none", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 12.2, 12)", + "insert into ct1 using stb tags('ct1') values(now, 13.3, 13)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 14.4, 14);", + "select * from tb;"], + "res": [True, True, False, False, False, True, False] + }, + "test_db_write_table_read": { + "db_privilege": "write", + "stable_priviege": "none", + "child_table_ct1_privilege": "none", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "read", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 15.5, 15)", + "insert into ct1 using stb tags('ct1') values(now, 16.6, 16)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 17.7, 17);", + "select * from tb;"], + "res": [True, True, False, False, False, True, True] + }, + "test_db_write_childtable_read": { + "db_privilege": "write", + "stable_priviege": "none", + "child_table_ct1_privilege": "read", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "none", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 18.8, 18)", + "insert into ct1 using stb tags('ct1') values(now, 19.9, 19)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 20.0, 20);", + "select * from tb;"], + "res": [True, True, True, True, False, True, False] + }, + "test_db_all_childtable_none": { + "db_privilege": "all", + "stable_priviege": "none", + "child_table_ct1_privilege": "none", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "none", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 20.2, 20)", + "insert into ct1 using stb tags('ct1') values(now, 21.21, 21)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 22.22, 22);", + "select * from tb;"], + "res": [True, True, True, True, True, True, True] + }, + "test_db_none_stable_all_childtable_none": { + "db_privilege": "none", + "stable_priviege": "all", + "child_table_ct1_privilege": "none", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "none", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 23.23, 23)", + "insert into ct1 using stb tags('ct1') values(now, 24.24, 24)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 25.25, 25);", + "select * from tb;"], + "res": [True, True, True, True, True, False, False] + }, + "test_db_no_permission_childtable_all": { + "db_privilege": "none", + "stable_priviege": "none", + "child_table_ct1_privilege": "all", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "none", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 26.26, 26)", + "insert into ct1 using stb tags('ct1') values(now, 27.27, 27)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 28.28, 28);", + "select * from tb;"], + "res": [False, True, True, True, False, False, False] + }, + "test_db_none_stable_none_table_all": { + "db_privilege": "none", + "stable_priviege": "none", + "child_table_ct1_privilege": "none", + "child_table_ct2_privilege": "none", + "table_tb_privilege": "all", + "sql": ["insert into ct2 using stb tags('ct2') values(now, 26.26, 26)", + "insert into ct1 using stb tags('ct1') values(now, 27.27, 27)", + "select * from stb;", + "select * from ct1;", + "select * from ct2;", + "insert into tb values(now, 29.29, 29);", + "select * from tb;"], + "res": [False, False, False, False, False, True, True] + } + } + + def prepare_data(self): + """Create the db and data for test + """ + tdLog.debug("Start to prepare the data for test") + # create datebase + tdSql.execute(f"create database {self.dbname}") + tdSql.execute(f"use {self.dbname}") + + # create stable + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname, self.stable_column_dict, self.tag_dict)) + tdLog.debug("Create stable {} successfully".format(self.stbname)) + + # insert data into child table + for ctname in self.ctbname_list: + tdSql.execute(f"insert into {ctname} using {self.stbname} tags('{ctname}') values(now, 1.1, 1)") + tdSql.execute(f"insert into {ctname} using {self.stbname} tags('{ctname}') values(now, 2.1, 2)") + + # create common table + tdSql.execute(self.setsql.set_create_normaltable_sql(self.common_tbname, self.common_table_dict)) + tdLog.debug("Create common table {} successfully".format(self.common_tbname)) + + # insert data into common table + tdSql.execute(f"insert into {self.common_tbname} values(now, 1.1, 1)") + tdSql.execute(f"insert into {self.common_tbname} values(now, 2.2, 2)") + tdLog.debug("Finish to prepare the data") + + def create_user(self): + """Create the user for test + """ + tdSql.execute(f'create user {self.username} pass "{self.password}"') + tdLog.debug("sql:" + f'create user {self.username} pass "{self.password}" successfully') + + def grant_privilege(self, username, privilege, table, tag_condition=None): + """Add the privilege for the user + """ + try: + if tag_condition: + tdSql.execute(f'grant {privilege} on {self.dbname}.{table} with {tag_condition} to {username}') + else: + tdSql.execute(f'grant {privilege} on {self.dbname}.{table} to {username}') + time.sleep(2) + tdLog.debug("Grant {} privilege on {}.{} with condition {} to {} successfully".format(privilege, self.dbname, table, tag_condition, username)) + except Exception as ex: + tdLog.exit(ex) + + def remove_privilege(self, username, privilege, table, tag_condition=None): + """Remove the privilege for the user + """ + try: + if tag_condition: + tdSql.execute(f'revoke {privilege} on {self.dbname}.{table} with {tag_condition} from {username}') + else: + tdSql.execute(f'revoke {privilege} on {self.dbname}.{table} from {username}') + tdLog.debug("Revoke {} privilege on {}.{} with condition {} from {} successfully".format(privilege, self.dbname, table, tag_condition, username)) + except Exception as ex: + tdLog.exit(ex) + + def run(self): + self.create_user() + # prepare the test data + self.prepare_data() + + for case_name in self.cases.keys(): + tdLog.debug("Execute the case {} with params {}".format(case_name, str(self.cases[case_name]))) + # grant privilege for user test if case need + if self.cases[case_name]["db_privilege"] != "none": + self.grant_privilege(self.username, self.cases[case_name]["db_privilege"], "*") + if self.cases[case_name]["stable_priviege"] != "none": + self.grant_privilege(self.username, self.cases[case_name]["stable_priviege"], self.stbname) + if self.cases[case_name]["child_table_ct1_privilege"] != "none" and self.cases[case_name]["child_table_ct2_privilege"] != "none": + self.grant_privilege(self.username, self.cases[case_name]["child_table_ct1_privilege"], self.stbname, "ctbname='ct1' or ctbname='ct2'") + elif self.cases[case_name]["child_table_ct1_privilege"] != "none": + self.grant_privilege(self.username, self.cases[case_name]["child_table_ct1_privilege"], self.stbname, "ctbname='ct1'") + elif self.cases[case_name]["child_table_ct2_privilege"] != "none": + self.grant_privilege(self.username, self.cases[case_name]["child_table_ct2_privilege"], self.stbname, "ctbname='ct2'") + if self.cases[case_name]["table_tb_privilege"] != "none": + self.grant_privilege(self.username, self.cases[case_name]["table_tb_privilege"], self.common_tbname) + # connect db with user test + testconn = taos.connect(user=self.username, password=self.password) + if case_name != "test_db_table_both_no_permission": + testconn.execute("use %s;" % self.dbname) + # check privilege of user test from ins_user_privileges table + res = testconn.query("select * from information_schema.ins_user_privileges;") + tdLog.debug("Current information_schema.ins_user_privileges values: {}".format(res.fetch_all())) + # check privilege of user test by executing sql query + for index in range(len(self.cases[case_name]["sql"])): + tdLog.debug("Execute sql: {}".format(self.cases[case_name]["sql"][index])) + try: + # for write privilege + if "insert " in self.cases[case_name]["sql"][index]: + testconn.execute(self.cases[case_name]["sql"][index]) + # check the expected result + if self.cases[case_name]["res"][index]: + tdLog.debug("Write data with sql {} successfully".format(self.cases[case_name]["sql"][index])) + # for read privilege + elif "select " in self.cases[case_name]["sql"][index]: + res = testconn.query(self.cases[case_name]["sql"][index]) + data = res.fetch_all() + tdLog.debug("query result: {}".format(data)) + # check query results by cases + if case_name in ["test_db_no_permission_childtable_read", "test_db_write_childtable_read", "test_db_no_permission_childtable_all"] and self.cases[case_name]["sql"][index] == "select * from ct2;": + if not self.cases[case_name]["res"][index]: + if 0 == len(data): + tdLog.debug("Query with sql {} successfully as expected with empty result".format(self.cases[case_name]["sql"][index])) + continue + else: + tdLog.exit("Query with sql {} failed with result {}".format(self.cases[case_name]["sql"][index], data)) + # check the expected result + if self.cases[case_name]["res"][index]: + if len(data) > 0: + tdLog.debug("Query with sql {} successfully".format(self.cases[case_name]["sql"][index])) + else: + tdLog.exit("Query with sql {} failed with result {}".format(self.cases[case_name]["sql"][index], data)) + else: + tdLog.exit("Execute query sql {} successfully, but expected failed".format(self.cases[case_name]["sql"][index])) + except BaseException as ex: + # check the expect false result + if not self.cases[case_name]["res"][index]: + tdLog.debug("Execute sql {} failed with {} as expected".format(self.cases[case_name]["sql"][index], str(ex))) + continue + # unexpected exception + else: + tdLog.exit(ex) + # remove the privilege + if self.cases[case_name]["db_privilege"] != "none": + self.remove_privilege(self.username, self.cases[case_name]["db_privilege"], "*") + if self.cases[case_name]["stable_priviege"] != "none": + self.remove_privilege(self.username, self.cases[case_name]["stable_priviege"], self.stbname) + if self.cases[case_name]["child_table_ct1_privilege"] != "none": + self.remove_privilege(self.username, self.cases[case_name]["child_table_ct1_privilege"], self.stbname, "ctbname='ct1'") + if self.cases[case_name]["child_table_ct2_privilege"] != "none": + self.remove_privilege(self.username, self.cases[case_name]["child_table_ct2_privilege"], self.stbname, "ctbname='ct2'") + if self.cases[case_name]["table_tb_privilege"] != "none": + self.remove_privilege(self.username, self.cases[case_name]["table_tb_privilege"], self.common_tbname) + # close the connection of user test + testconn.close() + + def stop(self): + # remove the user + tdSql.execute(f'drop user {self.username}') + # close the connection + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/user_privilege_multi_users.py b/tests/system-test/0-others/user_privilege_multi_users.py new file mode 100644 index 0000000000000000000000000000000000000000..8812f42e7bdd537e5f718adb7438c854fcbc598c --- /dev/null +++ b/tests/system-test/0-others/user_privilege_multi_users.py @@ -0,0 +1,126 @@ +from itertools import product +import taos +import random +from taos.tmq import * +from util.cases import * +from util.common import * +from util.log import * +from util.sql import * +from util.sqlset import * + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + # init the tdsql + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + # user info + self.userNum = 100 + self.basic_username = "user" + self.password = "pwd" + + # db info + self.dbname = "user_privilege_multi_users" + self.stbname = 'stb' + self.ctbname_num = 100 + self.column_dict = { + 'ts': 'timestamp', + 'col1': 'float', + 'col2': 'int', + } + self.tag_dict = { + 'ctbname': 'binary(10)' + } + + self.privilege_list = [] + + def prepare_data(self): + """Create the db and data for test + """ + # create datebase + tdSql.execute(f"create database {self.dbname}") + tdLog.debug("sql:" + f"create database {self.dbname}") + tdSql.execute(f"use {self.dbname}") + tdLog.debug("sql:" + f"use {self.dbname}") + + # create super table + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname, self.column_dict, self.tag_dict)) + tdLog.debug("Create stable {} successfully".format(self.stbname)) + for ctbIndex in range(self.ctbname_num): + ctname = f"ctb{ctbIndex}" + tdSql.execute(f"create table {ctname} using {self.stbname} tags('{ctname}')") + tdLog.debug("sql:" + f"create table {ctname} using {self.stbname} tags('{ctname}')") + + def create_multiusers(self): + """Create the user for test + """ + for userIndex in range(self.userNum): + username = f"{self.basic_username}{userIndex}" + tdSql.execute(f'create user {username} pass "{self.password}"') + tdLog.debug("sql:" + f'create user {username} pass "{self.password}"') + + def grant_privilege(self): + """Add the privilege for the users + """ + try: + for userIndex in range(self.userNum): + username = f"{self.basic_username}{userIndex}" + privilege = random.choice(["read", "write", "all"]) + condition = f"ctbname='ctb{userIndex}'" + self.privilege_list.append({ + "username": username, + "privilege": privilege, + "condition": condition + }) + tdSql.execute(f'grant {privilege} on {self.dbname}.{self.stbname} with {condition} to {username}') + tdLog.debug("sql:" + f'grant {privilege} on {self.dbname}.{self.stbname} with {condition} to {username}') + except Exception as ex: + tdLog.exit(ex) + + def remove_privilege(self): + """Remove the privilege for the users + """ + try: + for item in self.privilege_list: + username = item["username"] + privilege = item["privilege"] + condition = item["condition"] + tdSql.execute(f'revoke {privilege} on {self.dbname}.{self.stbname} with {condition} from {username}') + tdLog.debug("sql:" + f'revoke {privilege} on {self.dbname}.{self.stbname} with {condition} from {username}') + except Exception as ex: + tdLog.exit(ex) + + def run(self): + """ + Check the information from information_schema.ins_user_privileges + """ + self.create_multiusers() + self.prepare_data() + # grant privilege to users + self.grant_privilege() + # check information_schema.ins_user_privileges + tdSql.query("select * from information_schema.ins_user_privileges;") + tdLog.debug("Current information_schema.ins_user_privileges values: {}".format(tdSql.queryResult)) + if len(tdSql.queryResult) >= self.userNum: + tdLog.debug("case passed") + else: + tdLog.exit("The privilege number in information_schema.ins_user_privileges is incorrect") + + def stop(self): + # remove the privilege + self.remove_privilege() + # clear env + tdSql.execute(f"drop database {self.dbname}") + # remove the users + for userIndex in range(self.userNum): + username = f"{self.basic_username}{userIndex}" + tdSql.execute(f'drop user {username}') + # close the connection + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/user_privilege_show.py b/tests/system-test/0-others/user_privilege_show.py new file mode 100644 index 0000000000000000000000000000000000000000..9f49778ba87d039c81ec99761c161e1a7e965fa2 --- /dev/null +++ b/tests/system-test/0-others/user_privilege_show.py @@ -0,0 +1,267 @@ +from itertools import product +import taos +from taos.tmq import * +from util.cases import * +from util.common import * +from util.log import * +from util.sql import * +from util.sqlset import * + + +class TDTestCase: + """This test case is used to veirfy the show create stable/table command for + the different user privilege(TS-3469) + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + # init the tdsql + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + # user info + self.username = 'test' + self.password = 'test' + # db info + self.dbname = "user_privilege_show" + self.stbname = 'stb' + self.common_tbname = "tb" + self.ctbname_list = ["ct1", "ct2"] + self.column_dict = { + 'ts': 'timestamp', + 'col1': 'float', + 'col2': 'int', + } + self.tag_dict = { + 'ctbname': 'binary(10)' + } + + # privilege check scenario info + self.privilege_check_dic = {} + self.senario_type = ["stable", "table", "ctable"] + self.priv_type = ["read", "write", "all", "none"] + # stable senarios + # include the show stable xxx command test senarios and expect result, true as have privilege, false as no privilege + # the list element is (db_privilege, stable_privilege, expect_res) + st_senarios_list = [] + for senario in list(product(self.priv_type, repeat=2)): + expect_res = True + if senario == ("write", "write") or senario == ("none", "none") or senario == ("none", "write") or senario == ("write", "none"): + expect_res = False + st_senarios_list.append(senario + (expect_res,)) + # self.privilege_check_dic["stable"] = st_senarios_list + + # table senarios + # the list element is (db_privilege, table_privilege, expect_res) + self.privilege_check_dic["table"] = st_senarios_list + + # child table senarios + # the list element is (db_privilege, stable_privilege, ctable_privilege, expect_res) + ct_senarios_list = [] + for senario in list(product(self.priv_type, repeat=3)): + expect_res = True + if senario[2] == "write" or (senario[2] == "none" and senario[1] == "write") or (senario[2] == "none" and senario[1] == "none" and senario[0] == "write"): + expect_res = False + ct_senarios_list.append(senario + (expect_res,)) + self.privilege_check_dic["ctable"] = ct_senarios_list + + def prepare_data(self, senario_type): + """Create the db and data for test + """ + if senario_type == "stable": + # db name + self.dbname = self.dbname + '_stable' + elif senario_type == "table": + # db name + self.dbname = self.dbname + '_table' + else: + # db name + self.dbname = self.dbname + '_ctable' + + # create datebase + tdSql.execute(f"create database {self.dbname}") + tdLog.debug("sql:" + f"create database {self.dbname}") + tdSql.execute(f"use {self.dbname}") + tdLog.debug("sql:" + f"use {self.dbname}") + + # create tables + if "_stable" in self.dbname: + # create stable + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname, self.column_dict, self.tag_dict)) + tdLog.debug("Create stable {} successfully".format(self.stbname)) + elif "_table" in self.dbname: + # create common table + tdSql.execute(f"create table {self.common_tbname}(ts timestamp, col1 float, col2 int)") + tdLog.debug("sql:" + f"create table {self.common_tbname}(ts timestamp, col1 float, col2 int)") + else: + # create stable and child table + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname, self.column_dict, self.tag_dict)) + tdLog.debug("Create stable {} successfully".format(self.stbname)) + for ctname in self.ctbname_list: + tdSql.execute(f"create table {ctname} using {self.stbname} tags('{ctname}')") + tdLog.debug("sql:" + f"create table {ctname} using {self.stbname} tags('{ctname}')") + + def create_user(self): + """Create the user for test + """ + tdSql.execute(f'create user {self.username} pass "{self.password}"') + tdLog.debug("sql:" + f'create user {self.username} pass "{self.password}"') + + def grant_privilege(self, username, privilege, privilege_obj, ctable_include=False, tag_condition=None): + """Add the privilege for the user + """ + try: + if ctable_include and tag_condition: + tdSql.execute(f'grant {privilege} on {self.dbname}.{privilege_obj} with {tag_condition} to {username}') + tdLog.debug("sql:" + f'grant {privilege} on {self.dbname}.{privilege_obj} with {tag_condition} to {username}') + else: + tdSql.execute(f'grant {privilege} on {self.dbname}.{privilege_obj} to {username}') + tdLog.debug("sql:" + f'grant {privilege} on {self.dbname}.{privilege_obj} to {username}') + except Exception as ex: + tdLog.exit(ex) + + def remove_privilege(self, username, privilege, privilege_obj, ctable_include=False, tag_condition=None): + """Remove the privilege for the user + """ + try: + if ctable_include and tag_condition: + tdSql.execute(f'revoke {privilege} on {self.dbname}.{privilege_obj} with {tag_condition} from {username}') + tdLog.debug("sql:" + f'revoke {privilege} on {self.dbname}.{privilege_obj} with {tag_condition} from {username}') + else: + tdSql.execute(f'revoke {privilege} on {self.dbname}.{privilege_obj} from {username}') + tdLog.debug("sql:" + f'revoke {privilege} on {self.dbname}.{privilege_obj} from {username}') + except Exception as ex: + tdLog.exit(ex) + + def run(self): + """Currently, the test case can't be executed for all of the privilege combinations cause + the table privilege isn't finished by dev team, only left one senario: + db read privilege for user and show create table command; will udpate the test case once + the table privilege function is finished + """ + self.create_user() + + # temp solution only for the db read privilege verification + self.prepare_data("table") + # grant db read privilege + self.grant_privilege(self.username, "read", "*") + # create the taos connection with -utest -ptest + testconn = taos.connect(user=self.username, password=self.password) + testconn.execute("use %s;" % self.dbname) + # show the user privileges + res = testconn.query("select * from information_schema.ins_user_privileges;") + tdLog.debug("Current information_schema.ins_user_privileges values: {}".format(res.fetch_all())) + # query execution + sql = "show create table " + self.common_tbname + ";" + tdLog.debug("sql: %s" % sql) + res = testconn.query(sql) + # query result + tdLog.debug("sql res:" + str(res.fetch_all())) + # remove the privilege + self.remove_privilege(self.username, "read", "*") + # clear env + testconn.close() + tdSql.execute(f"drop database {self.dbname}") + + """ + for senario_type in self.privilege_check_dic.keys(): + tdLog.debug(f"---------check the {senario_type} privilege----------") + self.prepare_data(senario_type) + for senario in self.privilege_check_dic[senario_type]: + # grant db privilege + if senario[0] != "none": + self.grant_privilege(self.username, senario[0], "*") + # grant stable privilege + if senario[1] != "none": + self.grant_privilege(self.username, senario[1], self.stbname if senario_type == "stable" or senario_type == "ctable" else self.common_tbname) + if senario_type == "stable" or senario_type == "table": + tdLog.debug(f"check the db privilege: {senario[0]}, (s)table privilege: {senario[1]}") + else: + if senario[2] != "none": + # grant child table privilege + self.grant_privilege(self.username, senario[2], self.stbname, True, "ctbname='ct1'") + tdLog.debug(f"check the db privilege: {senario[0]}, (s)table privilege: {senario[1]}, ctable privilege: {senario[2]}") + testconn = taos.connect(user=self.username, password=self.password) + tdLog.debug("Create taos connection with user: {}, password: {}".format(self.username, self.password)) + try: + testconn.execute("use %s;" % self.dbname) + except BaseException as ex: + if (senario_type in ["stable", "table"] and senario[0] == "none" and senario[1] == "none") or (senario_type == "ctable" and senario[0] == "none" and senario[1] == "none" and senario[2] == "none"): + continue + else: + tdLog.exit(ex) + + # query privileges for user + res = testconn.query("select * from information_schema.ins_user_privileges;") + tdLog.debug("Current information_schema.ins_user_privileges values: {}".format(res.fetch_all())) + + if senario_type == "stable" or senario_type == "table": + sql = "show create " + (("stable " + self.stbname) if senario_type == "stable" else (f"table {self.dbname}." + self.common_tbname + ";")) + if senario[2]: + tdLog.debug("sql: %s" % sql) + tdLog.debug(f"expected result: {senario[2]}") + res = testconn.query(sql) + tdLog.debug("sql res:" + res.fetch_all()) + else: + exception_flag = False + try: + tdLog.debug("sql: %s" % sql) + tdLog.debug(f"expected result: {senario[2]}") + res = testconn.query(sql) + tdLog.debug("sql res:" + res.fetch_all()) + except BaseException: + exception_flag = True + caller = inspect.getframeinfo(inspect.stack()[1][0]) + tdLog.debug(f"{caller.filename}({caller.lineno}) failed to check the db privilege {senario[0]} and stable privilege {senario[1]} failed as expected") + if not exception_flag: + pass + # tdLog.exit("The expected exception isn't occurred") + else: + sql = f"show create table {self.dbname}.{self.ctbname_list[0]};" + if senario[3]: + tdLog.debug("sql: %s" % sql) + tdLog.debug(f"expected result: {senario[3]}") + res = testconn.query(sql) + tdLog.debug(res.fetch_all()) + else: + exception_flag = False + try: + tdLog.debug("sql: %s" % sql) + tdLog.debug(f"expected result: {senario[3]}") + res = testconn.query(sql) + tdLog.debug(res.fetch_all()) + except BaseException: + exception_flag = True + caller = inspect.getframeinfo(inspect.stack()[1][0]) + tdLog.debug(f"{caller.filename}({caller.lineno}) failed to check the db privilege {senario[0]}, stable privilege {senario[1]} and ctable privilege {senario[2]} failed as expected") + if not exception_flag: + pass + # tdLog.exit("The expected exception isn't occurred") + + # remove db privilege + if senario[0] != "none": + self.remove_privilege(self.username, senario[0], "*") + # remove stable privilege + if senario[1] != "none": + self.remove_privilege(self.username, senario[1], self.stbname if senario_type == "stable" else self.common_tbname) + # remove child table privilege + if senario_type == "ctable": + if senario[2] != "none": + self.remove_privilege(self.username, senario[2], self.ctbname_list[0], True, "ctbname='ct1'") + testconn.close() + + # remove the database + tdSql.execute(f"drop database {self.dbname}") + # reset the dbname + self.dbname = "user_privilege_show" + """ + + def stop(self): + # remove the user + tdSql.execute(f'drop user {self.username}') + # close the connection + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/boundary.py b/tests/system-test/1-insert/boundary.py index 29dcbc7c461b01d61c784cbaa30c2a724e156e2a..4476236ca65666942e67ecca3c2aaf0abaa48038 100644 --- a/tests/system-test/1-insert/boundary.py +++ b/tests/system-test/1-insert/boundary.py @@ -33,7 +33,7 @@ class TDTestCase: self.colname_length_boundary = self.boundary.COL_KEY_MAX_LENGTH self.tagname_length_boundary = self.boundary.TAG_KEY_MAX_LENGTH self.username_length_boundary = 23 - self.password_length_boundary = 128 + self.password_length_boundary = 31 def dbname_length_check(self): dbname_length = randint(1,self.dbname_length_boundary-1) for dbname in [tdCom.get_long_name(self.dbname_length_boundary),tdCom.get_long_name(dbname_length)]: diff --git a/tests/system-test/1-insert/db_tb_name_check.py b/tests/system-test/1-insert/db_tb_name_check.py index 23bb53962038a416590a28b99bdd2763bf52a954..fa43603e258f67b8cead63f30130424fdbc60461 100644 --- a/tests/system-test/1-insert/db_tb_name_check.py +++ b/tests/system-test/1-insert/db_tb_name_check.py @@ -44,7 +44,7 @@ class TDTestCase: new_dbname = list(dbname) new_dbname.insert(i,j) dbname_1 = ''.join(new_dbname) - tdSql.execute(f'create database if not exists `{dbname_1}`') + tdSql.execute(f'create database if not exists `{dbname_1}` vgroups 1 replica 1') tdSql.query('select * from information_schema.ins_databases') tdSql.checkEqual(tdSql.queryResult[2][0],str(dbname_1)) tdSql.execute(f'drop database `{dbname_1}`') @@ -56,7 +56,7 @@ class TDTestCase: def tb_name_check(self): dbname = tdCom.getLongName(10) - tdSql.execute(f'create database if not exists `{dbname}`') + tdSql.execute(f'create database if not exists `{dbname}` vgroups 1 replica 1') tdSql.execute(f'use `{dbname}`') tbname = tdCom.getLongName(5) for i in self.special_name: diff --git a/tests/system-test/1-insert/delete_stable.py b/tests/system-test/1-insert/delete_stable.py index 8ebe7b6692e31bba12bdc0a3cbc885112eb96562..67561c51e506056d34eff5a0a7f1ee3eb7367fdb 100644 --- a/tests/system-test/1-insert/delete_stable.py +++ b/tests/system-test/1-insert/delete_stable.py @@ -24,10 +24,11 @@ from util.common import * from util.sqlset import TDSetSql class TDTestCase: + updatecfgDict = {'tsdbdebugFlag': 143} def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) self.dbname = 'db_test' self.ns_dbname = 'ns_test' self.us_dbname = 'us_test' diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py index 9954b3557e23df193a4c4f84c819fc508c39f5bb..8775450ff0eee5ce1bd82d6eb83dc2f85ed44ba4 100644 --- a/tests/system-test/1-insert/drop.py +++ b/tests/system-test/1-insert/drop.py @@ -129,6 +129,12 @@ class TDTestCase: tdSql.query(f'select * from information_schema.ins_topics where topic_name = "{topic_name}"') tdSql.checkEqual(tdSql.queryResult[0][3],f'create topic {topic_name} as select c0 from {self.dbname}.{stbname}') tdSql.execute(f'drop topic {topic_name}') + + #TD-25222 + long_topic_name="hhhhjjhhhhqwertyuiasdfghjklzxcvbnmhhhhjjhhhhqwertyuiasdfghjklzxcvbnmhhhhjjhhhhqwertyuiasdfghjklzxcvbnm" + tdSql.execute(f'create topic {long_topic_name} as select * from {self.dbname}.{stbname}') + tdSql.execute(f'drop topic {long_topic_name}') + tdSql.execute(f'drop database {self.dbname}') def drop_stream_check(self): diff --git a/tests/system-test/1-insert/manyVgroups.json b/tests/system-test/1-insert/manyVgroups.json index 3b0fa96b08f73e26e11c35c89d6673268f764ddc..8c6f39cf9665633ac501dc032293fe2d1e4d7aec 100644 --- a/tests/system-test/1-insert/manyVgroups.json +++ b/tests/system-test/1-insert/manyVgroups.json @@ -11,7 +11,7 @@ "confirm_parameter_prompt": "no", "insert_interval": 0, "interlace_rows": 0, - "num_of_records_per_req": 100000, + "num_of_records_per_req": 10000, "databases": [ { "dbinfo": { @@ -73,4 +73,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py index 857a8e3a32cfab505629a3b8e41397a37b4b73bd..91ef1e70a3ac1fe7350eaf216fe33bdfa119025b 100644 --- a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py @@ -24,6 +24,8 @@ import threading import json class TDTestCase: + updatecfgDict = {'clientCfg': {'smlDot2Underline': 0}} + def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index 351cf49e3a217a44e93bbaf9c8c69ce2fa76c190..d419aee12cd34c20b6a3d1e5256b1d704df5ad2b 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -28,6 +28,8 @@ if platform.system().lower() == 'windows': sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') class TDTestCase: + updatecfgDict = {'clientCfg': {'smlDot2Underline': 0}} + def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) diff --git a/tests/system-test/1-insert/precisionNS.py b/tests/system-test/1-insert/precisionNS.py new file mode 100644 index 0000000000000000000000000000000000000000..be8f1e21dcbebb71befca96e1652ba58d09e3913 --- /dev/null +++ b/tests/system-test/1-insert/precisionNS.py @@ -0,0 +1,293 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null," + else: + value += f"{c2}," + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + # c3 same with ts + value += f"{self.ts})" + + # move next + self.ts += 1 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + + sql = "flush database db;" + tdLog.info(sql) + tdSql.execute(sql) + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1680000000000*1000*1000 + self.childCnt = 5 + self.childRow = 10000 + self.batchSize = 5000 + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # create database db + sql = f"create database db vgroups 2 precision 'ns' " + tdLog.info(sql) + tdSql.execute(sql) + sql = f"use db" + tdSql.execute(sql) + + # create super talbe st + sql = f"create table st(ts timestamp, c1 int, c2 bigint, ts1 timestamp) tags(area int)" + tdLog.info(sql) + tdSql.execute(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table t{i} using st tags({i}) " + tdSql.execute(sql) + + # create stream + sql = "create stream ma into sta as select count(ts) from st interval(100b)" + tdLog.info(sql) + tdSql.execute(sql) + + # insert data + self.insertData() + + # check data correct + def checkExpect(self, sql, expectVal): + tdSql.query(sql) + rowCnt = tdSql.getRows() + for i in range(rowCnt): + val = tdSql.getData(i,0) + if val != expectVal: + tdLog.exit(f"Not expect . query={val} expect={expectVal} i={i} sql={sql}") + return False + + tdLog.info(f"check expect ok. sql={sql} expect ={expectVal} rowCnt={rowCnt}") + return True + + + + + # check time macro + def checkTimeMacro(self): + # 2 week + val = 2 + nsval = val*7*24*60*60*1000*1000*1000 + expectVal = self.childCnt * self.childRow + sql = f"select count(ts) from st where timediff(ts - {val}w, ts1) = {nsval} " + self.checkExpect(sql, expectVal) + + # 20 day + val = 20 + nsval = val*24*60*60*1000*1000*1000 + uint = "d" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {nsval} " + self.checkExpect(sql, expectVal) + + # 30 hour + val = 30 + nsval = val*60*60*1000*1000*1000 + uint = "h" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {nsval} " + self.checkExpect(sql, expectVal) + + # 90 minutes + val = 90 + nsval = val*60*1000*1000*1000 + uint = "m" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {nsval} " + self.checkExpect(sql, expectVal) + # 2s + val = 2 + nsval = val*1000*1000*1000 + uint = "s" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {nsval} " + self.checkExpect(sql, expectVal) + # 20a + val = 5 + nsval = val*1000*1000 + uint = "a" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {nsval} " + self.checkExpect(sql, expectVal) + # 300u + val = 300 + nsval = val*1000 + uint = "u" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {nsval} " + self.checkExpect(sql, expectVal) + # 8b + val = 8 + sql = f"select timediff(ts - {val}b, ts1) from st " + self.checkExpect(sql, val) + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.clock_gettime(time.CLOCK_REALTIME) + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # where + def checkWhere(self): + cnt = 300 + start = self.ts - cnt + sql = f"select count(ts) from st where ts >= {start} and ts <= {self.ts}" + self.checkExpect(sql, cnt) + + for i in range(50): + cnt = random.randint(1,40000) + base = 2000 + start = self.ts - cnt - base + end = self.ts - base + sql = f"select count(ts) from st where ts >= {start} and ts < {end}" + self.checkExpect(sql, cnt) + + # stream + def checkStream(self): + allRows = self.childCnt * self.childRow + # ensure write data is expected + sql = "select count(*) from (select diff(ts) as a from (select ts from st order by ts asc)) where a=1;" + self.checkExpect(sql, allRows - 1) + + # stream count is ok + sql =f"select count(*) from sta" + cnt = int(allRows / 100) - 1 # last window is not close, so need reduce one + self.checkExpect(sql, cnt) + + # check fields + sql =f"select count(*) from sta where `count(ts)` != 100" + self.checkExpect(sql, 0) + + # check timestamp + sql =f"select count(*) from (select diff(`_wstart`) from sta)" + self.checkExpect(sql, cnt - 1) + sql =f"select count(*) from (select diff(`_wstart`) as a from sta) where a != 100" + self.checkExpect(sql, 0) + + # run + def run(self): + # prepare env + self.prepareEnv() + + # time macro like 1w 1d 1h 1m 1s 1a 1u 1b + self.checkTimeMacro() + + # check where + self.checkWhere() + + # check stream + self.checkStream() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/precisionUS.py b/tests/system-test/1-insert/precisionUS.py new file mode 100644 index 0000000000000000000000000000000000000000..1b41d6601008c6a62ec1034fd3f93252861c4680 --- /dev/null +++ b/tests/system-test/1-insert/precisionUS.py @@ -0,0 +1,287 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null," + else: + value += f"{c2}," + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + # c3 same with ts + value += f"{self.ts})" + + # move next + self.ts += 1 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + + sql = "flush database db;" + tdLog.info(sql) + tdSql.execute(sql) + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1680000000000*1000 + self.childCnt = 5 + self.childRow = 10000 + self.batchSize = 5000 + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # create database db + sql = f"create database db vgroups 2 precision 'us' " + tdLog.info(sql) + tdSql.execute(sql) + sql = f"use db" + tdSql.execute(sql) + + # create super talbe st + sql = f"create table st(ts timestamp, c1 int, c2 bigint, ts1 timestamp) tags(area int)" + tdLog.info(sql) + tdSql.execute(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table t{i} using st tags({i}) " + tdSql.execute(sql) + + # create stream + sql = "create stream ma into sta as select count(ts) from st interval(100u)" + tdLog.info(sql) + tdSql.execute(sql) + + # insert data + self.insertData() + + # check data correct + def checkExpect(self, sql, expectVal): + tdSql.query(sql) + rowCnt = tdSql.getRows() + for i in range(rowCnt): + val = tdSql.getData(i,0) + if val != expectVal: + tdLog.exit(f"Not expect . query={val} expect={expectVal} i={i} sql={sql}") + return False + + tdLog.info(f"check expect ok. sql={sql} expect ={expectVal} rowCnt={rowCnt}") + return True + + + # check time macro + def checkTimeMacro(self): + # 2 week + val = 2 + usval = val*7*24*60*60*1000*1000 + expectVal = self.childCnt * self.childRow + sql = f"select count(ts) from st where timediff(ts - {val}w, ts1) = {usval} " + self.checkExpect(sql, expectVal) + + # 20 day + val = 20 + usval = val*24*60*60*1000*1000 + uint = "d" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {usval} " + self.checkExpect(sql, expectVal) + + # 30 hour + val = 30 + usval = val*60*60*1000*1000 + uint = "h" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {usval} " + self.checkExpect(sql, expectVal) + + # 90 minutes + val = 90 + usval = val*60*1000*1000 + uint = "m" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {usval} " + self.checkExpect(sql, expectVal) + # 2s + val = 2 + usval = val*1000*1000 + uint = "s" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {usval} " + self.checkExpect(sql, expectVal) + # 20a + val = 20 + usval = val*1000 + uint = "a" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {usval} " + self.checkExpect(sql, expectVal) + # 300u + val = 300 + usval = val*1 + uint = "u" + sql = f"select count(ts) from st where timediff(ts - {val}{uint}, ts1) = {usval} " + self.checkExpect(sql, expectVal) + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.clock_gettime(time.CLOCK_REALTIME) + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # where + def checkWhere(self): + cnt = 300 + start = self.ts - cnt + sql = f"select count(ts) from st where ts >= {start} and ts <= {self.ts}" + self.checkExpect(sql, cnt) + + for i in range(50): + cnt = random.randint(1,40000) + base = 2000 + start = self.ts - cnt - base + end = self.ts - base + sql = f"select count(ts) from st where ts >= {start} and ts < {end}" + self.checkExpect(sql, cnt) + + # stream + def checkStream(self): + allRows = self.childCnt * self.childRow + # ensure write data is expected + sql = "select count(*) from (select diff(ts) as a from (select ts from st order by ts asc)) where a=1;" + self.checkExpect(sql, allRows - 1) + + # stream count is ok + sql =f"select count(*) from sta" + cnt = int(allRows / 100) - 1 # last window is not close, so need reduce one + self.checkExpect(sql, cnt) + + # check fields + sql =f"select count(*) from sta where `count(ts)` != 100" + self.checkExpect(sql, 0) + + # check timestamp + sql =f"select count(*) from (select diff(`_wstart`) from sta)" + self.checkExpect(sql, cnt - 1) + sql =f"select count(*) from (select diff(`_wstart`) as a from sta) where a != 100" + self.checkExpect(sql, 0) + + # run + def run(self): + # prepare env + self.prepareEnv() + + # time macro like 1w 1d 1h 1m 1s 1a 1u + self.checkTimeMacro() + + # check where + self.checkWhere() + + # check stream + self.checkStream() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/rowlength64k.json b/tests/system-test/1-insert/rowlength64k.json new file mode 100755 index 0000000000000000000000000000000000000000..a0a3e7522ca7b90991e6f797b9af93efd4b95768 --- /dev/null +++ b/tests/system-test/1-insert/rowlength64k.json @@ -0,0 +1,128 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1, + "max_sql_len": 102400000, + "databases": [{ + "dbinfo": { + "name": "json_test", + "drop": "yes", + "replica": 1, + "precision": "ms", + "keep": 36500, + "minRows": 100, + "maxRows": 4096 + }, + "super_tables": [{ + "name": "stb_old", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb_old_", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "sample", + "sample_file": "./tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.csv", + "tags_file": "", + "columns": [{"type": "INT","count":1000}, {"type": "BINARY", "len": 16, "count":20}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + },{ + "name": "stb_new", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb_new_", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT","count":4000}, {"type": "BINARY", "len": 16, "count":90}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":3}] + },{ + "name": "stb_mix", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb_mix_", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT","count":500},{"type": "SMALLINT","count":500},{"type": "TINYINT","count":500},{"type": "DOUBLE","count":500},{"type": "FLOAT","count":500},{"type": "BOOL","count":500},{"type": "BIGINT","count":500},{"type": "NCHAR","len": 20,"count":300},{"type": "BINARY","len": 34,"count":290},{"type": "BINARY","len": 101,"count":1}], + "tags": [{"type": "INT", "count":3}, {"type": "NCHAR", "len": 10, "count":1}] + },{ + "name": "stb_excel", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb_excel_", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT","count":500},{"type": "SMALLINT","count":500},{"type": "SMALLINT","count":500},{"type": "DOUBLE","count":500},{"type": "FLOAT","count":500},{"type": "BOOL","count":500},{"type": "BIGINT","count":500},{"type": "NCHAR","len": 30,"count":300},{"type": "BINARY","len": 47,"count":290},{"type": "BINARY","len": 211,"count":1}], + "tags": [{"type": "INT", "count":3}, {"type": "NCHAR", "len": 10, "count":1}] + }] + }] +} \ No newline at end of file diff --git a/tests/system-test/1-insert/rowlength64k.py b/tests/system-test/1-insert/rowlength64k.py new file mode 100755 index 0000000000000000000000000000000000000000..6e36d66e77bbdf48bdd536543e2c42721367d4f5 --- /dev/null +++ b/tests/system-test/1-insert/rowlength64k.py @@ -0,0 +1,1291 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import os +import time +import taos +import subprocess +import string +from faker import Faker +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +from util.dnodes import * + +class TDTestCase: + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1} + + def init(self, conn, logSql, replicaVar): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.testcasePath = os.path.split(__file__)[0] + self.testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + + now = time.time() + self.ts = int(round(now * 1000)) + self.num = 100 + + def get_random_string(self, length): + letters = string.ascii_lowercase + result_str = ''.join(random.choice(letters) for i in range(length)) + return result_str + + def ins_query(self): + sql = 'select * from information_schema.ins_tables where table_name match "table"' + tdSql.query(sql) + + self.stable_query() + + def stable_query(self): + # select * from stable_1 where loc match '[a-z]'; + show_sql = "show db.stables;" + tdSql.query(show_sql) + queryRows = len(tdSql.queryResult) + for i in range(queryRows): + show_sql = "show db.stables;" + tdSql.query(show_sql) + stable_name = tdSql.queryResult[i][0] + + stable_sql = "select * from db.%s where loc match '[a-z]'" %stable_name + tdSql.query(stable_sql) + + def run_8(self): + + print("==============step8,stable table , mix data type==============") + sql = "create stable db.stable_16(ts timestamp, " + sql += "col4090 int ," + sql += "col4091 binary(65517))" + sql += " tags (loc binary(16370),tag_1 int,tag_2 int,tag_3 int) " + tdLog.info(len(sql)) + tdSql.execute(sql) + sql = '''create table db.table_160 using db.stable_16 + tags('%s' , '1' , '2' , '3' );'''% self.get_random_string(16370) + tdSql.execute(sql) + + for i in range(self.num): + sql = "insert into db.table_160 values(%d, " + sql += "'%d'," % i + sql += "'%s')" % self.get_random_string(65517) + tdSql.execute(sql % (self.ts + i)) + time.sleep(1) + tdSql.query("select count(*) from db.table_160") + tdSql.checkData(0, 0, self.num) + tdSql.query("select * from db.table_160") + tdSql.checkRows(self.num) + tdSql.checkCols(3) + + self.ins_query() + + #insert null value + tdLog.info('test insert null value') + sql = '''create table db.table_161 using db.stable_16 + tags('table_61' , '1' , '2' , '3' );''' + tdSql.execute(sql) + + for i in range(self.num): + sql = "insert into db.table_161(ts) values(%d) " + tdSql.execute(sql % (self.ts + i)) + time.sleep(1) + tdSql.query("select count(*) from db.table_161") + tdSql.checkData(0, 0, self.num) + tdSql.query("select * from db.table_161") + tdSql.checkRows(self.num) + tdSql.checkCols(3) + + #define TSDB_MAX_BYTES_PER_ROW 65531 TSDB_MAX_TAGS_LEN 16384 + #ts:8\int:4\smallint:2\bigint:8\bool:1\float:4\tinyint:1\nchar:4*（）+2[offset]\binary:1*（）+2[offset] + tdLog.info('test super table max bytes per row 65531') + sql = "create table db.stable_17(ts timestamp, " + sql += "col4090 int," + sql += "col4091 binary(65517))" + sql += " tags (loc binary(16370),tag_1 int,tag_2 int,tag_3 int) " #4*3+16370+2 + tdSql.execute(sql) + sql = '''create table db.table_170 using db.stable_17 + tags('%s' , '1' , '2' , '3' );'''% self.get_random_string(16370) + tdSql.execute(sql) + tdSql.query("select * from db.table_170") + tdSql.checkCols(3) + tdSql.query("describe db.table_170") + tdSql.checkRows(7) + + self.ins_query() + + tdLog.info('test super table drop and add column or tag') + sql = "alter stable db.stable_17 drop column col4091; " + tdSql.execute(sql) + sql = "select * from db.stable_17; " + tdSql.query(sql) + tdSql.checkCols(6) + sql = "alter table db.stable_17 add column col4091 binary(65518); " + tdSql.error(sql) + sql = "alter table db.stable_17 add column col4091 binary(65517); " + tdSql.execute(sql) + sql = "select * from db.stable_17; " + tdSql.query(sql) + tdSql.checkCols(7) + + self.ins_query() + + sql = "alter stable db.stable_17 drop tag loc; " + tdSql.execute(sql) + sql = "select * from db.stable_17; " + tdSql.query(sql) + tdSql.checkCols(6) + sql = "alter table db.stable_17 add tag loc binary(16371); " + tdSql.error(sql) + sql = "alter table db.stable_17 add tag loc binary(16370); " + tdSql.execute(sql) + sql = "select * from db.stable_17; " + tdSql.query(sql) + tdSql.checkCols(7) + + sql = "alter stable db.stable_17 drop tag tag_1; " + tdSql.execute(sql) + sql = "select * from db.stable_17; " + tdSql.query(sql) + tdSql.checkCols(6) + sql = "alter table db.stable_17 add tag tag_1 int; " + tdSql.execute(sql) + sql = "select * from db.stable_17; " + tdSql.query(sql) + tdSql.checkCols(7) + sql = "alter table db.stable_17 add tag loc1 nchar(10); " + tdSql.error(sql) + + tdLog.info('test super table max bytes per row 65531') + sql = "create table db.stable_18(ts timestamp, " + sql += "col4091 binary(65518))" + sql += " tags (loc nchar(10),tag_1 int,tag_2 int,tag_3 int) " + tdSql.error(sql) + + tdLog.info('test super table max bytes per row tag 16384') + sql = "create table db.stable_18(ts timestamp, " + sql += "col4091 binary(65517))" + sql += " tags (loc binary(16371),tag_1 int,tag_2 int,tag_3 int) " + tdSql.error(sql) + + self.ins_query() + + def run_9(self): + + print("==============step9,stable table , mix data type==============") + sql = "create stable db.stable_26(ts timestamp, " + sql += "col4090 int ," + sql += "col4091 binary(65517))" + sql += " tags (loc nchar(4092),tag_1 int,tag_2 int,tag_3 int) " + tdLog.info(len(sql)) + tdSql.execute(sql) + sql = '''create table db.table_260 using db.stable_26 + tags('%s' , '1' , '2' , '3' );'''% self.get_random_string(4092) + tdSql.execute(sql) + + for i in range(self.num): + sql = "insert into db.table_260 values(%d, " + sql += "'%d'," % i + sql += "'%s')" % self.get_random_string(65517) + tdSql.execute(sql % (self.ts + i)) + time.sleep(1) + tdSql.query("select count(*) from db.table_260") + tdSql.checkData(0, 0, self.num) + tdSql.query("select * from db.table_260") + tdSql.checkRows(self.num) + tdSql.checkCols(3) + + self.ins_query() + + #insert null value + tdLog.info('test insert null value') + sql = '''create table db.table_261 using db.stable_26 + tags('table_261' , '1' , '2' , '3' );''' + tdSql.execute(sql) + + for i in range(self.num): + sql = "insert into db.table_261(ts) values(%d) " + tdSql.execute(sql % (self.ts + i)) + time.sleep(1) + tdSql.query("select count(*) from db.table_261") + tdSql.checkData(0, 0, self.num) + tdSql.query("select * from db.table_261") + tdSql.checkRows(self.num) + tdSql.checkCols(3) + + #define TSDB_MAX_BYTES_PER_ROW 65531 TSDB_MAX_TAGS_LEN 16384 + #ts:8\int:4\smallint:2\bigint:8\bool:1\float:4\tinyint:1\nchar:4*（）+2[offset]\binary:1*（）+2[offset] + tdLog.info('test super table max bytes per row 65531') + sql = "create table db.stable_27(ts timestamp, " + sql += "col4090 int," + sql += "col4091 binary(65517))" + sql += " tags (loc nchar(4092),tag_1 int,tag_2 int,tag_3 int) " #4*3+16370+2 + tdSql.execute(sql) + sql = '''create table db.table_270 using db.stable_27 + tags('%s' , '1' , '2' , '3' );'''% self.get_random_string(4092) + tdSql.execute(sql) + tdSql.query("select * from db.table_270") + tdSql.checkCols(3) + tdSql.query("describe db.table_270") + tdSql.checkRows(7) + + self.ins_query() + + tdLog.info('test super table drop and add column or tag') + sql = "alter stable db.stable_27 drop column col4091; " + tdSql.execute(sql) + sql = "select * from db.stable_27; " + tdSql.query(sql) + tdSql.checkCols(6) + sql = "alter table db.stable_27 add column col4091 binary(65518); " + tdSql.error(sql) + sql = "alter table db.stable_27 add column col4091 binary(65517); " + tdSql.execute(sql) + sql = "select * from db.stable_27; " + tdSql.query(sql) + tdSql.checkCols(7) + + self.ins_query() + + sql = "alter stable db.stable_27 drop tag loc; " + tdSql.execute(sql) + sql = "select * from db.stable_27; " + tdSql.query(sql) + tdSql.checkCols(6) + sql = "alter table db.stable_27 add tag loc binary(16371); " + tdSql.error(sql) + sql = "alter table db.stable_27 add tag loc binary(16370); " + tdSql.execute(sql) + sql = "select * from db.stable_27; " + tdSql.query(sql) + tdSql.checkCols(7) + + sql = "alter stable db.stable_27 drop tag tag_1; " + tdSql.execute(sql) + sql = "select * from db.stable_27; " + tdSql.query(sql) + tdSql.checkCols(6) + sql = "alter table db.stable_27 add tag tag_1 int; " + tdSql.execute(sql) + sql = "select * from db.stable_27; " + tdSql.query(sql) + tdSql.checkCols(7) + sql = "alter table db.stable_27 add tag loc1 nchar(10); " + tdSql.error(sql) + + tdLog.info('test super table max bytes per row 65531') + sql = "create table db.stable_28(ts timestamp, " + sql += "col4091 binary(65518))" + sql += " tags (loc nchar(10),tag_1 int,tag_2 int,tag_3 int) " + tdSql.error(sql) + + tdLog.info('test super table max bytes per row tag 16384') + sql = "create table db.stable_28(ts timestamp, " + sql += "col4091 binary(65517))" + sql += " tags (loc binary(16371),tag_1 int,tag_2 int,tag_3 int) " + tdSql.error(sql) + + self.ins_query() + + def run_1(self): + + + print("==============step1, regular table, 1 ts + 4094 cols + 1 binary==============") + startTime = time.time() + sql = "create table db.regular_table_1(ts timestamp, " + for i in range(4094): + sql += "col%d int, " % (i + 1) + sql += "col4095 binary(22))" + tdLog.info(len(sql)) + tdSql.execute(sql) + + for i in range(self.num): + sql = "insert into db.regular_table_1 values(%d, " + for j in range(4094): + str = "'%s', " % random.randint(0,1000) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + time.sleep(1) + tdSql.query("select count(*) from db.regular_table_1") + tdSql.checkData(0, 0, self.num) + tdSql.query("select * from db.regular_table_1") + tdSql.checkRows(self.num) + tdSql.checkCols(4096) + + self.ins_query() + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + #insert in order + tdLog.info('test insert in order') + for i in range(self.num): + sql = "insert into db.regular_table_1 (ts,col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col4095) values(%d, " + for j in range(10): + str = "'%s', " % random.randint(0,1000) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i + 1000)) + time.sleep(1) + tdSql.query("select count(*) from db.regular_table_1") + tdSql.checkData(0, 0, 2*self.num) + tdSql.query("select * from db.regular_table_1") + tdSql.checkRows(2*self.num) + tdSql.checkCols(4096) + + #insert out of order + tdLog.info('test insert out of order') + for i in range(self.num): + sql = "insert into db.regular_table_1 (ts,col123,col2213,col331,col41,col523,col236,col71,col813,col912,col1320,col4095) values(%d, " + for j in range(10): + str = "'%s', " % random.randint(0,1000) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i + 2000)) + time.sleep(1) + tdSql.query("select count(*) from db.regular_table_1") + tdSql.checkData(0, 0, 3*self.num) + tdSql.query("select * from db.regular_table_1") + tdSql.checkRows(3*self.num) + tdSql.checkCols(4096) + + self.ins_query() + + def run_2(self): + + print("==============step2,regular table error col or value==============") + tdLog.info('test regular table exceeds row num') + # column > 4096 + sql = "create table db.regular_table_2(ts timestamp, " + for i in range(4095): + sql += "col%d int, " % (i + 1) + sql += "col4096 binary(22))" + tdLog.info(len(sql)) + tdSql.error(sql) + + self.ins_query() + + # column > 4096 + sql = "insert into db.regular_table_1 values(%d, " + for j in range(4095): + str = "'%s', " % random.randint(0,1000) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.error(sql) + + # insert column < 4096 + sql = "insert into db.regular_table_1 values(%d, " + for j in range(4092): + str = "'%s', " % random.randint(0,1000) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.error(sql) + + # alter column > 4096 + sql = "alter table db.regular_table_1 add column max int; " + tdSql.error(sql) + + self.ins_query() + + def run_3(self): + + + print("==============step3,regular table , mix data type==============") + startTime = time.time() + sql = "create table db.regular_table_3(ts timestamp, " + for i in range(2000): + sql += "col%d int, " % (i + 1) + for i in range(2000,4094): + sql += "col%d bigint, " % (i + 1) + sql += "col4095 binary(22))" + tdLog.info(len(sql)) + tdSql.execute(sql) + + for i in range(self.num): + sql = "insert into db.regular_table_3 values(%d, " + for j in range(4094): + str = "'%s', " % random.randint(0,1000) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + time.sleep(1) + tdSql.query("select count(*) from db.regular_table_3") + tdSql.checkData(0, 0, self.num) + tdSql.query("select * from db.regular_table_3") + tdSql.checkRows(self.num) + tdSql.checkCols(4096) + + self.ins_query() + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + sql = "create table db.regular_table_4(ts timestamp, " + for i in range(500): + sql += "int_%d int, " % (i + 1) + for i in range(500,1000): + sql += "smallint_%d smallint, " % (i + 1) + for i in range(1000,1500): + sql += "tinyint_%d tinyint, " % (i + 1) + for i in range(1500,2000): + sql += "double_%d double, " % (i + 1) + for i in range(2000,2500): + sql += "float_%d float, " % (i + 1) + for i in range(2500,3000): + sql += "bool_%d bool, " % (i + 1) + for i in range(3000,3500): + sql += "bigint_%d bigint, " % (i + 1) + for i in range(3500,3800): + sql += "nchar_%d nchar(4), " % (i + 1) + for i in range(3800,4090): + sql += "binary_%d binary(10), " % (i + 1) + for i in range(4090,4094): + sql += "timestamp_%d timestamp, " % (i + 1) + sql += "col4095 binary(22))" + tdLog.info(len(sql)) + tdSql.execute(sql) + + self.ins_query() + + for i in range(self.num): + sql = "insert into db.regular_table_4 values(%d, " + for j in range(500): + str = "'%s', " % random.randint(-2147483647,2147483647) + sql += str + for j in range(500,1000): + str = "'%s', " % random.randint(-32767,32767 ) + sql += str + for j in range(1000,1500): + str = "'%s', " % random.randint(-127,127) + sql += str + for j in range(1500,2000): + str = "'%s', " % random.randint(-922337203685477580700,922337203685477580700) + sql += str + for j in range(2000,2500): + str = "'%s', " % random.randint(-92233720368547758070,92233720368547758070) + sql += str + for j in range(2500,3000): + str = "'%s', " % random.choice(['true','false']) + sql += str + for j in range(3000,3500): + str = "'%s', " % random.randint(-9223372036854775807,9223372036854775807) + sql += str + for j in range(3500,3800): + str = "'%s', " % self.get_random_string(4) + sql += str + for j in range(3800,4090): + str = "'%s', " % self.get_random_string(10) + sql += str + for j in range(4090,4094): + str = "%s, " % (self.ts + j) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + time.sleep(1) + tdSql.query("select count(*) from db.regular_table_4") + tdSql.checkData(0, 0, self.num) + tdSql.query("select * from db.regular_table_4") + tdSql.checkRows(self.num) + tdSql.checkCols(4096) + tdLog.info("end ,now new one") + + self.ins_query() + + #insert null value + tdLog.info('test insert null value') + for i in range(self.num): + sql = "insert into db.regular_table_4 values(%d, " + for j in range(2500): + str = "'%s', " % random.choice(['NULL' ,'NULL' ,'NULL' ,1 , 10 ,100 ,-100 ,-10, 88 ,66 ,'NULL' ,'NULL' ,'NULL' ]) + sql += str + for j in range(2500,3000): + str = "'%s', " % random.choice(['true' ,'false']) + sql += str + for j in range(3000,3500): + str = "'%s', " % random.randint(-9223372036854775807,9223372036854775807) + sql += str + for j in range(3500,3800): + str = "'%s', " % self.get_random_string(4) + sql += str + for j in range(3800,4090): + str = "'%s', " % self.get_random_string(10) + sql += str + for j in range(4090,4094): + str = "%s, " % (self.ts + j) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i + 10000)) + time.sleep(1) + tdSql.query("select count(*) from db.regular_table_4") + tdSql.checkData(0, 0, 2*self.num) + tdSql.query("select * from db.regular_table_4") + tdSql.checkRows(2*self.num) + tdSql.checkCols(4096) + + #insert in order + tdLog.info('test insert in order') + for i in range(self.num): + sql = "insert into db.regular_table_4 (ts,int_2,int_22,int_169,smallint_537,smallint_607,tinyint_1030,tinyint_1491,double_1629,double_1808,float_2075,col4095) values(%d, " + for j in range(10): + str = "'%s', " % random.randint(0,100) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i + 1000)) + time.sleep(1) + tdSql.query("select count(*) from db.regular_table_4") + tdSql.checkData(0, 0, 3*self.num) + tdSql.query("select * from db.regular_table_4") + tdSql.checkRows(3*self.num) + tdSql.checkCols(4096) + + self.ins_query() + + #insert out of order + tdLog.info('test insert out of order') + for i in range(self.num): + sql = "insert into db.regular_table_4 (ts,int_169,float_2075,int_369,tinyint_1491,tinyint_1030,float_2360,smallint_537,double_1808,double_1608,double_1629,col4095) values(%d, " + for j in range(10): + str = "'%s', " % random.randint(0,100) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i + 2000)) + time.sleep(1) + tdSql.query("select count(*) from db.regular_table_4") + tdSql.checkData(0, 0, 4*self.num) + tdSql.query("select * from db.regular_table_4") + tdSql.checkRows(4*self.num) + tdSql.checkCols(4096) + + #define TSDB_MAX_BYTES_PER_ROW 49151[old:1024 && 16384] + # 3.0 old: tag binary max is 16384, col+ts binary max 49151 + # 3.0 new: tag binary max is 16384-2, col+ts binary max 65531 + #ts:8\int:4\smallint:2\bigint:8\bool:1\float:4\tinyint:1\nchar:4*（）+2[offset]\binary:1*（）+2[offset] + tdLog.info('test regular_table max bytes per row 65531') + sql = "create table db.regular_table_5(ts timestamp, " #1*8 sum=8 + for i in range(500): + sql += "int_%d int, " % (i + 1) #500*4=2000 sum=2008 + for i in range(500,1000): + sql += "smallint_%d smallint, " % (i + 1) #500*2=1000 sum=3008 + for i in range(1000,1500): + sql += "tinyint_%d tinyint, " % (i + 1) #500*1=500 sum=3508 + for i in range(1500,2000): + sql += "double_%d double, " % (i + 1) #500*8=4000 sum=7508 + for i in range(2000,2500): + sql += "float_%d float, " % (i + 1) #500*4=2000 sum=9508 + for i in range(2500,3000): + sql += "bool_%d bool, " % (i + 1) #500*1=500 sum=10008 + for i in range(3000,3500): + sql += "bigint_%d bigint, " % (i + 1) #500*8=4000 sum=14008 + for i in range(3500,3800): + sql += "nchar_%d nchar(32), " % (i + 1) #300*(32*4+2)=39000 sum=53008 + for i in range(3800,4090): + sql += "binary_%d binary(40), " % (i + 1) #290*(40+2)=12180 sum=65188 + for i in range(4090,4094): + sql += "timestamp_%d timestamp, " % (i + 1) #4*8=32 sum=65220 + sql += "col4095 binary(309))" #309+2=311 sum=65531 + tdSql.execute(sql) + tdSql.query("select * from db.regular_table_5") + tdSql.checkCols(4096) + + sql = "alter table db.regular_table_5 modify column col4095 binary(310); " + tdSql.error(sql) + + self.ins_query() + + # drop and add + sql = "alter table db.regular_table_5 drop column col4095; " + tdSql.execute(sql) + sql = "select * from db.regular_table_5; " + tdSql.query(sql) + tdSql.checkCols(4095) + sql = "alter table db.regular_table_5 add column col4095 binary(310); " + tdSql.error(sql) + sql = "alter table db.regular_table_5 add column col4095 binary(309); " + tdSql.execute(sql) + sql = "select * from db.regular_table_5; " + tdSql.query(sql) + tdSql.checkCols(4096) + + #out TSDB_MAX_BYTES_PER_ROW 65531 + tdLog.info('test regular_table max bytes per row out 65531') + sql = "create table db.regular_table_6(ts timestamp, " + for i in range(500): + sql += "int_%d int, " % (i + 1) + for i in range(500,1000): + sql += "smallint_%d smallint, " % (i + 1) + for i in range(1000,1500): + sql += "tinyint_%d tinyint, " % (i + 1) + for i in range(1500,2000): + sql += "double_%d double, " % (i + 1) + for i in range(2000,2500): + sql += "float_%d float, " % (i + 1) + for i in range(2500,3000): + sql += "bool_%d bool, " % (i + 1) + for i in range(3000,3500): + sql += "bigint_%d bigint, " % (i + 1) + for i in range(3500,3800): + sql += "nchar_%d nchar(32), " % (i + 1) + for i in range(3800,4090): + sql += "binary_%d binary(40), " % (i + 1) + for i in range(4090,4094): + sql += "timestamp_%d timestamp, " % (i + 1) + sql += "col4095 binary(310))" + tdLog.info(len(sql)) + tdSql.error(sql) + + self.ins_query() + + def run_4(self): + + print("==============step4, super table , 1 ts + 4090 cols + 4 tags ==============") + startTime = time.time() + sql = "create stable db.stable_1(ts timestamp, " + for i in range(4090): + sql += "col%d int, " % (i + 1) + sql += "col4091 binary(22))" + sql += " tags (loc nchar(10),tag_1 int,tag_2 int,tag_3 int) " + tdLog.info(len(sql)) + tdSql.execute(sql) + sql = '''create table db.table_0 using db.stable_1 + tags('%s' , '1' , '2' , '3' );'''% self.get_random_string(10) + tdSql.execute(sql) + + for i in range(self.num): + sql = "insert into db.table_0 values(%d, " + for j in range(4090): + str = "'%s', " % random.randint(0,1000) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + time.sleep(1) + tdSql.query("select count(*) from db.table_0") + tdSql.checkData(0, 0, self.num) + tdSql.query("select * from db.table_0") + tdSql.checkRows(self.num) + tdSql.checkCols(4092) + + sql = '''create table db.table_1 using db.stable_1 + tags('%s' , '1' , '2' , '3' );'''% self.get_random_string(10) + tdSql.execute(sql) + + self.ins_query() + + for i in range(self.num): + sql = "insert into db.table_1 values(%d, " + for j in range(2080): + sql += "'%d', " % random.randint(0,1000) + for j in range(2080,4080): + sql += "'%s', " % 'NULL' + for j in range(4080,4090): + sql += "'%s', " % random.randint(0,10000) + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + time.sleep(1) + tdSql.query("select count(*) from db.table_1") + tdSql.checkData(0, 0, self.num) + tdSql.query("select * from db.table_1") + tdSql.checkRows(self.num) + tdSql.checkCols(4092) + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + #insert in order + tdLog.info('test insert in order') + for i in range(self.num): + sql = "insert into db.table_1 (ts,col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col4091) values(%d, " + for j in range(10): + str = "'%s', " % random.randint(0,1000) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i + 1000)) + time.sleep(1) + tdSql.query("select count(*) from db.table_1") + tdSql.checkData(0, 0, 2*self.num) + tdSql.query("select * from db.table_1") + tdSql.checkRows(2*self.num) + tdSql.checkCols(4092) + + #insert out of order + tdLog.info('test insert out of order') + for i in range(self.num): + sql = "insert into db.table_1 (ts,col123,col2213,col331,col41,col523,col236,col71,col813,col912,col1320,col4091) values(%d, " + for j in range(10): + str = "'%s', " % random.randint(0,1000) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i + 2000)) + time.sleep(1) + tdSql.query("select count(*) from db.table_1") + tdSql.checkData(0, 0, 3*self.num) + tdSql.query("select * from db.table_1") + tdSql.checkRows(3*self.num) + tdSql.checkCols(4092) + + self.ins_query() + + def run_5(self): + + print("==============step5,stable table , mix data type==============") + sql = "create stable db.stable_3(ts timestamp, " + for i in range(500): + sql += "int_%d int, " % (i + 1) + for i in range(500,1000): + sql += "smallint_%d smallint, " % (i + 1) + for i in range(1000,1500): + sql += "tinyint_%d tinyint, " % (i + 1) + for i in range(1500,2000): + sql += "double_%d double, " % (i + 1) + for i in range(2000,2500): + sql += "float_%d float, " % (i + 1) + for i in range(2500,3000): + sql += "bool_%d bool, " % (i + 1) + for i in range(3000,3500): + sql += "bigint_%d bigint, " % (i + 1) + for i in range(3500,3800): + sql += "nchar_%d nchar(4), " % (i + 1) + for i in range(3800,4090): + sql += "binary_%d binary(10), " % (i + 1) + sql += "col4091 binary(22))" + sql += " tags (loc binary(16370),tag_1 int,tag_2 int,tag_3 int) " + tdLog.info(len(sql)) + tdSql.execute(sql) + sql = '''create table db.table_30 using db.stable_3 + tags('%s' , '1' , '2' , '3' );'''%self.get_random_string(16370) + tdSql.execute(sql) + + self.ins_query() + + for i in range(self.num): + sql = "insert into db.table_30 values(%d, " + for j in range(500): + str = "'%s', " % random.randint(-2147483647,2147483647) + sql += str + for j in range(500,1000): + str = "'%s', " % random.randint(-32767,32767 ) + sql += str + for j in range(1000,1500): + str = "'%s', " % random.randint(-127,127) + sql += str + for j in range(1500,2000): + str = "'%s', " % random.randint(-922337203685477580700,922337203685477580700) + sql += str + for j in range(2000,2500): + str = "'%s', " % random.randint(-92233720368547758070,92233720368547758070) + sql += str + for j in range(2500,3000): + str = "'%s', " % random.choice(['true','false']) + sql += str + for j in range(3000,3500): + str = "'%s', " % random.randint(-9223372036854775807,9223372036854775807) + sql += str + for j in range(3500,3800): + str = "'%s', " % self.get_random_string(4) + sql += str + for j in range(3800,4090): + str = "'%s', " % self.get_random_string(10) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + time.sleep(1) + tdSql.query("select count(*) from db.table_30") + tdSql.checkData(0, 0, self.num) + tdSql.query("select * from db.table_30") + tdSql.checkRows(self.num) + tdSql.checkCols(4092) + + #insert null value + tdLog.info('test insert null value') + sql = '''create table db.table_31 using db.stable_3 + tags('%s' , '1' , '2' , '3' );'''%self.get_random_string(16370) + tdSql.execute(sql) + + for i in range(self.num): + sql = "insert into db.table_31 values(%d, " + for j in range(2500): + str = "'%s', " % random.choice(['NULL' ,'NULL' ,'NULL' ,1 , 10 ,100 ,-100 ,-10, 88 ,66 ,'NULL' ,'NULL' ,'NULL' ]) + sql += str + for j in range(2500,3000): + str = "'%s', " % random.choice(['true' ,'false']) + sql += str + for j in range(3000,3500): + str = "'%s', " % random.randint(-9223372036854775807,9223372036854775807) + sql += str + for j in range(3500,3800): + str = "'%s', " % self.get_random_string(4) + sql += str + for j in range(3800,4090): + str = "'%s', " % self.get_random_string(10) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + time.sleep(1) + tdSql.query("select count(*) from db.table_31") + tdSql.checkData(0, 0, self.num) + tdSql.query("select * from db.table_31") + tdSql.checkRows(self.num) + tdSql.checkCols(4092) + + self.ins_query() + + #insert in order + tdLog.info('test insert in order') + for i in range(self.num): + sql = "insert into db.table_31 (ts,int_2,int_22,int_169,smallint_537,smallint_607,tinyint_1030,tinyint_1491,double_1629,double_1808,float_2075,col4091) values(%d, " + for j in range(10): + str = "'%s', " % random.randint(0,100) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i + 1000)) + time.sleep(1) + tdSql.query("select count(*) from db.table_31") + tdSql.checkData(0, 0, 2*self.num) + tdSql.query("select * from db.table_31") + tdSql.checkRows(2*self.num) + tdSql.checkCols(4092) + + #insert out of order + tdLog.info('test insert out of order') + for i in range(self.num): + sql = "insert into db.table_31 (ts,int_169,float_2075,int_369,tinyint_1491,tinyint_1030,float_2360,smallint_537,double_1808,double_1608,double_1629,col4091) values(%d, " + for j in range(10): + str = "'%s', " % random.randint(0,100) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i + 2000)) + time.sleep(1) + tdSql.query("select count(*) from db.table_31") + tdSql.checkData(0, 0, 3*self.num) + tdSql.query("select * from db.table_31") + tdSql.checkRows(3*self.num) + tdSql.checkCols(4092) + + #define TSDB_MAX_BYTES_PER_ROW 65531 TSDB_MAX_TAGS_LEN 16384 + #ts:8\int:4\smallint:2\bigint:8\bool:1\float:4\tinyint:1\nchar:4*（）+2[offset]\binary:1*（）+2[offset] + tdLog.info('test super table max bytes per row 65531') + sql = "create table db.stable_4(ts timestamp, " #1*8 sum=8 + for i in range(500): + sql += "int_%d int, " % (i + 1) #500*4=2000 sum=2008 + for i in range(500,1000): + sql += "smallint_%d smallint, " % (i + 1) #500*2=1000 sum=3008 + for i in range(1000,1500): + sql += "tinyint_%d tinyint, " % (i + 1) #500*1=500 sum=3508 + for i in range(1500,2000): + sql += "double_%d double, " % (i + 1) #500*8=4000 sum=7508 + for i in range(2000,2500): + sql += "float_%d float, " % (i + 1) #500*4=2000 sum=9508 + for i in range(2500,3000): + sql += "bool_%d bool, " % (i + 1) #500*1=500 sum=10008 + for i in range(3000,3500): + sql += "bigint_%d bigint, " % (i + 1) #500*8=4000 sum=14008 + for i in range(3500,3800): + sql += "nchar_%d nchar(32), " % (i + 1) #300*(32*4+2)=39000 sum=53008 + for i in range(3800,4090): + sql += "binary_%d binary(40), " % (i + 1) #290*(40+2)=12180 sum=65188 + sql += "col4091 binary(341))" #341+2=343 sum=65531 + sql += " tags (loc binary(16370),tag_1 int,tag_2 int,tag_3 int) " + tdSql.execute(sql) + sql = '''create table db.table_40 using db.stable_4 + tags('%s' , '1' , '2' , '3' );'''%self.get_random_string(16370) + tdSql.execute(sql) + tdSql.query("select * from db.table_40") + tdSql.checkCols(4092) + tdSql.query("describe db.table_40") + tdSql.checkRows(4096) + + tdLog.info('test super table drop and add column or tag') + sql = "alter stable db.stable_4 drop column col4091; " + tdSql.execute(sql) + sql = "select * from db.stable_4; " + tdSql.query(sql) + tdSql.checkCols(4095) + sql = "alter table db.stable_4 add column col4091 binary(342); " + tdSql.error(sql) + sql = "alter table db.stable_4 add column col4091 binary(341); " + tdSql.execute(sql) + sql = "select * from db.stable_4; " + tdSql.query(sql) + tdSql.checkCols(4096) + + self.ins_query() + + sql = "alter stable db.stable_4 drop tag tag_1; " + tdSql.execute(sql) + sql = "select * from db.stable_4; " + tdSql.query(sql) + tdSql.checkCols(4095) + sql = "alter table db.stable_4 add tag tag_1 int; " + tdSql.execute(sql) + sql = "select * from db.stable_4; " + tdSql.query(sql) + tdSql.checkCols(4096) + sql = "alter table db.stable_4 add tag loc1 nchar(10); " + tdSql.error(sql) + + tdLog.info('test super table max bytes per row 65531') + sql = "create table db.stable_5(ts timestamp, " + for i in range(500): + sql += "int_%d int, " % (i + 1) + for i in range(500,1000): + sql += "smallint_%d smallint, " % (i + 1) + for i in range(1000,1500): + sql += "tinyint_%d tinyint, " % (i + 1) + for i in range(1500,2000): + sql += "double_%d double, " % (i + 1) + for i in range(2000,2500): + sql += "float_%d float, " % (i + 1) + for i in range(2500,3000): + sql += "bool_%d bool, " % (i + 1) + for i in range(3000,3500): + sql += "bigint_%d bigint, " % (i + 1) + for i in range(3500,3800): + sql += "nchar_%d nchar(32), " % (i + 1) + for i in range(3800,4090): + sql += "binary_%d binary(40), " % (i + 1) + sql += "col4091 binary(342))" + sql += " tags (loc nchar(10),tag_1 int,tag_2 int,tag_3 int) " + tdSql.error(sql) + + self.ins_query() + + def run_6(self): + + + print("==============step6,stable table , mix data type==============") + sql = "create stable db.stable_6(ts timestamp, " + for i in range(500): + sql += "int_%d int, " % (i + 1) + for i in range(500,1000): + sql += "smallint_%d smallint, " % (i + 1) + for i in range(1000,1500): + sql += "tinyint_%d tinyint, " % (i + 1) + for i in range(1500,2000): + sql += "double_%d double, " % (i + 1) + for i in range(2000,2500): + sql += "float_%d float, " % (i + 1) + for i in range(2500,3000): + sql += "bool_%d bool, " % (i + 1) + for i in range(3000,3500): + sql += "bigint_%d bigint, " % (i + 1) + for i in range(3500,3800): + sql += "nchar_%d nchar(4), " % (i + 1) + for i in range(3800,4090): + sql += "binary_%d binary(10), " % (i + 1) + sql += "col4091 binary(22))" + sql += " tags (loc binary(16370),tag_1 int,tag_2 int,tag_3 int) " + tdLog.info(len(sql)) + tdSql.execute(sql) + sql = '''create table db.table_60 using db.stable_6 + tags('%s' , '1' , '2' , '3' );'''%self.get_random_string(16370) + tdSql.execute(sql) + + self.ins_query() + + for i in range(self.num): + sql = "insert into db.table_60 values(%d, " + for j in range(500): + str = "'%s', " % random.randint(-2147483647,2147483647) + sql += str + for j in range(500,1000): + str = "'%s', " % random.randint(-32767,32767 ) + sql += str + for j in range(1000,1500): + str = "'%s', " % random.randint(-127,127) + sql += str + for j in range(1500,2000): + str = "'%s', " % random.randint(-922337203685477580700,922337203685477580700) + sql += str + for j in range(2000,2500): + str = "'%s', " % random.randint(-92233720368547758070,92233720368547758070) + sql += str + for j in range(2500,3000): + str = "'%s', " % random.choice(['true','false']) + sql += str + for j in range(3000,3500): + str = "'%s', " % random.randint(-9223372036854775807,9223372036854775807) + sql += str + for j in range(3500,3800): + str = "'%s', " % self.get_random_string(4) + sql += str + for j in range(3800,4090): + str = "'%s', " % self.get_random_string(10) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + time.sleep(1) + tdSql.query("select count(*) from db.table_60") + tdSql.checkData(0, 0, self.num) + tdSql.query("select * from db.table_60") + tdSql.checkRows(self.num) + tdSql.checkCols(4092) + + #insert null value + tdLog.info('test insert null value') + sql = '''create table db.table_61 using db.stable_6 + tags('%s' , '1' , '2' , '3' );'''%self.get_random_string(16370) + tdSql.execute(sql) + + self.ins_query() + + for i in range(self.num): + sql = "insert into db.table_61 values(%d, " + for j in range(2500): + str = "'%s', " % random.choice(['NULL' ,'NULL' ,'NULL' ,1 , 10 ,100 ,-100 ,-10, 88 ,66 ,'NULL' ,'NULL' ,'NULL' ]) + sql += str + for j in range(2500,3000): + str = "'%s', " % random.choice(['true' ,'false']) + sql += str + for j in range(3000,3500): + str = "'%s', " % random.randint(-9223372036854775807,9223372036854775807) + sql += str + for j in range(3500,3800): + str = "'%s', " % self.get_random_string(4) + sql += str + for j in range(3800,4090): + str = "'%s', " % self.get_random_string(10) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + time.sleep(1) + tdSql.query("select count(*) from db.table_61") + tdSql.checkData(0, 0, self.num) + tdSql.query("select * from db.table_61") + tdSql.checkRows(self.num) + tdSql.checkCols(4092) + + #insert in order + tdLog.info('test insert in order') + for i in range(self.num): + sql = "insert into db.table_61 (ts,int_2,int_22,int_169,smallint_537,smallint_607,tinyint_1030,tinyint_1491,double_1629,double_1808,float_2075,col4091) values(%d, " + for j in range(10): + str = "'%s', " % random.randint(0,100) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i + 1000)) + time.sleep(1) + tdSql.query("select count(*) from db.table_61") + tdSql.checkData(0, 0, 2*self.num) + tdSql.query("select * from db.table_61") + tdSql.checkRows(2*self.num) + tdSql.checkCols(4092) + + #insert out of order + tdLog.info('test insert out of order') + for i in range(self.num): + sql = "insert into db.table_61 (ts,int_169,float_2075,int_369,tinyint_1491,tinyint_1030,float_2360,smallint_537,double_1808,double_1608,double_1629,col4091) values(%d, " + for j in range(10): + str = "'%s', " % random.randint(0,100) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i + 2000)) + time.sleep(1) + tdSql.query("select count(*) from db.table_61") + tdSql.checkData(0, 0, 3*self.num) + tdSql.query("select * from db.table_61") + tdSql.checkRows(3*self.num) + tdSql.checkCols(4092) + + self.ins_query() + + #define TSDB_MAX_BYTES_PER_ROW 65531 TSDB_MAX_TAGS_LEN 16384 + #ts:8\int:4\smallint:2\bigint:8\bool:1\float:4\tinyint:1\nchar:4*（）+2[offset]\binary:1*（）+2[offset] + tdLog.info('test super table max bytes per row 65531') + sql = "create table db.stable_7(ts timestamp, " #1*8 sum=8 + for i in range(500): + sql += "int_%d int, " % (i + 1) #500*4=2000 sum=2008 + for i in range(500,1000): + sql += "smallint_%d smallint, " % (i + 1) #500*2=1000 sum=3008 + for i in range(1000,1500): + sql += "tinyint_%d tinyint, " % (i + 1) #500*1=500 sum=3508 + for i in range(1500,2000): + sql += "double_%d double, " % (i + 1) #500*8=4000 sum=7508 + for i in range(2000,2500): + sql += "float_%d float, " % (i + 1) #500*4=2000 sum=9508 + for i in range(2500,3000): + sql += "bool_%d bool, " % (i + 1) #500*1=500 sum=10008 + for i in range(3000,3500): + sql += "bigint_%d bigint, " % (i + 1) #500*8=4000 sum=14008 + for i in range(3500,3800): + sql += "nchar_%d nchar(32), " % (i + 1) #300*(32*4+2)=39000 sum=53008 + for i in range(3800,4090): + sql += "binary_%d binary(40), " % (i + 1) #290*(40+2)=12180 sum=65188 + sql += "col4091 binary(341))" #341+2=343 sum=65531 + sql += " tags (loc binary(16370),tag_1 int,tag_2 int,tag_3 int) " #4*3+16370+2 + tdSql.execute(sql) + sql = '''create table db.table_70 using db.stable_7 + tags('%s' , '1' , '2' , '3' );'''%self.get_random_string(16370) + tdSql.execute(sql) + tdSql.query("select * from db.table_70") + tdSql.checkCols(4092) + tdSql.query("describe db.table_70") + tdSql.checkRows(4096) + + tdLog.info('test super table drop and add column or tag') + sql = "alter stable db.stable_7 drop column col4091; " + tdSql.execute(sql) + sql = "select * from db.stable_7; " + tdSql.query(sql) + tdSql.checkCols(4095) + sql = "alter table db.stable_7 add column col4091 binary(342); " + tdSql.error(sql) + sql = "alter table db.stable_7 add column col4091 binary(341); " + tdSql.execute(sql) + sql = "select * from db.stable_7; " + tdSql.query(sql) + tdSql.checkCols(4096) + + sql = "alter stable db.stable_7 drop tag loc; " + tdSql.execute(sql) + sql = "select * from db.stable_7; " + tdSql.query(sql) + tdSql.checkCols(4095) + sql = "alter table db.stable_7 add tag loc binary(16371); " + tdSql.error(sql) + sql = "alter table db.stable_7 add tag loc binary(16370); " + tdSql.execute(sql) + sql = "select * from db.stable_7; " + tdSql.query(sql) + tdSql.checkCols(4096) + + sql = "alter stable db.stable_7 drop tag tag_1; " + tdSql.execute(sql) + sql = "select * from db.stable_7; " + tdSql.query(sql) + tdSql.checkCols(4095) + sql = "alter table db.stable_7 add tag tag_1 int; " + tdSql.execute(sql) + sql = "select * from db.stable_7; " + tdSql.query(sql) + tdSql.checkCols(4096) + sql = "alter table db.stable_7 add tag loc1 nchar(10); " + tdSql.error(sql) + + self.ins_query() + + tdLog.info('test super table max bytes per row 65531') + sql = "create table db.stable_8(ts timestamp, " + for i in range(500): + sql += "int_%d int, " % (i + 1) + for i in range(500,1000): + sql += "smallint_%d smallint, " % (i + 1) + for i in range(1000,1500): + sql += "tinyint_%d tinyint, " % (i + 1) + for i in range(1500,2000): + sql += "double_%d double, " % (i + 1) + for i in range(2000,2500): + sql += "float_%d float, " % (i + 1) + for i in range(2500,3000): + sql += "bool_%d bool, " % (i + 1) + for i in range(3000,3500): + sql += "bigint_%d bigint, " % (i + 1) + for i in range(3500,3800): + sql += "nchar_%d nchar(32), " % (i + 1) + for i in range(3800,4090): + sql += "binary_%d binary(40), " % (i + 1) + sql += "col4091 binary(342))" + sql += " tags (loc nchar(10),tag_1 int,tag_2 int,tag_3 int) " + tdSql.error(sql) + + tdLog.info('test super table max bytes per row tag 16384') + sql = "create table db.stable_8(ts timestamp, " + for i in range(500): + sql += "int_%d int, " % (i + 1) + for i in range(500,1000): + sql += "smallint_%d smallint, " % (i + 1) + for i in range(1000,1500): + sql += "tinyint_%d tinyint, " % (i + 1) + for i in range(1500,2000): + sql += "double_%d double, " % (i + 1) + for i in range(2000,2500): + sql += "float_%d float, " % (i + 1) + for i in range(2500,3000): + sql += "bool_%d bool, " % (i + 1) + for i in range(3000,3500): + sql += "bigint_%d bigint, " % (i + 1) + for i in range(3500,3800): + sql += "nchar_%d nchar(32), " % (i + 1) + for i in range(3800,4090): + sql += "binary_%d binary(40), " % (i + 1) + sql += "col4091 binary(341))" + sql += " tags (loc binary(16371),tag_1 int,tag_2 int,tag_3 int) " + tdSql.error(sql) + + def run_7(self): + + + print("==============step7, super table error col ==============") + tdLog.info('test exceeds row num') + # column + tag > 4096 + sql = "create stable db.stable_2(ts timestamp, " + for i in range(4091): + sql += "col%d int, " % (i + 1) + sql += "col4092 binary(22))" + sql += " tags (loc nchar(10),tag_1 int,tag_2 int,tag_3 int) " + tdLog.info(len(sql)) + tdSql.error(sql) + + self.ins_query() + + # column + tag > 4096 + sql = "create stable db.stable_2(ts timestamp, " + for i in range(4090): + sql += "col%d int, " % (i + 1) + sql += "col4091 binary(22))" + sql += " tags (loc nchar(10),tag_1 int,tag_2 int,tag_3 int,tag_4 int) " + tdLog.info(len(sql)) + tdSql.error(sql) + + # alter column + tag > 4096 + sql = "alter table db.stable_1 add column max int; " + tdSql.error(sql) + + sql = "alter table db.stable_1 add tag max int; " + tdSql.error(sql) + + sql = "alter table db.stable_4 modify column col4091 binary(102); " + tdSql.error(sql) + sql = "alter table db.stable_4 modify tag loc nchar(20); " + tdSql.query("select * from db.table_70") + tdSql.checkCols(4092) + tdSql.query("describe db.table_70") + tdSql.checkRows(4096) + + self.ins_query() + + + def run(self): + tdSql.prepare() + + startTime_all = time.time() + self.run_8() + self.run_9() + self.run_1() + self.run_2() + # self.run_3() + # self.run_4() + # self.run_5() + # self.run_6() + # self.run_7() + + endTime_all = time.time() + print("total time %ds" % (endTime_all - startTime_all)) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/1-insert/rowlength64k_1.py b/tests/system-test/1-insert/rowlength64k_1.py new file mode 100755 index 0000000000000000000000000000000000000000..b641cfd58a9ff64eb2421dd3d19066e0609ec21b --- /dev/null +++ b/tests/system-test/1-insert/rowlength64k_1.py @@ -0,0 +1,40 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from util.cases import tdCases +from .rowlength64k import * + +class TDTestCase(TDTestCase): + + + def run(self): + tdSql.prepare() + + startTime_all = time.time() + #self.run_1() + # self.run_2() + self.run_3() + #self.run_4() + + endTime_all = time.time() + print("total time %ds" % (endTime_all - startTime_all)) + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/rowlength64k_2.py b/tests/system-test/1-insert/rowlength64k_2.py new file mode 100755 index 0000000000000000000000000000000000000000..897419c242f419909f514ffef245eb026be2869e --- /dev/null +++ b/tests/system-test/1-insert/rowlength64k_2.py @@ -0,0 +1,44 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from util.cases import tdCases +from .rowlength64k import * + +class TDTestCase(TDTestCase): + + + def run(self): + tdSql.prepare() + + startTime_all = time.time() + # self.run_1() + # self.run_2() + # self.run_3() + self.run_4() + #self.run_5() + # self.run_5() + # self.run_6() + # self.run_7() + + endTime_all = time.time() + print("total time %ds" % (endTime_all - startTime_all)) + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/rowlength64k_3.py b/tests/system-test/1-insert/rowlength64k_3.py new file mode 100755 index 0000000000000000000000000000000000000000..ebcc207de605802193404e6247e56b4be3abb45c --- /dev/null +++ b/tests/system-test/1-insert/rowlength64k_3.py @@ -0,0 +1,42 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from util.cases import tdCases +from .rowlength64k import * + +class TDTestCase(TDTestCase): + + + def run(self): + tdSql.prepare() + + startTime_all = time.time() + # self.run_1() + # self.run_2() + # self.run_3() + # self.run_4() + self.run_6() + self.run_7() + + endTime_all = time.time() + print("total time %ds" % (endTime_all - startTime_all)) + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/rowlength64k_4.py b/tests/system-test/1-insert/rowlength64k_4.py new file mode 100755 index 0000000000000000000000000000000000000000..a5abe3e26840d16eb751cbec45d13f056d37ae67 --- /dev/null +++ b/tests/system-test/1-insert/rowlength64k_4.py @@ -0,0 +1,44 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from util.cases import tdCases +from .rowlength64k import * + +class TDTestCase(TDTestCase): + + + def run(self): + tdSql.prepare() + + startTime_all = time.time() + # self.run_1() + # self.run_2() + # self.run_3() + #self.run_4() + self.run_5() + # self.run_5() + # self.run_6() + # self.run_7() + + endTime_all = time.time() + print("total time %ds" % (endTime_all - startTime_all)) + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/rowlength64k_benchmark.py b/tests/system-test/1-insert/rowlength64k_benchmark.py new file mode 100755 index 0000000000000000000000000000000000000000..98a72e1166b7e95e544b5a6b93d09fa126e02665 --- /dev/null +++ b/tests/system-test/1-insert/rowlength64k_benchmark.py @@ -0,0 +1,192 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import os +import time +import taos +import subprocess +import string +from faker import Faker +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +from util.dnodes import * + +class TDTestCase: + updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1} + + def init(self, conn, logSql, replicaVar): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.testcasePath = os.path.split(__file__)[0] + self.testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + + now = time.time() + self.ts = int(round(now * 1000)) + self.num = 100 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + tdSql.prepare() + + startTime_all = time.time() + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + #-N:regular table -d:database name -t:table num -n:rows num per table -l:col num -y:force + #regular old && new + startTime = time.time() + os.system("%staosBenchmark -N -d regular_old -t 1 -n 10 -l 1023 -y" % binPath) + tdSql.execute("use regular_old") + tdSql.query("show tables;") + tdSql.checkRows(1) + tdSql.query("select * from meters;") + tdSql.checkCols(1024) + tdSql.query("describe meters;") + tdSql.checkRows(1024) + + os.system("%staosBenchmark -N -d regular_new -t 1 -n 10 -l 4095 -y" % binPath) + tdSql.execute("use regular_new") + tdSql.query("show tables;") + tdSql.checkRows(1) + tdSql.query("select * from meters;") + tdSql.checkCols(4096) + tdSql.query("describe meters;") + tdSql.checkRows(4096) + + #super table -d:database name -t:table num -n:rows num per table -l:col num -y:force + os.system("%staosBenchmark -d super_old -t 1 -n 10 -l 1021 -y" % binPath) + tdSql.execute("use super_old") + tdSql.query("show tables;") + tdSql.checkRows(1) + tdSql.query("select * from meters;") + tdSql.checkCols(1024) + tdSql.query("select * from d0;") + tdSql.checkCols(1022) + tdSql.query("describe meters;") + tdSql.checkRows(1024) + tdSql.query("describe d0;") + tdSql.checkRows(1024) + + os.system("%staosBenchmark -d super_new -t 1 -n 10 -l 4093 -y" % binPath) + tdSql.execute("use super_new") + tdSql.query("show tables;") + tdSql.checkRows(1) + tdSql.query("select * from meters;") + tdSql.checkCols(4096) + tdSql.query("select * from d0;") + tdSql.checkCols(4094) + tdSql.query("describe meters;") + tdSql.checkRows(4096) + tdSql.query("describe d0;") + tdSql.checkRows(4096) + tdSql.execute("create table stb_new1_1 using meters tags(1,2)") + tdSql.query("select * from stb_new1_1") + tdSql.checkCols(4094) + tdSql.query("describe stb_new1_1;") + tdSql.checkRows(4096) + + # insert: create one or mutiple tables per sql and insert multiple rows per sql + os.system("%staosBenchmark -f %s/rowlength64k.json -y " % (binPath,self.testcasePath)) + tdSql.execute("use json_test") + tdSql.query("select count (tbname) from stb_old") + tdSql.checkData(0, 0, 10) + + tdSql.query("select * from stb_old") + tdSql.checkRows(10) + tdSql.checkCols(1024) + + tdSql.query("select count (tbname) from stb_new") + tdSql.checkData(0, 0, 10) + + tdSql.query("select * from stb_new") + tdSql.checkRows(10) + tdSql.checkCols(4096) + tdSql.query("describe stb_new;") + tdSql.checkRows(4096) + tdSql.query("select * from stb_new_0") + tdSql.checkRows(10) + tdSql.checkCols(4091) + tdSql.query("describe stb_new_0;") + tdSql.checkRows(4096) + tdSql.execute("create table stb_new1_1 using stb_new tags(1,2,3,4,5)") + tdSql.query("select * from stb_new1_1") + tdSql.checkCols(4091) + tdSql.query("describe stb_new1_1;") + tdSql.checkRows(4096) + + tdSql.query("select count (tbname) from stb_mix") + tdSql.checkData(0, 0, 10) + + tdSql.query("select * from stb_mix") + tdSql.checkRows(10) + tdSql.checkCols(4096) + tdSql.query("describe stb_mix;") + tdSql.checkRows(4096) + tdSql.query("select * from stb_mix_0") + tdSql.checkRows(10) + tdSql.checkCols(4092) + tdSql.query("describe stb_mix_0;") + tdSql.checkRows(4096) + + tdSql.query("select count (tbname) from stb_excel") + tdSql.checkData(0, 0, 10) + + tdSql.query("select * from stb_excel") + tdSql.checkRows(10) + tdSql.checkCols(4096) + tdSql.query("describe stb_excel;") + tdSql.checkRows(4096) + tdSql.query("select * from stb_excel_0") + tdSql.checkRows(10) + tdSql.checkCols(4092) + tdSql.query("describe stb_excel_0;") + tdSql.checkRows(4096) + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + endTime_all = time.time() + print("total time %ds" % (endTime_all - startTime_all)) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/1-insert/table_param_ttl.py b/tests/system-test/1-insert/table_param_ttl.py index 51d7d596cd9d94e6078fc7d448583c91a27e1780..6cc978a76ce6f71e2dc0b68fc5561bde7e9a6656 100644 --- a/tests/system-test/1-insert/table_param_ttl.py +++ b/tests/system-test/1-insert/table_param_ttl.py @@ -35,14 +35,14 @@ class TDTestCase: tdSql.execute(f'create table db.{self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.ttl_param}') tdSql.query(f'show db.tables') tdSql.checkRows(self.tbnum) - sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval']) + sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval'] + 1) tdSql.query(f'show db.tables') tdSql.checkRows(0) for i in range(self.tbnum): tdSql.execute(f'create table db.{self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.default_ttl}') for i in range(int(self.tbnum/2)): tdSql.execute(f'alter table db.{self.ntbname}_{i} ttl {self.modify_ttl}') - sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval']) + sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval'] + 1) tdSql.query(f'show db.tables') tdSql.checkRows(self.tbnum - int(self.tbnum/2)) tdSql.execute('drop database db') @@ -54,7 +54,7 @@ class TDTestCase: tdSql.execute(f'create table db.{self.stbname}_{i} using db.{self.stbname} tags({i}) ttl {self.ttl_param}') tdSql.query(f'show db.tables') tdSql.checkRows(self.tbnum) - sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval']) + sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval'] + 1) tdSql.query(f'show db.tables') tdSql.checkRows(0) for i in range(self.tbnum): @@ -63,7 +63,7 @@ class TDTestCase: tdSql.checkRows(self.tbnum) for i in range(int(self.tbnum/2)): tdSql.execute(f'alter table db.{self.stbname}_{i} ttl {self.modify_ttl}') - sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval']) + sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval'] + 1) tdSql.query(f'show db.tables') tdSql.checkRows(self.tbnum - int(self.tbnum/2)) tdSql.execute('drop database db') @@ -75,7 +75,7 @@ class TDTestCase: tdSql.execute(f'insert into db.{self.stbname}_{i} using db.{self.stbname} tags({i}) ttl {self.ttl_param} values(now,1)') tdSql.query(f'show db.tables') tdSql.checkRows(self.tbnum) - sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval']) + sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval'] + 1) tdSql.query(f'show db.tables') tdSql.checkRows(0) tdSql.execute('drop database db') diff --git a/tests/system-test/2-query/Timediff.py b/tests/system-test/2-query/Timediff.py index 4e72c07b303499145c24cec63138367d54750a34..a7366a4007135d528741ced696705925fc69a70f 100644 --- a/tests/system-test/2-query/Timediff.py +++ b/tests/system-test/2-query/Timediff.py @@ -4,6 +4,8 @@ from util.cases import * from util.gettime import * class TDTestCase: + updatecfgDict = {'keepColumnName': 1} + def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") @@ -27,14 +29,14 @@ class TDTestCase: self.ctbname = f'{self.dbname}.ctb' self.subtractor = 1 # unit:s def check_tbtype(self,tb_type): - if tb_type.lower() == 'ntb': + if tb_type.lower() == 'ntb': tdSql.query(f'select timediff(ts,{self.subtractor}) from {self.ntbname}') elif tb_type.lower() == 'ctb': tdSql.query(f'select timediff(ts,{self.subtractor}) from {self.ctbname}') elif tb_type.lower() == 'stb': tdSql.query(f'select timediff(ts,{self.subtractor}) from {self.stbname}') def check_tb_type(self,unit,tb_type): - if tb_type.lower() == 'ntb': + if tb_type.lower() == 'ntb': tdSql.query(f'select timediff(ts,{self.subtractor},{unit}) from {self.ntbname}') elif tb_type.lower() == 'ctb': tdSql.query(f'select timediff(ts,{self.subtractor},{unit}) from {self.ctbname}') @@ -43,7 +45,7 @@ class TDTestCase: def data_check(self,date_time,precision,tb_type): for unit in self.time_unit: if (unit.lower() == '1u' and precision.lower() == 'ms') or (unit.lower() == '1b' and precision.lower() == 'us') or (unit.lower() == '1b' and precision.lower() == 'ms'): - if tb_type.lower() == 'ntb': + if tb_type.lower() == 'ntb': tdSql.error(f'select timediff(ts,{self.subtractor},{unit}) from {self.ntbname}') elif tb_type.lower() == 'ctb': tdSql.error(f'select timediff(ts,{self.subtractor},{unit}) from {self.ctbname}') @@ -66,7 +68,7 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[i][0],int(((date_time[i]/1000)-self.subtractor)/60/60)) elif unit.lower() == '1d': for i in range(len(self.ts_str)): - tdSql.checkEqual(tdSql.queryResult[i][0],int(((date_time[i]/1000)-self.subtractor)/60/60/24)) + tdSql.checkEqual(tdSql.queryResult[i][0],int(((date_time[i]/1000)-self.subtractor)/60/60/24)) elif unit.lower() == '1w': for i in range(len(self.ts_str)): tdSql.checkEqual(tdSql.queryResult[i][0],int(((date_time[i]/1000)-self.subtractor)/60/60/24/7)) @@ -97,7 +99,7 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[i][0],int(((date_time[i]/1000)-self.subtractor*1000))) elif unit.lower() == '1u': for i in range(len(self.ts_str)): - tdSql.checkEqual(tdSql.queryResult[i][0],int(((date_time[i])-self.subtractor*1000000))) + tdSql.checkEqual(tdSql.queryResult[i][0],int(((date_time[i])-self.subtractor*1000000))) self.check_tbtype(tb_type) tdSql.checkRows(len(self.ts_str)) for i in range(len(self.ts_str)): @@ -185,8 +187,16 @@ class TDTestCase: elif precision.lower() == 'ns': for i in range(len(self.ts_str)): tdSql.checkEqual(tdSql.queryResult[i][0],int(((date_time[i])-self.subtractor*1000000000))) - + def function_multi_res_param(self): + tdSql.execute(f'drop database if exists {self.dbname}') + tdSql.execute(f'create database {self.dbname}') + tdSql.execute(f'use {self.dbname}') + tdSql.execute(f'create table {self.ntbname} (ts timestamp,c0 int)') + tdSql.execute(f'insert into {self.ntbname} values("2023-01-01 00:00:00",1)') + tdSql.execute(f'insert into {self.ntbname} values("2023-01-01 00:01:00",2)') + tdSql.query(f'select timediff(last(ts), first(ts)) from {self.ntbname}') + tdSql.checkData(0, 0, 60000) @@ -194,7 +204,8 @@ class TDTestCase: self.function_check_ntb() self.function_check_stb() self.function_without_param() - + self.function_multi_res_param() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/columnLenUpdated.py b/tests/system-test/2-query/columnLenUpdated.py index e43b32a716017702382ce0384bb377d692e64684..4c92236fca00bca845ed01857e591718a3cf0246 100644 --- a/tests/system-test/2-query/columnLenUpdated.py +++ b/tests/system-test/2-query/columnLenUpdated.py @@ -26,7 +26,7 @@ def taos_command (buildPath, key, value, expectString, sqlString=''): taosCmd = buildPath + '/build/bin/taos ' cfgPath = buildPath + "/../sim/psim/cfg" - taosCmd = taosCmd + ' -c' + cfgPath + ' -' + key + taosCmd = taosCmd + ' -c ' + cfgPath + ' -' + key if len(value) != 0: taosCmd = taosCmd + ' ' + value @@ -202,7 +202,7 @@ class TDTestCase: if retCode != "TAOS_OK": tdLog.exit("taos -s fail") - tdSql.query("select count(*) from stb group by tg1") + tdSql.query("select count(*) from stb group by tg1 order by count(*) desc") tdSql.checkData(0, 0, 2) tdSql.checkData(1, 0, 1) diff --git a/tests/system-test/2-query/count_null.py b/tests/system-test/2-query/count_null.py new file mode 100644 index 0000000000000000000000000000000000000000..6d2c8db8d6b4bd33ae75213ab838279ff9e65503 --- /dev/null +++ b/tests/system-test/2-query/count_null.py @@ -0,0 +1,144 @@ +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def check_results(self): + tdSql.query(f"select count(*) from tb1") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c2) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c3) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c4) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c5) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c6) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c7) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c8) from tb1") + tdSql.checkData(0, 0, 0) + + tdSql.query(f"select count(*) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c2) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c3) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c4) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c5) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c6) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c7) from tb2") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c8) from tb2") + tdSql.checkData(0, 0, 0) + + for i in range (3, 6): + tdSql.query(f"select count(*) from tb{i}") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c2) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c3) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c4) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c5) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c6) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c7) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c8) from tb{i}") + tdSql.checkData(0, 0, 10000) + + + def run(self): + dbname = 'db' + tbnames = ['tb1', 'tb2', 'tb3', 'tb4', 'tb5', 'tb6'] + num_rows = 20000 + num_tables = 6 + ts_base = 1685548800000 + + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + + for i in range (num_tables): + tdSql.execute( + f'''create table if not exists {dbname}.{tbnames[i]} + (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10)) + + ''' + ) + + + tdLog.printNoPrefix("==========step2:insert data") + + for i in range(num_rows): + tdSql.execute(f"insert into {dbname}.{tbnames[0]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + + for i in range(num_rows): + tdSql.execute(f"insert into {dbname}.{tbnames[1]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)") + + for i in range(num_rows): + if i % 2 == 0: + tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + for i in range(num_rows): + if i % 2 == 0: + tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, null, null, null, null, null, null, null, 'binary', 'nchar')") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)") + + for i in range(num_rows): + if i < num_rows / 2: + tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + for i in range(num_rows): + if i >= num_rows / 2: + tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + + tdLog.printNoPrefix("==========step3:check result in memory") + self.check_results(); + + tdLog.printNoPrefix("==========step3:check result from disk") + tdSql.execute(f"flush database db") + self.check_results(); + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/diff.py b/tests/system-test/2-query/diff.py index d48a01db6af023966ec9fc8307d93cfab37c4f87..c6f233eefab1f8275357cc07c023c28c2cb377e7 100644 --- a/tests/system-test/2-query/diff.py +++ b/tests/system-test/2-query/diff.py @@ -23,7 +23,7 @@ class TDTestCase: tdSql.execute( f"create table {dbname}.ntb(ts timestamp,c1 int,c2 double,c3 float)") tdSql.execute( - f"insert into {dbname}.ntb values(now,1,1.0,10.5)(now+1s,10,-100.0,5.1)(now+10s,-1,15.1,5.0)") + f"insert into {dbname}.ntb values('2023-01-01 00:00:01',1,1.0,10.5)('2023-01-01 00:00:02',10,-100.0,5.1)('2023-01-01 00:00:03',-1,15.1,5.0)") tdSql.query(f"select diff(c1,0) from {dbname}.ntb") tdSql.checkRows(2) @@ -52,6 +52,95 @@ class TDTestCase: tdSql.checkData(0, 0, None) tdSql.checkData(1, 0, None) + # handle null values + tdSql.execute( + f"create table {dbname}.ntb_null(ts timestamp,c1 int,c2 double,c3 float,c4 bool)") + tdSql.execute(f"insert into {dbname}.ntb_null values(now, 1, 1.0, NULL, NULL)") + tdSql.execute(f"insert into {dbname}.ntb_null values(now, NULL, 2.0, 2.0, NULL)") + tdSql.execute(f"insert into {dbname}.ntb_null values(now, 2, NULL, NULL, false)") + tdSql.execute(f"insert into {dbname}.ntb_null values(now, NULL, 1.0, 1.0, NULL)") + tdSql.execute(f"insert into {dbname}.ntb_null values(now, NULL, 3.0, NULL, true)") + tdSql.execute(f"insert into {dbname}.ntb_null values(now, 3, NULL, 3.0, NULL)") + tdSql.execute(f"insert into {dbname}.ntb_null values(now, 1, NULL, NULL, true)") + + tdSql.query(f"select diff(c1) from {dbname}.ntb_null") + tdSql.checkRows(6) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, -2) + + tdSql.query(f"select diff(c2) from {dbname}.ntb_null") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, -1) + tdSql.checkData(3, 0, 2) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + + tdSql.query(f"select diff(c3) from {dbname}.ntb_null") + tdSql.checkRows(6) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, -1) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, 2) + tdSql.checkData(5, 0, None) + + tdSql.query(f"select diff(c4) from {dbname}.ntb_null") + tdSql.checkRows(6) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, 0) + + tdSql.query(f"select diff(c1),diff(c2),diff(c3),diff(c4) from {dbname}.ntb_null") + tdSql.checkRows(6) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, -2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, None) + tdSql.checkData(2, 1, -1) + tdSql.checkData(3, 1, 2) + tdSql.checkData(4, 1, None) + tdSql.checkData(5, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, -1) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, 2) + tdSql.checkData(5, 2, None) + tdSql.checkData(0, 3, None) + tdSql.checkData(1, 3, None) + tdSql.checkData(2, 3, None) + tdSql.checkData(3, 3, 1) + tdSql.checkData(4, 3, None) + tdSql.checkData(5, 3, 0) + + tdSql.query(f"select diff(c1),diff(c2),diff(c3),diff(c4) from {dbname}.ntb_null where c1 is not null") + tdSql.checkRows(3) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, -2) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(2, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(0, 3, None) + tdSql.checkData(1, 3, None) + tdSql.checkData(2, 3, 1) + tdSql.execute(f'''create table {dbname}.stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute(f"create table {dbname}.stb_1 using {dbname}.stb tags('beijing')") @@ -103,6 +192,9 @@ class TDTestCase: tdSql.error(f"select diff(col1,1.23) from {dbname}.stb_1") tdSql.error(f"select diff(col1,-1) from {dbname}.stb_1") tdSql.query(f"select ts,diff(col1),ts from {dbname}.stb_1") + tdSql.error(f"select diff(col1, 1),diff(col2) from {dbname}.stb_1") + tdSql.error(f"select diff(col1, 1),diff(col2, 0) from {dbname}.stb_1") + tdSql.error(f"select diff(col1, 1),diff(col2, 1) from {dbname}.stb_1") tdSql.query(f"select diff(ts) from {dbname}.stb_1") tdSql.checkRows(10) @@ -141,6 +233,40 @@ class TDTestCase: tdSql.checkRows(19) tdSql.checkData(0,0,None) + # TD-25098 + + tdSql.query(f"select ts, diff(c1) from {dbname}.ntb order by ts") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2023-01-01 00:00:02.000') + tdSql.checkData(1, 0, '2023-01-01 00:00:03.000') + + tdSql.checkData(0, 1, 9) + tdSql.checkData(1, 1, -11) + + tdSql.query(f"select ts, diff(c1) from {dbname}.ntb order by ts desc") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2023-01-01 00:00:03.000') + tdSql.checkData(1, 0, '2023-01-01 00:00:02.000') + + tdSql.checkData(0, 1, -11) + tdSql.checkData(1, 1, 9) + + tdSql.query(f"select ts, diff(c1) from (select * from {dbname}.ntb order by ts)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2023-01-01 00:00:02.000') + tdSql.checkData(1, 0, '2023-01-01 00:00:03.000') + + tdSql.checkData(0, 1, 9) + tdSql.checkData(1, 1, -11) + + tdSql.query(f"select ts, diff(c1) from (select * from {dbname}.ntb order by ts desc)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2023-01-01 00:00:02.000') + tdSql.checkData(1, 0, '2023-01-01 00:00:01.000') + + tdSql.checkData(0, 1, 11) + tdSql.checkData(1, 1, -9) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/function_diff.py b/tests/system-test/2-query/function_diff.py index c3f3789a69752b204393f6355d73b480cde90e3a..493e59265e1f64398b1a84d45a39ffde7f83e7e7 100644 --- a/tests/system-test/2-query/function_diff.py +++ b/tests/system-test/2-query/function_diff.py @@ -127,22 +127,33 @@ class TDTestCase: return else: - tdSql.query(f"select {col} from {table_expr} {re.sub('limit [0-9]*|offset [0-9]*','',condition)}") + sql = f"select {col} from {table_expr} {re.sub('limit [0-9]*|offset [0-9]*','',condition)}" + tdSql.query(sql) offset_val = condition.split("offset")[1].split(" ")[1] if "offset" in condition else 0 pre_result = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] if (platform.system().lower() == 'windows' and pre_result.dtype == 'int32'): pre_result = np.array(pre_result, dtype = 'int64') pre_diff = np.diff(pre_result)[offset_val:] - tdSql.query(self.diff_query_form( - col=col, alias=alias, table_expr=table_expr, condition=condition - )) - - for i in range(tdSql.queryRows): - print(f"case in {line}: ", end='') - if isinstance(pre_diff[i] , float ): - pass - else: - tdSql.checkData(i, 0, pre_diff[i]) + if len(pre_diff) > 0: + sql =self.diff_query_form(col=col, alias=alias, table_expr=table_expr, condition=condition) + tdSql.query(sql) + j = 0 + diff_cnt = len(pre_diff) + for i in range(tdSql.queryRows): + print(f"case in {line}: i={i} j={j} pre_diff[j]={pre_diff[j]} ", end='') + if isinstance(pre_diff[j] , float ): + if j + 1 < diff_cnt: + j += 1 + pass + else: + if tdSql.getData(i,0) != None: + tdSql.checkData(i, 0, pre_diff[j]) + if j + 1 < diff_cnt: + j += 1 + else: + print(f"getData i={i} is None j={j} ") + else: + print("pre_diff len is zero.") pass @@ -354,31 +365,31 @@ class TDTestCase: tdSql.checkRows(229) tdSql.checkData(0,0,0) tdSql.query("select diff(c1) from db.stb1 partition by tbname ") - tdSql.checkRows(190) + tdSql.checkRows(220) tdSql.query("select diff(st1+c1) from db.stb1 partition by tbname") - tdSql.checkRows(190) + tdSql.checkRows(220) tdSql.query("select diff(st1+c1) from db.stb1 partition by tbname") - tdSql.checkRows(190) + tdSql.checkRows(220) tdSql.query("select diff(st1+c1) from db.stb1 partition by tbname") - tdSql.checkRows(190) + tdSql.checkRows(220) # bug need fix tdSql.query("select diff(st1+c1) from db.stb1 partition by tbname") - tdSql.checkRows(190) + tdSql.checkRows(220) # bug need fix tdSql.query("select tbname , diff(c1) from db.stb1 partition by tbname") - tdSql.checkRows(190) + tdSql.checkRows(220) tdSql.query("select tbname , diff(st1) from db.stb1 partition by tbname") tdSql.checkRows(220) # partition by tags tdSql.query("select st1 , diff(c1) from db.stb1 partition by st1") - tdSql.checkRows(190) + tdSql.checkRows(220) tdSql.query("select diff(c1) from db.stb1 partition by st1") - tdSql.checkRows(190) + tdSql.checkRows(220) def diff_test_run(self) : diff --git a/tests/system-test/2-query/geometry.py b/tests/system-test/2-query/geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..e430b62fb2150f707457bca0bb7ea9b772957639 --- /dev/null +++ b/tests/system-test/2-query/geometry.py @@ -0,0 +1,257 @@ +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + # WKT strings to be input as GEOMETRY type + self.point = "POINT (3.000000 6.000000)" + self.lineString = "LINESTRING (1.000000 1.000000, 2.000000 2.000000, 5.000000 5.000000)" + self.polygon = "POLYGON ((3.000000 6.000000, 5.000000 6.000000, 5.000000 8.000000, 3.000000 8.000000, 3.000000 6.000000))" + + # expected errno + self.errno_TSC_SQL_SYNTAX_ERROR = -2147483114; + self.errno_PAR_SYNTAX_ERROR = -2147473920 + + self.errno_FUNTION_PARA_NUM = -2147473407; + self.errno_FUNTION_PARA_TYPE = -2147473406; + self.errno_FUNTION_PARA_VALUE = -2147473405; + + def prepare_data(self, dbname = "db"): + tdSql.execute( + f'''create table {dbname}.t1 + (ts timestamp, c1 int, c2 float, c3 varchar(256), c4 geometry(512)) + ''' + ) + tdSql.execute( + f'''create table {dbname}.stb1 + (ts timestamp, c1 int, c2 float, c3 varchar(256), c4 geometry(512)) tags (t1 int) + ''' + ) + for i in range(2): + tdSql.execute(f'create table {dbname}.ct{i+1} using {dbname}.stb1 tags ( {i+1} )') + + values = f''' + (now()-1s, 1, 1.1, '{self.point}', '{self.point}') + (now(), 2, 2.2, '{self.lineString}', '{self.lineString}') + (now()+1s, 3, 3.3, '{self.polygon}', '{self.polygon}') + (now()+2s, 4, 4.4, NULL, NULL) + ''' + tdSql.execute(f"insert into {dbname}.t1 values{values}") + tdSql.execute(f"insert into {dbname}.ct1 values{values}") + tdSql.execute(f"insert into {dbname}.ct2 values{values}") + + # the following errors would happen when casting a string to GEOMETRY by ST_GeomFromText(), but raise an error as syntax error + # wrong WTK + tdSql.error(f"insert into {dbname}.ct2 values(now(), 1, 1.1, NULL, 'POIN(1.0 1.5)')", self.errno_TSC_SQL_SYNTAX_ERROR) + # wrong WTK at all + tdSql.error(f"insert into {dbname}.ct2 values(now(), 1, 1.1, NULL, 'XXX')", self.errno_TSC_SQL_SYNTAX_ERROR) + # empty WTK + tdSql.error(f"insert into {dbname}.ct2 values(now(), 1, 1.1, NULL, '')", self.errno_TSC_SQL_SYNTAX_ERROR) + # wrong type + tdSql.error(f"insert into {dbname}.ct2 values(now(), 1, 1.1, NULL, 2)", self.errno_TSC_SQL_SYNTAX_ERROR) + + def geomFromText_test(self, dbname = "db"): + # [ToDo] remove ST_AsText() calling in geomFromText_test once GEOMETRY type is supported in taos-connector-python + + # column input, including NULL value + tdSql.query(f"select ST_AsText(ST_GeomFromText(c3)), ST_AsText(c4) from {dbname}.t1") + for i in range(tdSql.queryRows): + tdSql.checkEqual(tdSql.queryResult[i][0], tdSql.queryResult[i][1]) + + # constant input + tdSql.query(f"select ST_AsText(ST_GeomFromText('{self.point}'))") + tdSql.checkEqual(tdSql.queryResult[0][0], self.point) + + # empty input + tdSql.query(f"select ST_AsText(ST_GeomFromText(''))") + tdSql.checkEqual(tdSql.queryResult[0][0], None) + + # NULL input + tdSql.query(f"select ST_AsText(ST_GeomFromText(NULL))") + tdSql.checkEqual(tdSql.queryResult[0][0], None) + + # wrong type input + tdSql.error(f"select ST_GeomFromText(2)", self.errno_FUNTION_PARA_TYPE) + tdSql.error(f"select ST_GeomFromText(c1) from {dbname}.t1", self.errno_FUNTION_PARA_TYPE) + + # wrong number of params input + tdSql.error(f"select ST_GeomFromText()", self.errno_PAR_SYNTAX_ERROR) + tdSql.error(f"select ST_GeomFromText(c3, c3) from {dbname}.t1", self.errno_FUNTION_PARA_NUM) + + # wrong param content input + # lack of the last letter of 'POINT' + tdSql.error(f"select ST_GeomFromText('POIN(1.0 1.5)')", self.errno_FUNTION_PARA_VALUE) + # redundant comma at the end + tdSql.error(f"select ST_GeomFromText('LINESTRING(1.0 1.0, 2.0 2.0, 5.0 5.0,)')", self.errno_FUNTION_PARA_VALUE) + # the first point and last one are not same + tdSql.error(f"select ST_GeomFromText('POLYGON((3.0 6.0, 5.0 6.0, 5.0 8.0, 3.0 8.0))')", self.errno_FUNTION_PARA_VALUE) + # wrong WTK at all + tdSql.error(f"select ST_GeomFromText('XXX')", self.errno_FUNTION_PARA_VALUE) + + def asText_test(self, dbname = "db"): + # column input, including NULL value + tdSql.query(f"select c3, ST_AsText(c4) from {dbname}.ct1") + for i in range(tdSql.queryRows): + tdSql.checkEqual(tdSql.queryResult[i][0], tdSql.queryResult[i][1]) + + # constant input + tdSql.query(f"select ST_AsText(c4) from {dbname}.ct1 where c1 = 1") + tdSql.checkEqual(tdSql.queryResult[0][0], self.point) + + # empty input should NOT happen for GEOMETRY type + + # NULL input + tdSql.query(f"select ST_AsText(NULL)") + tdSql.checkEqual(tdSql.queryResult[0][0], None) + + # wrong type input + tdSql.error(f"select ST_AsText('XXX')", self.errno_FUNTION_PARA_TYPE) + tdSql.error(f"select ST_AsText(c2) from {dbname}.ct1", self.errno_FUNTION_PARA_TYPE) + + # wrong number of params input + tdSql.error(f"select ST_AsText() from {dbname}.ct1", self.errno_PAR_SYNTAX_ERROR) + tdSql.error(f"select ST_AsText(c4, c4) from {dbname}.ct1", self.errno_FUNTION_PARA_NUM) + + # wrong param content input should NOT happen for GEOMETRY type + + def geomRelationFunc_test(self, geomRelationFuncName, expectedResults, dbname = "db"): + # two columns input, including NULL value + tdSql.query(f"select {geomRelationFuncName}(ST_GeomFromText(c3), c4) from {dbname}.t1") + for i in range(tdSql.queryRows): + tdSql.checkData(i, 0, expectedResults[0][i]) + + # constant and column input + tdSql.query(f"select {geomRelationFuncName}(ST_GeomFromText('{self.point}'), c4) from {dbname}.t1") + for i in range(tdSql.queryRows): + tdSql.checkData(i, 0, expectedResults[1][i]) + + # column and constant input + tdSql.query(f"select {geomRelationFuncName}(c4, ST_GeomFromText('{self.point}')) from {dbname}.t1") + for i in range(tdSql.queryRows): + tdSql.checkData(i, 0, expectedResults[2][i]) + + # two constants input + tdSql.query(f"select {geomRelationFuncName}(ST_GeomFromText('{self.point}'), ST_GeomFromText('{self.lineString}'))") + tdSql.checkEqual(tdSql.queryResult[0][0], expectedResults[3]) + + tdSql.query(f"select {geomRelationFuncName}(ST_GeomFromText('{self.polygon}'), ST_GeomFromText('{self.point}'))") + tdSql.checkEqual(tdSql.queryResult[0][0], expectedResults[4]) + + # NULL type input + tdSql.query(f"select {geomRelationFuncName}(NULL, ST_GeomFromText('{self.point}'))") + tdSql.checkEqual(tdSql.queryResult[0][0], None) + + tdSql.query(f"select {geomRelationFuncName}(ST_GeomFromText('{self.lineString}'), NULL)") + tdSql.checkEqual(tdSql.queryResult[0][0], None) + + tdSql.query(f"select {geomRelationFuncName}(NULL, NULL)") + tdSql.checkEqual(tdSql.queryResult[0][0], None) + + # wrong type input + tdSql.error(f"select {geomRelationFuncName}(c1, c4) from {dbname}.t1", self.errno_FUNTION_PARA_TYPE) + tdSql.error(f"select {geomRelationFuncName}(c4, c2) from {dbname}.t1", self.errno_FUNTION_PARA_TYPE) + tdSql.error(f"select {geomRelationFuncName}(c4, 'XXX') from {dbname}.t1", self.errno_FUNTION_PARA_TYPE) + + # wrong number of params input + tdSql.error(f"select {geomRelationFuncName}(c4) from {dbname}.t1", self.errno_FUNTION_PARA_NUM) + tdSql.error(f"select {geomRelationFuncName}(ST_GeomFromText(c3), c4, c4) from {dbname}.t1", self.errno_FUNTION_PARA_NUM) + + # used in where clause + tdSql.query(f"select c3 from {dbname}.t1 where {geomRelationFuncName}(ST_GeomFromText('{self.point}'), c4)=true") + tdSql.checkEqual(tdSql.queryRows, expectedResults[5][0]) + for i in range(tdSql.queryRows): + tdSql.checkData(i, 0, expectedResults[5][i+1]) + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1: create tables and insert data") + self.prepare_data() + + tdLog.printNoPrefix("==========step2: ST_GeomFromText function test") + self.geomFromText_test() + + tdLog.printNoPrefix("==========step3: ST_AsText function test") + self.asText_test() + + tdLog.printNoPrefix("==========step4: ST_Intersects function test") + expectedResults = [ + [True, True, True, None], # two columns + [True, False, True, None], # constant and column + [True, False, True, None], # column and constant + False, # two constants 1 + True, # two constants 2 + [2, self.point, self.polygon] # in where clause + ] + self.geomRelationFunc_test('ST_Intersects', expectedResults) + + tdLog.printNoPrefix("==========step5: ST_Equals function test") + expectedResults = [ + [True, True, True, None], # two columns + [True, False, False, None], # constant and column + [True, False, False, None], # column and constant + False, # two constants 1 + False, # two constants 2 + [1, self.point] # in where clause + ] + self.geomRelationFunc_test('ST_Equals', expectedResults) + + tdLog.printNoPrefix("==========step6: ST_Touches function test") + expectedResults = [ + [False, False, False, None], # two columns + [False, False, True, None], # constant and column + [False, False, True, None], # column and constant + False, # two constants 1 + True, # two constants 2 + [1, self.polygon] # in where clause + ] + self.geomRelationFunc_test('ST_Touches', expectedResults) + + tdLog.printNoPrefix("==========step7: ST_Covers function test") + expectedResults = [ + [True, True, True, None], # two columns + [True, False, False, None], # constant and column + [True, False, True, None], # column and constant + False, # two constants 1 + True, # two constants 2 + [1, self.point] # in where clause + ] + self.geomRelationFunc_test('ST_Covers', expectedResults) + + tdLog.printNoPrefix("==========step8: ST_Contains function test") + expectedResults = [ + [True, True, True, None], # two columns + [True, False, False, None], # constant and column + [True, False, False, None], # column and constant + False, # two constants 1 + False, # two constants 2 + [1, self.point] # in where clause + ] + self.geomRelationFunc_test('ST_Contains', expectedResults) + + tdLog.printNoPrefix("==========step9: ST_ContainsProperly function test") + expectedResults = [ + [True, False, False, None], # two columns + [True, False, False, None], # constant and column + [True, False, False, None], # column and constant + False, # two constants 1 + False, # two constants 2 + [1, self.point] # in where clause + ] + self.geomRelationFunc_test('ST_ContainsProperly', expectedResults) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index cdbfa0de84ad51dbbbf5e9146ce43797cf13e5b8..986c63839b1c6e9cfc8c8c6b857a668146d03237 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -26,21 +26,35 @@ class TDTestCase: ctbname3 = "ctb3" num_of_ctables = 3 + tbname_null = "tb_null" + ctbname1_null = "ctb1_null" + ctbname2_null = "ctb2_null" + ctbname3_null = "ctb3_null" + stbname_null = "stb_null" + + tbname_single = "tb_single" + ctbname1_single = "ctb1_single" + ctbname2_single = "ctb2_single" + ctbname3_single = "ctb3_single" + stbname_single = "stb_single" + tdSql.prepare() tdLog.printNoPrefix("==========step1:create table") tdSql.execute( f'''create table if not exists {dbname}.{tbname} - (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10)) + (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10), c9 tinyint unsigned, c10 smallint unsigned, c11 int unsigned, c12 bigint unsigned) ''' ) tdLog.printNoPrefix("==========step2:insert data") - tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:05', 5, 5, 5, 5, 5.0, 5.0, true, 'varchar', 'nchar')") - tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:10', 10, 10, 10, 10, 10.0, 10.0, true, 'varchar', 'nchar')") - tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:15', 15, 15, 15, 15, 15.0, 15.0, true, 'varchar', 'nchar')") + tdSql.execute(f"use db") + + tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:05', 5, 5, 5, 5, 5.0, 5.0, true, 'varchar', 'nchar', 5, 5, 5, 5)") + tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:10', 10, 10, 10, 10, 10.0, 10.0, true, 'varchar', 'nchar', 10, 10, 10, 10)") + tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:15', 15, 15, 15, 15, 15.0, 15.0, true, 'varchar', 'nchar', 15, 15, 15, 15)") tdLog.printNoPrefix("==========step3:fill null") @@ -115,21 +129,266 @@ class TDTestCase: tdLog.printNoPrefix("==========step4:fill value") ## {. . .} - tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") + col_list = {'c0', 'c1', 'c2', 'c3', 'c9', 'c10', 'c11', 'c12'} + for col in col_list: + tdSql.query(f"select interp({col}) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 1) + tdSql.checkData(9, 0, 1) + tdSql.checkData(10, 0, 1) + tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, 1) + + for col in col_list: + tdSql.query(f"select interp({col}) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1.0)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 1) + tdSql.checkData(9, 0, 1) + tdSql.checkData(10, 0, 1) + tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, 1) + + for col in col_list: + tdSql.query(f"select interp({col}) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, true)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 1) + tdSql.checkData(9, 0, 1) + tdSql.checkData(10, 0, 1) + tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, 1) + + for col in col_list: + tdSql.query(f"select interp({col}) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, NULL)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, None) + + tdSql.query(f"select interp(c4) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") tdSql.checkRows(13) - tdSql.checkData(0, 0, 1) - tdSql.checkData(1, 0, 5) - tdSql.checkData(2, 0, 1) - tdSql.checkData(3, 0, 1) - tdSql.checkData(4, 0, 1) - tdSql.checkData(5, 0, 1) - tdSql.checkData(6, 0, 10) - tdSql.checkData(7, 0, 1) - tdSql.checkData(8, 0, 1) - tdSql.checkData(9, 0, 1) - tdSql.checkData(10, 0, 1) - tdSql.checkData(11, 0, 15) - tdSql.checkData(12, 0, 1) + tdSql.checkData(0, 0, 1.0) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, 1.0) + tdSql.checkData(3, 0, 1.0) + tdSql.checkData(4, 0, 1.0) + tdSql.checkData(5, 0, 1.0) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, 1.0) + tdSql.checkData(8, 0, 1.0) + tdSql.checkData(9, 0, 1.0) + tdSql.checkData(10, 0, 1.0) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, 1.0) + + tdSql.query(f"select interp(c4) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1.0)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1.0) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, 1.0) + tdSql.checkData(3, 0, 1.0) + tdSql.checkData(4, 0, 1.0) + tdSql.checkData(5, 0, 1.0) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, 1.0) + tdSql.checkData(8, 0, 1.0) + tdSql.checkData(9, 0, 1.0) + tdSql.checkData(10, 0, 1.0) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, 1.0) + + tdSql.query(f"select interp(c4) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, true)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1.0) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, 1.0) + tdSql.checkData(3, 0, 1.0) + tdSql.checkData(4, 0, 1.0) + tdSql.checkData(5, 0, 1.0) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, 1.0) + tdSql.checkData(8, 0, 1.0) + tdSql.checkData(9, 0, 1.0) + tdSql.checkData(10, 0, 1.0) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, 1.0) + + tdSql.query(f"select interp(c4) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, NULL)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, None) + + tdSql.query(f"select interp(c5) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1.0) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, 1.0) + tdSql.checkData(3, 0, 1.0) + tdSql.checkData(4, 0, 1.0) + tdSql.checkData(5, 0, 1.0) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, 1.0) + tdSql.checkData(8, 0, 1.0) + tdSql.checkData(9, 0, 1.0) + tdSql.checkData(10, 0, 1.0) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, 1.0) + + tdSql.query(f"select interp(c5) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1.0)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1.0) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, 1.0) + tdSql.checkData(3, 0, 1.0) + tdSql.checkData(4, 0, 1.0) + tdSql.checkData(5, 0, 1.0) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, 1.0) + tdSql.checkData(8, 0, 1.0) + tdSql.checkData(9, 0, 1.0) + tdSql.checkData(10, 0, 1.0) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, 1.0) + + tdSql.query(f"select interp(c5) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, true)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1.0) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, 1.0) + tdSql.checkData(3, 0, 1.0) + tdSql.checkData(4, 0, 1.0) + tdSql.checkData(5, 0, 1.0) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, 1.0) + tdSql.checkData(8, 0, 1.0) + tdSql.checkData(9, 0, 1.0) + tdSql.checkData(10, 0, 1.0) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, 1.0) + + tdSql.query(f"select interp(c5) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, NULL)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, None) + + tdSql.query(f"select interp(c6) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, True) + tdSql.checkData(1, 0, True) + tdSql.checkData(2, 0, True) + tdSql.checkData(3, 0, True) + tdSql.checkData(4, 0, True) + tdSql.checkData(5, 0, True) + tdSql.checkData(6, 0, True) + tdSql.checkData(7, 0, True) + tdSql.checkData(8, 0, True) + tdSql.checkData(9, 0, True) + tdSql.checkData(10, 0, True) + tdSql.checkData(11, 0, True) + tdSql.checkData(12, 0, True) + + tdSql.query(f"select interp(c6) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1.0)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, True) + tdSql.checkData(1, 0, True) + tdSql.checkData(2, 0, True) + tdSql.checkData(3, 0, True) + tdSql.checkData(4, 0, True) + tdSql.checkData(5, 0, True) + tdSql.checkData(6, 0, True) + tdSql.checkData(7, 0, True) + tdSql.checkData(8, 0, True) + tdSql.checkData(9, 0, True) + tdSql.checkData(10, 0, True) + tdSql.checkData(11, 0, True) + tdSql.checkData(12, 0, True) + + tdSql.query(f"select interp(c6) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, true)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, True) + tdSql.checkData(1, 0, True) + tdSql.checkData(2, 0, True) + tdSql.checkData(3, 0, True) + tdSql.checkData(4, 0, True) + tdSql.checkData(5, 0, True) + tdSql.checkData(6, 0, True) + tdSql.checkData(7, 0, True) + tdSql.checkData(8, 0, True) + tdSql.checkData(9, 0, True) + tdSql.checkData(10, 0, True) + tdSql.checkData(11, 0, True) + tdSql.checkData(12, 0, True) + + tdSql.query(f"select interp(c6) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, NULL)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, True) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, True) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, True) + tdSql.checkData(12, 0, None) ## {} ... tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(value, 1)") @@ -182,6 +441,130 @@ class TDTestCase: tdSql.checkData(2, 0, 1) tdSql.checkData(3, 0, 1) + ## test fill value with string + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 'abc')") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, 0) + tdSql.checkData(2, 0, 0) + tdSql.checkData(3, 0, 0) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, '123')") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 123) + tdSql.checkData(1, 0, 123) + tdSql.checkData(2, 0, 123) + tdSql.checkData(3, 0, 123) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, '123.123')") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 123) + tdSql.checkData(1, 0, 123) + tdSql.checkData(2, 0, 123) + tdSql.checkData(3, 0, 123) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, '12abc')") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 12) + tdSql.checkData(1, 0, 12) + tdSql.checkData(2, 0, 12) + tdSql.checkData(3, 0, 12) + + ## test fill value with scalar expression + # data types + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1 + 2)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c1) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1 + 2)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c2) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1 + 2)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c3) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1 + 2)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c4) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1 + 2)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3.0) + tdSql.checkData(1, 0, 3.0) + tdSql.checkData(2, 0, 3.0) + tdSql.checkData(3, 0, 3.0) + + tdSql.query(f"select interp(c5) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1 + 2)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3.0) + tdSql.checkData(1, 0, 3.0) + tdSql.checkData(2, 0, 3.0) + tdSql.checkData(3, 0, 3.0) + + tdSql.query(f"select interp(c6) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1 + 2)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, True) + tdSql.checkData(1, 0, True) + tdSql.checkData(2, 0, True) + tdSql.checkData(3, 0, True) + + # expr types + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1.0 + 2.0)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1 + 2.5)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1 + '2')") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1 + '2.0')") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, '3' + 'abc')") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, '2' + '1abc')") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 3) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 3) + + tdLog.printNoPrefix("==========step5:fill prev") ## {. . .} @@ -1722,47 +2105,10 @@ class TDTestCase: tdSql.checkData(60, 1, 60) # # test fill value - tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(value, 123)") + tdSql.query(f"select _irowts, interp(c0), _irowts, interp(c1), _irowts from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(value, 123, 456)") tdSql.checkRows(61) - tdSql.checkCols(2) - tdSql.checkData(0, 0, 0) # - tdSql.checkData(1, 0, 123) - tdSql.checkData(4, 0, 123) - tdSql.checkData(5, 0, None) # - tdSql.checkData(6, 0, 123) - tdSql.checkData(9, 0, 123) - tdSql.checkData(10, 0, 10) # - tdSql.checkData(11, 0, 123) - tdSql.checkData(14, 0, 123) - tdSql.checkData(15, 0, None) # - tdSql.checkData(16, 0, 123) - tdSql.checkData(19, 0, 123) - tdSql.checkData(20, 0, 20) # - tdSql.checkData(21, 0, 123) - tdSql.checkData(24, 0, 123) - tdSql.checkData(25, 0, None) # - tdSql.checkData(26, 0, 123) - tdSql.checkData(29, 0, 123) - tdSql.checkData(30, 0, 30) # - tdSql.checkData(31, 0, 123) - tdSql.checkData(34, 0, 123) - tdSql.checkData(35, 0, 35) # - tdSql.checkData(36, 0, 123) - tdSql.checkData(39, 0, 123) - tdSql.checkData(40, 0, 40) # - tdSql.checkData(41, 0, 123) - tdSql.checkData(44, 0, 123) - tdSql.checkData(45, 0, None) # - tdSql.checkData(46, 0, 123) - tdSql.checkData(49, 0, 123) - tdSql.checkData(50, 0, 50) # - tdSql.checkData(51, 0, 123) - tdSql.checkData(54, 0, 123) - tdSql.checkData(55, 0, None) # - tdSql.checkData(59, 0, 123) - tdSql.checkData(60, 0, 55) # - - tdSql.checkData(0, 1, None) # + tdSql.checkCols(5) + tdSql.checkData(0, 1, 0) # tdSql.checkData(1, 1, 123) tdSql.checkData(4, 1, 123) tdSql.checkData(5, 1, None) # @@ -1774,7 +2120,7 @@ class TDTestCase: tdSql.checkData(15, 1, None) # tdSql.checkData(16, 1, 123) tdSql.checkData(19, 1, 123) - tdSql.checkData(20, 1, None) # + tdSql.checkData(20, 1, 20) # tdSql.checkData(21, 1, 123) tdSql.checkData(24, 1, 123) tdSql.checkData(25, 1, None) # @@ -1783,22 +2129,137 @@ class TDTestCase: tdSql.checkData(30, 1, 30) # tdSql.checkData(31, 1, 123) tdSql.checkData(34, 1, 123) - tdSql.checkData(35, 1, None) # + tdSql.checkData(35, 1, 35) # tdSql.checkData(36, 1, 123) tdSql.checkData(39, 1, 123) tdSql.checkData(40, 1, 40) # tdSql.checkData(41, 1, 123) tdSql.checkData(44, 1, 123) - tdSql.checkData(45, 1, 45) # + tdSql.checkData(45, 1, None) # tdSql.checkData(46, 1, 123) tdSql.checkData(49, 1, 123) - tdSql.checkData(50, 1, None) # + tdSql.checkData(50, 1, 50) # tdSql.checkData(51, 1, 123) tdSql.checkData(54, 1, 123) tdSql.checkData(55, 1, None) # - tdSql.checkData(56, 1, 123) tdSql.checkData(59, 1, 123) - tdSql.checkData(60, 1, 60) # + tdSql.checkData(60, 1, 55) # + + tdSql.checkData(0, 3, None) # + tdSql.checkData(1, 3, 456) + tdSql.checkData(4, 3, 456) + tdSql.checkData(5, 3, None) # + tdSql.checkData(6, 3, 456) + tdSql.checkData(9, 3, 456) + tdSql.checkData(10, 3, 10) # + tdSql.checkData(11, 3, 456) + tdSql.checkData(14, 3, 456) + tdSql.checkData(15, 3, None) # + tdSql.checkData(16, 3, 456) + tdSql.checkData(19, 3, 456) + tdSql.checkData(20, 3, None) # + tdSql.checkData(21, 3, 456) + tdSql.checkData(24, 3, 456) + tdSql.checkData(25, 3, None) # + tdSql.checkData(26, 3, 456) + tdSql.checkData(29, 3, 456) + tdSql.checkData(30, 3, 30) # + tdSql.checkData(31, 3, 456) + tdSql.checkData(34, 3, 456) + tdSql.checkData(35, 3, None) # + tdSql.checkData(36, 3, 456) + tdSql.checkData(39, 3, 456) + tdSql.checkData(40, 3, 40) # + tdSql.checkData(41, 3, 456) + tdSql.checkData(44, 3, 456) + tdSql.checkData(45, 3, 45) # + tdSql.checkData(46, 3, 456) + tdSql.checkData(49, 3, 456) + tdSql.checkData(50, 3, None) # + tdSql.checkData(51, 3, 456) + tdSql.checkData(54, 3, 456) + tdSql.checkData(55, 3, None) # + tdSql.checkData(56, 3, 456) + tdSql.checkData(59, 3, 456) + tdSql.checkData(60, 3, 60) # + + tdSql.query(f"select _isfilled, interp(c0), _isfilled, interp(c1), _isfilled from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(value, 123 + 123, 234 + 234)") + tdSql.checkRows(61) + tdSql.checkCols(5) + tdSql.checkData(0, 1, 0) # + tdSql.checkData(1, 1, 246) + tdSql.checkData(4, 1, 246) + tdSql.checkData(5, 1, None) # + tdSql.checkData(6, 1, 246) + tdSql.checkData(9, 1, 246) + tdSql.checkData(10, 1, 10) # + tdSql.checkData(11, 1, 246) + tdSql.checkData(14, 1, 246) + tdSql.checkData(15, 1, None) # + tdSql.checkData(16, 1, 246) + tdSql.checkData(19, 1, 246) + tdSql.checkData(20, 1, 20) # + tdSql.checkData(21, 1, 246) + tdSql.checkData(24, 1, 246) + tdSql.checkData(25, 1, None) # + tdSql.checkData(26, 1, 246) + tdSql.checkData(29, 1, 246) + tdSql.checkData(30, 1, 30) # + tdSql.checkData(31, 1, 246) + tdSql.checkData(34, 1, 246) + tdSql.checkData(35, 1, 35) # + tdSql.checkData(36, 1, 246) + tdSql.checkData(39, 1, 246) + tdSql.checkData(40, 1, 40) # + tdSql.checkData(41, 1, 246) + tdSql.checkData(44, 1, 246) + tdSql.checkData(45, 1, None) # + tdSql.checkData(46, 1, 246) + tdSql.checkData(49, 1, 246) + tdSql.checkData(50, 1, 50) # + tdSql.checkData(51, 1, 246) + tdSql.checkData(54, 1, 246) + tdSql.checkData(55, 1, None) # + tdSql.checkData(59, 1, 246) + tdSql.checkData(60, 1, 55) # + + tdSql.checkData(0, 3, None) # + tdSql.checkData(1, 3, 468) + tdSql.checkData(4, 3, 468) + tdSql.checkData(5, 3, None) # + tdSql.checkData(6, 3, 468) + tdSql.checkData(9, 3, 468) + tdSql.checkData(10, 3, 10) # + tdSql.checkData(11, 3, 468) + tdSql.checkData(14, 3, 468) + tdSql.checkData(15, 3, None) # + tdSql.checkData(16, 3, 468) + tdSql.checkData(19, 3, 468) + tdSql.checkData(20, 3, None) # + tdSql.checkData(21, 3, 468) + tdSql.checkData(24, 3, 468) + tdSql.checkData(25, 3, None) # + tdSql.checkData(26, 3, 468) + tdSql.checkData(29, 3, 468) + tdSql.checkData(30, 3, 30) # + tdSql.checkData(31, 3, 468) + tdSql.checkData(34, 3, 468) + tdSql.checkData(35, 3, None) # + tdSql.checkData(36, 3, 468) + tdSql.checkData(39, 3, 468) + tdSql.checkData(40, 3, 40) # + tdSql.checkData(41, 3, 468) + tdSql.checkData(44, 3, 468) + tdSql.checkData(45, 3, 45) # + tdSql.checkData(46, 3, 468) + tdSql.checkData(49, 3, 468) + tdSql.checkData(50, 3, None) # + tdSql.checkData(51, 3, 468) + tdSql.checkData(54, 3, 468) + tdSql.checkData(55, 3, None) # + tdSql.checkData(56, 3, 468) + tdSql.checkData(59, 3, 468) + tdSql.checkData(60, 3, 60) # # test fill prev tdSql.query(f"select interp(c0),interp(c1) from {dbname}.{tbname1} range('2020-02-02 00:00:00', '2020-02-02 00:01:00') every(1s) fill(prev)") @@ -1973,7 +2434,7 @@ class TDTestCase: tdSql.checkData(3, i, None) tdSql.checkData(4, i, None) - tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(value, 1)") + tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(value, 1, 1, 1, 1)") tdSql.checkRows(5) tdSql.checkCols(4) @@ -2321,25 +2782,25 @@ class TDTestCase: tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') - tdSql.checkData(0, 2, False) + tdSql.checkData(0, 2, None) tdSql.checkData(1, 2, False) - tdSql.checkData(2, 2, False) + tdSql.checkData(2, 2, None) tdSql.checkData(3, 2, True) - tdSql.checkData(4, 2, False) + tdSql.checkData(4, 2, None) tdSql.checkData(5, 2, False) - tdSql.checkData(6, 2, False) + tdSql.checkData(6, 2, None) tdSql.checkData(7, 2, True) - tdSql.checkData(8, 2, False) + tdSql.checkData(8, 2, None) tdSql.checkData(9, 2, True) - tdSql.checkData(10, 2, False) + tdSql.checkData(10, 2, None) tdSql.checkData(11, 2, False) - tdSql.checkData(12, 2, False) + tdSql.checkData(12, 2, None) tdSql.checkData(13, 2, False) - tdSql.checkData(14, 2, False) + tdSql.checkData(14, 2, None) tdSql.checkData(15, 2, None) - tdSql.checkData(16, 2, False) + tdSql.checkData(16, 2, None) tdSql.checkData(17, 2, None) - tdSql.checkData(18, 2, False) + tdSql.checkData(18, 2, None) tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') @@ -2399,6 +2860,10 @@ class TDTestCase: tdSql.error(f"select interp(c0) from {dbname}.{tbname} where _isfilled = true range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") tdSql.error(f"select interp(c0) from {dbname}.{tbname} where _irowts > 0 range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + # fill value number mismatch + tdSql.error(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(value, 1, 2)") + tdSql.error(f"select interp(c0), interp(c1) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(value, 1)") + @@ -2912,9 +3377,2235 @@ class TDTestCase: tdSql.error(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname} range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(null)") tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname} partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:18') every(1s) fill(null)") + tdLog.printNoPrefix("======step 14: test interp ignore null values") + tdSql.execute( + f'''create table if not exists {dbname}.{tbname_null} + (ts timestamp, c0 int, c1 float, c2 bool) + ''' + ) + + tdSql.execute(f"insert into {dbname}.{tbname_null} values ('2020-02-02 00:00:01', 1, 1.0, true)") + tdSql.execute(f"insert into {dbname}.{tbname_null} values ('2020-02-02 00:00:02', NULL, NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname_null} values ('2020-02-02 00:00:03', 3, 3.0, false)") + tdSql.execute(f"insert into {dbname}.{tbname_null} values ('2020-02-02 00:00:04', NULL, NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname_null} values ('2020-02-02 00:00:05', NULL, NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname_null} values ('2020-02-02 00:00:06', 6, 6.0, true)") + tdSql.execute(f"insert into {dbname}.{tbname_null} values ('2020-02-02 00:00:07', NULL, NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname_null} values ('2020-02-02 00:00:08', 8, 8.0, false)") + tdSql.execute(f"insert into {dbname}.{tbname_null} values ('2020-02-02 00:00:09', 9, 9.0, true)") + tdSql.execute(f"insert into {dbname}.{tbname_null} values ('2020-02-02 00:00:10', NULL, NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{tbname_null} values ('2020-02-02 00:00:11', NULL, NULL, NULL)") - tdLog.printNoPrefix("======step 14: test interp pseudo columns") - tdSql.error(f"select _irowts, c6 from {dbname}.{tbname}") + tdSql.execute( + f'''create table if not exists {dbname}.{stbname_null} + (ts timestamp, c0 int, c1 float, c2 bool) tags (t0 int) + ''' + ) + + tdSql.execute( + f'''create table if not exists {dbname}.{ctbname1_null} using {dbname}.{stbname_null} tags(1) + ''' + ) + + tdSql.execute( + f'''create table if not exists {dbname}.{ctbname2_null} using {dbname}.{stbname_null} tags(2) + ''' + ) + + tdSql.execute( + f'''create table if not exists {dbname}.{ctbname3_null} using {dbname}.{stbname_null} tags(3) + ''' + ) + + tdSql.execute(f"insert into {dbname}.{ctbname1_null} values ('2020-02-01 00:00:01', 1, 1.0, true)") + tdSql.execute(f"insert into {dbname}.{ctbname1_null} values ('2020-02-01 00:00:07', NULL, NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{ctbname1_null} values ('2020-02-01 00:00:13', 13, 13.0, false)") + + tdSql.execute(f"insert into {dbname}.{ctbname2_null} values ('2020-02-01 00:00:03', NULL, NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{ctbname2_null} values ('2020-02-01 00:00:09', 9, 9.0, true)") + tdSql.execute(f"insert into {dbname}.{ctbname2_null} values ('2020-02-01 00:00:15', 15, 15.0, false)") + + tdSql.execute(f"insert into {dbname}.{ctbname3_null} values ('2020-02-01 00:00:05', NULL, NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{ctbname3_null} values ('2020-02-01 00:00:11', NULL, NULL, NULL)") + tdSql.execute(f"insert into {dbname}.{ctbname3_null} values ('2020-02-01 00:00:17', NULL, NULL, NULL)") + + # fill null + # normal table + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(NULL)") + + tdSql.checkRows(11) + for i in range (11): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 0) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(NULL)") + + tdSql.checkRows(11) + for i in range (11): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 1) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(NULL)") + + tdSql.checkRows(11) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, False) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, False) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, True) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 2, None) + + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_null} where c0 is not null range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(NULL)") + + tdSql.checkRows(11) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, False) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, False) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, True) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 2, None) + + # super table + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(NULL)") + + tdSql.checkRows(9) + for i in range (9): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, None) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 0) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(NULL)") + + tdSql.checkRows(9) + for i in range (9): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, None) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 1) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(NULL)") + + tdSql.checkRows(9) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, True) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, None) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, None) + + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} where c0 is not null range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(NULL)") + + tdSql.checkRows(9) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, True) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, None) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, None) + + tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0, 1) from {dbname}.{stbname_null} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(NULL)") + + tdSql.checkRows(27) + for i in range(0, 9): + tdSql.checkData(i, 0, 'ctb1_null') + + for i in range(9, 18): + tdSql.checkData(i, 0, 'ctb2_null') + + for i in range(18, 27): + tdSql.checkData(i, 0, 'ctb3_null') + + tdSql.checkData(0, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(8, 1, '2020-02-01 00:00:17.000') + + tdSql.checkData(9, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(17, 1, '2020-02-01 00:00:17.000') + + tdSql.checkData(18, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(26, 1, '2020-02-01 00:00:17.000') + + tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} where c0 is not null partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(NULL)") + + tdSql.checkRows(18) + for i in range(0, 9): + tdSql.checkData(i, 0, 'ctb1_null') + + for i in range(9, 17): + tdSql.checkData(i, 0, 'ctb2_null') + + tdSql.checkData(0, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(8, 1, '2020-02-01 00:00:17.000') + + tdSql.checkData(9, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(17, 1, '2020-02-01 00:00:17.000') + + # fill value + # normal table + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(value, 0)") + + tdSql.checkRows(11) + for i in range (11): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 0) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(value, 0)") + + tdSql.checkRows(11) + for i in range (11): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 1) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(value, 0)") + + tdSql.checkRows(11) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, False) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, False) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, True) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 0) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, 0) + tdSql.checkData(4, 2, 0) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, 0) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, 0) + tdSql.checkData(10, 2, 0) + + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_null} where c0 is not null range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(value, 0)") + + tdSql.checkRows(11) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, False) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, False) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, True) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 0) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, 0) + tdSql.checkData(4, 2, 0) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, 0) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, 0) + tdSql.checkData(10, 2, 0) + + # super table + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(value, 0)") + + tdSql.checkRows(9) + for i in range (9): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, None) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 0) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(value, 0)") + + tdSql.checkRows(9) + for i in range (9): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, None) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 1) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(value, 0)") + + tdSql.checkRows(9) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, True) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 0) + tdSql.checkData(2, 2, 0) + tdSql.checkData(3, 2, 0) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, 0) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, 0) + + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} where c0 is not null range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(value, 0)") + + tdSql.checkRows(9) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, True) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 0) + tdSql.checkData(2, 2, 0) + tdSql.checkData(3, 2, 0) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, 0) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, 0) + + tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0, 1) from {dbname}.{stbname_null} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(value, 0)") + + tdSql.checkRows(27) + for i in range(0, 9): + tdSql.checkData(i, 0, 'ctb1_null') + + for i in range(9, 18): + tdSql.checkData(i, 0, 'ctb2_null') + + for i in range(18, 27): + tdSql.checkData(i, 0, 'ctb3_null') + + tdSql.checkData(0, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(8, 1, '2020-02-01 00:00:17.000') + + tdSql.checkData(9, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(17, 1, '2020-02-01 00:00:17.000') + + tdSql.checkData(18, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(26, 1, '2020-02-01 00:00:17.000') + + tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} where c0 is not null partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(value, 0)") + + tdSql.checkRows(18) + for i in range(0, 9): + tdSql.checkData(i, 0, 'ctb1_null') + + for i in range(9, 18): + tdSql.checkData(i, 0, 'ctb2_null') + + tdSql.checkData(0, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(8, 1, '2020-02-01 00:00:17.000') + + tdSql.checkData(9, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(17, 1, '2020-02-01 00:00:17.000') + + # fill prev + # normal table + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(prev)") + + tdSql.checkRows(11) + for i in range (11): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 0) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(prev)") + + tdSql.checkRows(11) + for i in range (11): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 1) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(prev)") + + tdSql.checkRows(11) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, False) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, False) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, True) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 1) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, 3) + tdSql.checkData(4, 2, 3) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, 6) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, 9) + tdSql.checkData(10, 2, 9) + + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_null} where c0 is not null range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(prev)") + + tdSql.checkRows(11) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, False) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, False) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, True) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 1) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, 3) + tdSql.checkData(4, 2, 3) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, 6) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, 9) + tdSql.checkData(10, 2, 9) + + # super table + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(prev)") + + tdSql.checkRows(9) + for i in range (9): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, None) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 0) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(prev)") + + tdSql.checkRows(9) + for i in range (9): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, None) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 1) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(prev)") + + tdSql.checkRows(9) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, True) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 1) + tdSql.checkData(2, 2, 1) + tdSql.checkData(3, 2, 1) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, 9) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, 15) + + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} where c0 is not null range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(prev)") + + tdSql.checkRows(9) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, True) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 1) + tdSql.checkData(2, 2, 1) + tdSql.checkData(3, 2, 1) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, 9) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, 15) + + tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0, 1) from {dbname}.{stbname_null} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(prev)") + + tdSql.checkRows(14) + for i in range(0, 9): + tdSql.checkData(i, 0, 'ctb1_null') + + for i in range(9, 13): + tdSql.checkData(i, 0, 'ctb2_null') + + tdSql.checkData(0, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(8, 1, '2020-02-01 00:00:17.000') + + tdSql.checkData(9, 1, '2020-02-01 00:00:09.000') + tdSql.checkData(13, 1, '2020-02-01 00:00:17.000') + + tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} where c0 is not null partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(prev)") + + tdSql.checkRows(14) + for i in range(0, 9): + tdSql.checkData(i, 0, 'ctb1_null') + + for i in range(9, 13): + tdSql.checkData(i, 0, 'ctb2_null') + + tdSql.checkData(0, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(8, 1, '2020-02-01 00:00:17.000') + + tdSql.checkData(9, 1, '2020-02-01 00:00:09.000') + tdSql.checkData(13, 1, '2020-02-01 00:00:17.000') + + # fill next + # normal table + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(next)") + + tdSql.checkRows(11) + for i in range (11): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 0) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(next)") + + tdSql.checkRows(11) + for i in range (11): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 1) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(next)") + + tdSql.checkRows(9) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, False) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, 6) + tdSql.checkData(4, 2, 6) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, 8) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_null} where c0 is not null range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(next)") + + tdSql.checkRows(9) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, False) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, 6) + tdSql.checkData(4, 2, 6) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, 8) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + + # super table + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") + + tdSql.checkRows(9) + for i in range (9): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, None) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 0) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") + + tdSql.checkRows(9) + for i in range (9): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, None) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 1) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") + + tdSql.checkRows(8) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 9) + tdSql.checkData(2, 2, 9) + tdSql.checkData(3, 2, 9) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, 13) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} where c0 is not null range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") + + tdSql.checkRows(8) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 9) + tdSql.checkData(2, 2, 9) + tdSql.checkData(3, 2, 9) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, 13) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + + tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0, 1) from {dbname}.{stbname_null} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") + + tdSql.checkRows(15) + for i in range(0, 7): + tdSql.checkData(i, 0, 'ctb1_null') + + for i in range(7, 15): + tdSql.checkData(i, 0, 'ctb2_null') + + tdSql.checkData(0, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(6, 1, '2020-02-01 00:00:13.000') + + tdSql.checkData(7, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(14, 1, '2020-02-01 00:00:15.000') + + tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} where c0 is not null partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(next)") + + tdSql.checkRows(15) + for i in range(0, 7): + tdSql.checkData(i, 0, 'ctb1_null') + + for i in range(7, 15): + tdSql.checkData(i, 0, 'ctb2_null') + + tdSql.checkData(0, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(6, 1, '2020-02-01 00:00:13.000') + + tdSql.checkData(7, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(14, 1, '2020-02-01 00:00:15.000') + + # fill linear + # normal table + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(linear)") + + tdSql.checkRows(11) + for i in range (11): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 0) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(linear)") + + tdSql.checkRows(11) + for i in range (11): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 1) from {dbname}.{tbname_null} range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(linear)") + + tdSql.checkRows(9) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, False) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, 4) + tdSql.checkData(4, 2, 5) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, 7) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_null} where c0 is not null range('2020-02-02 00:00:01', '2020-02-02 00:00:11') every(1s) fill(linear)") + + tdSql.checkRows(9) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, False) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 2, 3) + tdSql.checkData(3, 2, 4) + tdSql.checkData(4, 2, 5) + tdSql.checkData(5, 2, 6) + tdSql.checkData(6, 2, 7) + tdSql.checkData(7, 2, 8) + tdSql.checkData(8, 2, 9) + + # super table + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(linear)") + + tdSql.checkRows(9) + for i in range (9): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, None) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 0) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(linear)") + + tdSql.checkRows(9) + for i in range (9): + tdSql.checkData(i, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 2, None) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, None) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + tdSql.checkData(8, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0, 1) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(linear)") + + tdSql.checkRows(8) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 2, 5) + tdSql.checkData(3, 2, 7) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, 11) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} where c0 is not null range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(linear)") + + tdSql.checkRows(8) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, False) + + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 2, 5) + tdSql.checkData(3, 2, 7) + tdSql.checkData(4, 2, 9) + tdSql.checkData(5, 2, 11) + tdSql.checkData(6, 2, 13) + tdSql.checkData(7, 2, 15) + + tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0, 1) from {dbname}.{stbname_null} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(linear)") + + tdSql.checkRows(11) + for i in range(0, 7): + tdSql.checkData(i, 0, 'ctb1_null') + + for i in range(7, 11): + tdSql.checkData(i, 0, 'ctb2_null') + + tdSql.checkData(0, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(6, 1, '2020-02-01 00:00:13.000') + + tdSql.checkData(7, 1, '2020-02-01 00:00:09.000') + tdSql.checkData(10, 1, '2020-02-01 00:00:15.000') + + tdSql.query(f"select tbname, _irowts, _isfilled, interp(c0) from {dbname}.{stbname_null} where c0 is not null partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(linear)") + + tdSql.checkRows(11) + for i in range(0, 7): + tdSql.checkData(i, 0, 'ctb1_null') + + for i in range(7, 11): + tdSql.checkData(i, 0, 'ctb2_null') + + tdSql.checkData(0, 1, '2020-02-01 00:00:01.000') + tdSql.checkData(6, 1, '2020-02-01 00:00:13.000') + + tdSql.checkData(7, 1, '2020-02-01 00:00:09.000') + tdSql.checkData(10, 1, '2020-02-01 00:00:15.000') + + # multiple column with ignoring null value is not allowed + + tdSql.query(f"select _irowts, _isfilled, interp(c0), interp(c1), interp(c2) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(linear)") + tdSql.query(f"select _irowts, _isfilled, interp(c0, 0), interp(c1, 0), interp(c2, 0) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(linear)") + tdSql.query(f"select _irowts, _isfilled, interp(c0), interp(c1, 0), interp(c2) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(linear)") + + tdSql.error(f"select _irowts, _isfilled, interp(c0), interp(c1, 1), interp(c2) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(linear)") + tdSql.error(f"select _irowts, _isfilled, interp(c0, 1), interp(c1, 0), interp(c2, 0) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(linear)") + tdSql.error(f"select _irowts, _isfilled, interp(c0), interp(c1, 0), interp(c2, 1) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(linear)") + tdSql.error(f"select _irowts, _isfilled, interp(c0, 1), interp(c1, 1), interp(c2, 1) from {dbname}.{stbname_null} range('2020-02-01 00:00:01', '2020-02-01 00:00:17') every(2s) fill(linear)") + + + + tdLog.printNoPrefix("======step 15: test interp pseudo columns") + tdSql.error(f"select _irowts, c6 from {dbname}.{tbname}") + + tdLog.printNoPrefix("======step 16: test interp in nested query") + tdSql.query(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{stbname}) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + tdSql.query(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1}) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + + tdSql.error(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{stbname}) partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + tdSql.error(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1}) partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + + tdSql.error(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union select * from {dbname}.{ctbname2}) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + tdSql.query(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union select * from {dbname}.{ctbname2} order by ts) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + + tdSql.error(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union all select * from {dbname}.{ctbname2}) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + tdSql.query(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union all select * from {dbname}.{ctbname2} order by ts) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + + tdSql.error(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union all select * from {dbname}.{ctbname2}) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + tdSql.query(f"select _irowts, _isfilled, interp(c0) from (select * from {dbname}.{ctbname1} union all select * from {dbname}.{ctbname2} order by ts) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from (select {ctbname1}.ts,{ctbname1}.c0 from {dbname}.{ctbname1}, {dbname}.{ctbname2} where {ctbname1}.ts = {ctbname2}.ts) range('2020-02-01 00:00:00', '2020-02-01 00:00:14') every(1s) fill(null)") + + tdLog.printNoPrefix("======step 17: test interp single point") + tdSql.execute( + f'''create table if not exists {dbname}.{tbname_single} + (ts timestamp, c0 int) + ''' + ) + + tdSql.execute(f"insert into {dbname}.{tbname_single} values ('2020-02-01 00:00:01', 1)") + tdSql.execute(f"insert into {dbname}.{tbname_single} values ('2020-02-01 00:00:03', 3)") + tdSql.execute(f"insert into {dbname}.{tbname_single} values ('2020-02-01 00:00:05', 5)") + + tdSql.execute( + f'''create table if not exists {dbname}.{stbname_single} + (ts timestamp, c0 int, c1 float, c2 bool) tags (t0 int) + ''' + ) + + tdSql.execute( + f'''create table if not exists {dbname}.{ctbname1_single} using {dbname}.{stbname_single} tags(1) + ''' + ) + + tdSql.execute( + f'''create table if not exists {dbname}.{ctbname2_single} using {dbname}.{stbname_single} tags(2) + ''' + ) + + tdSql.execute( + f'''create table if not exists {dbname}.{ctbname3_single} using {dbname}.{stbname_single} tags(3) + ''' + ) + + tdSql.execute(f"insert into {dbname}.{ctbname1_single} values ('2020-02-01 00:00:01', 1, 1.0, true)") + + tdSql.execute(f"insert into {dbname}.{ctbname2_single} values ('2020-02-01 00:00:03', 3, 3.0, false)") + + tdSql.execute(f"insert into {dbname}.{ctbname3_single} values ('2020-02-01 00:00:05', 5, 5.0, true)") + + # normal table + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(prev)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00') fill(prev)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(next)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06') fill(next)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(linear)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:00') fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:01') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 2) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:02') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 2) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:03') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 4) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:04') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 4) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:05') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(linear)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{tbname_single} range('2020-02-01 00:00:06') fill(linear)") + tdSql.checkRows(0) + + #super table + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06') fill(null)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06') fill(value, 0)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(prev)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00') fill(prev)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(next)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06') fill(next)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(linear)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:00') fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:01') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 2) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:02') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 2) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:03') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 4) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:04') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 4) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:05') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(linear)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} range('2020-02-01 00:00:06') fill(linear)") + tdSql.checkRows(0) + + # partition by tbname + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(null)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00') fill(null)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(null)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + for i in range(1, 3): + tdSql.checkData(i, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01') fill(null)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + for i in range(1, 3): + tdSql.checkData(i, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(null)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02') fill(null)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(null)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06') fill(null)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, None) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(value,0)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00') fill(value,0)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(value,0)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + for i in range(1, 3): + tdSql.checkData(i, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01') fill(value,0)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + for i in range(1, 3): + tdSql.checkData(i, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(value,0)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02') fill(value,0)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(value,0)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06') fill(value,0)") + tdSql.checkRows(3) + for i in range(3): + tdSql.checkData(i, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(i, 1, True) + tdSql.checkData(i, 2, 0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(prev)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00') fill(prev)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02') fill(prev)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(prev)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(1, 1, False) + tdSql.checkData(1, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:03') fill(prev)") + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(1, 1, False) + tdSql.checkData(1, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(prev)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:04') fill(prev)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(prev)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(2, 1, False) + tdSql.checkData(2, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:05') fill(prev)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(2, 1, False) + tdSql.checkData(2, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(prev)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(2, 1, True) + tdSql.checkData(2, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06') fill(prev)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(2, 1, True) + tdSql.checkData(2, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(next)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06') fill(next)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:05') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:04') fill(next)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(next)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.checkData(1, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:03') fill(next)") + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.checkData(1, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(next)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + tdSql.checkData(1, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02') fill(next)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 3) + tdSql.checkData(1, 0, '2020-02-01 00:00:02.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(next)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(2, 1, True) + tdSql.checkData(2, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01') fill(next)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(2, 1, True) + tdSql.checkData(2, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00', '2020-02-01 00:00:00') every(1s) fill(next)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(2, 1, True) + tdSql.checkData(2, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00') fill(next)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(0, 1, True) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(1, 1, True) + tdSql.checkData(1, 2, 3) + tdSql.checkData(2, 0, '2020-02-01 00:00:00.000') + tdSql.checkData(2, 1, True) + tdSql.checkData(2, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:00') fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01', '2020-02-01 00:00:01') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:01') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:01.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 1) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02', '2020-02-01 00:00:02') every(1s) fill(linear)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:02') fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:03', '2020-02-01 00:00:03') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:03') fill(linear)") + tdSql.checkData(0, 0, '2020-02-01 00:00:03.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 3) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:04', '2020-02-01 00:00:04') every(1s) fill(linear)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:04') fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:05', '2020-02-01 00:00:05') every(1s) fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:05') fill(linear)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(0, 2, 5) + + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06', '2020-02-01 00:00:06') every(1s) fill(linear)") + tdSql.checkRows(0) + tdSql.query(f"select _irowts, _isfilled, interp(c0) from {dbname}.{stbname_single} partition by tbname range('2020-02-01 00:00:06') fill(linear)") + tdSql.checkRows(0) def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/interval_limit_opt.py b/tests/system-test/2-query/interval_limit_opt.py new file mode 100644 index 0000000000000000000000000000000000000000..fef6e9facd1edf50a8bd8f64762d2f0a052444b8 --- /dev/null +++ b/tests/system-test/2-query/interval_limit_opt.py @@ -0,0 +1,266 @@ +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +# from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 4 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + self.duraion = '1h' + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d duration %s"%(dbName, vgroups, replica, duration)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, paraDict): + colString = tdCom.gen_column_type_str(colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"]) + tagString = tdCom.gen_tag_type_str(tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"]) + sqlString = f"create table if not exists %s.%s (%s) tags (%s)"%(paraDict["dbName"], paraDict["stbName"], colString, tagString) + tdLog.debug("%s"%(sqlString)) + tsql.execute(sqlString) + return + + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0): + for i in range(ctbNum): + sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % \ + (dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,(i+ctbStartIdx) % 5,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx) + tsql.execute(sqlString) + + tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,tsStep): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + for i in range(ctbNum): + rowsBatched = 0 + sql += " %s%d values "%(ctbPrefix,i) + for j in range(rowsPerTbl): + if (i < ctbNum/2): + sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10, j%10, j%10) + else: + sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10) + rowsBatched += 1 + if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsBatched = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s%d values " %(ctbPrefix,i) + else: + sql = "insert into " + if sql != pre_insert: + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'test', + 'dropFlag': 1, + 'vgroups': 2, + 'stbName': 'meters', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'FLOAT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'smallint', 'count':1},{'type': 'tinyint', 'count':1},{'type': 'bool', 'count':1},{'type': 'binary', 'len':10, 'count':1},{'type': 'nchar', 'len':10, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'nchar', 'len':20, 'count':1},{'type': 'binary', 'len':20, 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'smallint', 'count':1},{'type': 'DOUBLE', 'count':1}], + 'ctbPrefix': 't', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 10000, + 'batchNum': 3000, + 'startTs': 1537146000000, + 'tsStep': 600000} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdLog.info("create database") + self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"], vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion) + + tdLog.info("create stb") + self.create_stable(tsql=tdSql, paraDict=paraDict) + + tdLog.info("create child tables") + self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], \ + stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],\ + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict["ctbStartIdx"]) + self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],\ + ctbPrefix=paraDict["ctbPrefix"],ctbNum=paraDict["ctbNum"],\ + rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],\ + startTs=paraDict["startTs"],tsStep=paraDict["tsStep"]) + return + + def check_first_rows(self, all_rows, limited_rows, offset: int = 0): + for i in range(0, len(limited_rows) - 1): + if limited_rows[i] != all_rows[i + offset]: + tdLog.info("row: %d, row in all: %s" % (i+offset+1, str(all_rows[i+offset]))) + tdLog.info("row: %d, row in limted: %s" % (i+1, str(limited_rows[i]))) + tdLog.exit("row data check failed") + tdLog.info("all rows are the same as query without limit..") + + def query_and_check_with_slimit(self, sql: str, max_limit: int, step: int, offset: int = 0): + self.query_and_check_with_limit(sql, max_limit, step, offset, ' slimit ') + + def query_and_check_with_limit(self, sql: str, max_limit: int, step: int, offset: int = 0, limit_str: str = ' limit '): + for limit in range(0, max_limit, step): + limited_sql = sql + limit_str + str(offset) + "," + str(limit) + tdLog.info("query with sql: %s " % (sql) + limit_str + " %d,%d" % (offset, limit)) + all_rows = tdSql.getResult(sql) + limited_rows = tdSql.getResult(limited_sql) + tdLog.info("all rows: %d, limited rows: %d" % (len(all_rows), len(limited_rows))) + if limit_str == ' limit ': + if limit + offset <= len(all_rows) and len(limited_rows) != limit: + tdLog.exit("limited sql has less rows than limit value which is not right, \ + limit: %d, limited_rows: %d, all_rows: %d, offset: %d" % (limit, len(limited_rows), len(all_rows), offset)) + elif limit + offset > len(all_rows) and offset < len(all_rows) and offset + len(limited_rows) != len(all_rows): + tdLog.exit("limited sql has less rows than all_rows which is not right, \ + limit: %d, limited_rows: %d, all_rows: %d, offset: %d" % (limit, len(limited_rows), len(all_rows), offset)) + elif offset >= len(all_rows) and len(limited_rows) != 0: + tdLog.exit("limited rows should be zero, \ + limit: %d, limited_rows: %d, all_rows: %d, offset: %d" % (limit, len(limited_rows), len(all_rows), offset)) + + self.check_first_rows(all_rows, limited_rows, offset) + + def test_interval_limit_asc(self, offset: int = 0): + sqls = ["select _wstart, _wend, count(*), sum(c1), avg(c2), first(ts) from meters interval(1s) ", + "select _wstart, _wend, count(*), sum(c1), avg(c2), first(ts) from meters interval(1m) ", + "select _wstart, _wend, count(*), sum(c1), avg(c2), first(ts) from meters interval(1h) ", + "select _wstart, _wend, count(*), sum(c1), avg(c2), first(ts) from meters interval(1d) ", + "select _wstart, _wend, count(*), sum(c1), avg(c2), first(ts) from t1 interval(1s) ", + "select _wstart, _wend, count(*), sum(c1), avg(c2), first(ts) from t1 interval(1m) ", + "select _wstart, _wend, count(*), sum(c1), avg(c2), first(ts) from t1 interval(1h) ", + "select _wstart, _wend, count(*), sum(c1), avg(c2), first(ts) from t1 interval(1d) "] + for sql in sqls: + self.query_and_check_with_limit(sql, 5000, 500, offset) + + def test_interval_limit_desc(self, offset: int = 0): + sqls = ["select _wstart, _wend, count(*), sum(c1), avg(c2), last(ts) from meters interval(1s) ", + "select _wstart, _wend, count(*), sum(c1), avg(c2), last(ts) from meters interval(1m) ", + "select _wstart, _wend, count(*), sum(c1), avg(c2), last(ts) from meters interval(1h) ", + "select _wstart, _wend, count(*), sum(c1), avg(c2), last(ts) from meters interval(1d) ", + "select _wstart, _wend, count(*), sum(c1), avg(c2), last(ts) from t1 interval(1s) ", + "select _wstart, _wend, count(*), sum(c1), avg(c2), last(ts) from t1 interval(1m) ", + "select _wstart, _wend, count(*), sum(c1), avg(c2), last(ts) from t1 interval(1h) ", + "select _wstart, _wend, count(*), sum(c1), avg(c2), last(ts) from t1 interval(1d) "] + for sql in sqls: + self.query_and_check_with_limit(sql, 5000, 500, offset) + + def test_interval_limit_offset(self): + for offset in range(0, 1000, 500): + self.test_interval_limit_asc(offset) + self.test_interval_limit_desc(offset) + self.test_interval_fill_limit(offset) + self.test_interval_order_by_limit(offset) + self.test_interval_partition_by_slimit(offset) + + def test_interval_fill_limit(self, offset: int = 0): + sqls = [ + "select _wstart as a, _wend as b, count(*), sum(c1), avg(c2), first(ts) from meters \ + where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:30:00.000' interval(1s) fill(linear)", + "select _wstart as a, _wend as b, count(*), sum(c1), avg(c2), first(ts) from meters \ + where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:30:00.000' interval(1m) fill(linear)", + "select _wstart as a, _wend as b, count(*), sum(c1), avg(c2), first(ts) from meters \ + where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:30:00.000' interval(1h) fill(linear)", + "select _wstart as a, _wend as b, count(*), sum(c1), avg(c2), first(ts) from meters \ + where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:30:00.000' interval(1d) fill(linear)" + ] + for sql in sqls: + self.query_and_check_with_limit(sql, 5000, 1000, offset) + + def test_interval_order_by_limit(self, offset: int = 0): + sqls = [ + "select _wstart as a, _wend as b, count(*), sum(c1), avg(c2), first(ts) from meters \ + where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' interval(1m) order by b", + "select _wstart as a, _wend as b, count(*), sum(c1), avg(c2), first(ts) from meters \ + where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' interval(1m) order by a desc", + "select _wstart as a, _wend as b, count(*), sum(c1), avg(c2), last(ts) from meters \ + where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' interval(1m) order by a desc", + "select _wstart as a, _wend as b, count(*), sum(c1), avg(c2), first(ts) from meters \ + where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' interval(1m) order by count(*), sum(c1), a", + "select _wstart as a, _wend as b, count(*), sum(c1), avg(c2), first(ts) from meters \ + where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' interval(1m) order by a, count(*), sum(c1)", + "select _wstart as a, _wend as b, count(*), sum(c1), avg(c2), first(ts) from meters \ + where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' interval(1m) fill(linear) order by b", + "select _wstart as a, _wend as b, count(*), sum(c1), avg(c2), first(ts) from meters \ + where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' interval(1m) fill(linear) order by a desc", + "select _wstart as a, _wend as b, count(*), sum(c1), last(c2), first(ts) from meters \ + where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' interval(1m) fill(linear) order by a desc", + "select _wstart as a, _wend as b, count(*), sum(c1), avg(c2), first(ts) from meters \ + where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' interval(1m) fill(linear) order by count(*), sum(c1), a", + "select _wstart as a, _wend as b, count(*), sum(c1), avg(c2), first(ts) from meters \ + where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' interval(1m) fill(linear) order by a, count(*), sum(c1)", + ] + for sql in sqls: + self.query_and_check_with_limit(sql, 6000, 2000, offset) + + def test_interval_partition_by_slimit(self, offset: int = 0): + sqls = [ + "select _wstart as a, _wend as b, count(*), sum(c1), last(c2), first(ts) from meters " + "where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' partition by t1 interval(1m)", + "select _wstart as a, _wend as b, count(*), sum(c1), last(c2), first(ts) from meters " + "where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' partition by t1 interval(1h)", + "select _wstart as a, _wend as b, count(*), sum(c1), last(c2), first(ts) from meters " + "where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' partition by c3 interval(1m)", + ] + for sql in sqls: + self.query_and_check_with_slimit(sql, 10, 2, offset) + + def test_interval_partition_by_slimit_limit(self): + sql = "select * from (select _wstart as a, _wend as b, count(*), sum(c1), last(c2), first(ts),c3 from meters " \ + "where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' partition by c3 interval(1m) slimit 10 limit 2) order by c3 asc" + tdSql.query(sql) + tdSql.checkRows(20) + tdSql.checkData(0, 4, 0) + tdSql.checkData(1, 4, 0) + tdSql.checkData(2, 4, 1) + tdSql.checkData(3, 4, 1) + tdSql.checkData(18, 4, 9) + tdSql.checkData(19, 4, 9) + + sql = "select * from (select _wstart as a, _wend as b, count(*), sum(c1), last(c2), first(ts),c3 from meters " \ + "where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-10-17 09:30:00.000' partition by c3 interval(1m) slimit 2,2 limit 2) order by c3 asc" + tdSql.query(sql) + tdSql.checkRows(4) + tdSql.checkData(0, 4, 2) + tdSql.checkData(1, 4, 2) + tdSql.checkData(2, 4, 9) + tdSql.checkData(3, 4, 9) + + def run(self): + self.prepareTestEnv() + self.test_interval_limit_offset() + self.test_interval_partition_by_slimit_limit() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/leastsquares.py b/tests/system-test/2-query/leastsquares.py index 1718802a82a0ffb740918e1ff0e083a11168b3f4..3dfd1f6acaf1f1a636c6ee8e5944a63e6ef70851 100644 --- a/tests/system-test/2-query/leastsquares.py +++ b/tests/system-test/2-query/leastsquares.py @@ -219,15 +219,20 @@ class TDTestCase: tdSql.checkRows(1) res = tdSql.getData(0, 0) if res is None: - tdLog.exit("result is not correct") + tdLog.exit("result is not correct") - def __test_current(self): + def __test_current(self, dbname=DBNAME): # tdSql.query("explain select c1 from {dbname}.ct1") # tdSql.query("explain select 1 from {dbname}.ct2") # tdSql.query("explain select cast(ceil(c6) as bigint) from {dbname}.ct4 group by c6") # tdSql.query("explain select count(c3) from {dbname}.ct4 group by c7 having count(c3) > 0") # tdSql.query("explain select ct2.c3 from {dbname}.ct4 join ct2 on ct4.ts=ct2.ts") # tdSql.query("explain select c1 from stb1 where c1 is not null and c1 in (0, 1, 2) or c1 between 2 and 100 ") + # + tdSql.query(f"select leastsquares(c1, 1, 1) from (select c1 from {dbname}.nt1 group by c1)") + tdSql.query(f"select leastsquares(c1, 1, 1) from (select c1 from {dbname}.nt1 partition by c1)") + tdSql.query(f"select leastsquares(c1, 1, 1) from (select c1 from {dbname}.nt1 order by c1)") + tdSql.query(f"select leastsquares(c1, 1, 1) from (select c1 from {dbname}.nt1 union select c1 from {dbname}.nt1)") self.leastsquares_check() diff --git a/tests/system-test/2-query/limit.py b/tests/system-test/2-query/limit.py index c00e3b7d56af46f662ddb855a34e154e7c723b80..4774602d691cbcb516e3400ae14eea0a62d7fe0b 100644 --- a/tests/system-test/2-query/limit.py +++ b/tests/system-test/2-query/limit.py @@ -321,7 +321,7 @@ class TDTestCase: limit = 5 offset = paraDict["rowsPerTbl"] * 2 offset = offset - 2 - sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') order by t1 limit %d offset %d"%(limit, offset) + sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') order by t1, max(c1) limit %d offset %d"%(limit, offset) # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(1) diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py index dec24010fcf4a43fa16863a577ad3b49fb160e23..fbd3488aabc53ad9817b4f4632f490502457785d 100644 --- a/tests/system-test/2-query/max_partition.py +++ b/tests/system-test/2-query/max_partition.py @@ -172,7 +172,7 @@ class TDTestCase: tdSql.checkRows(90) tdSql.query(f"select c1 , diff(c1 , 0) from {dbname}.stb partition by c1") - tdSql.checkRows(90) + tdSql.checkRows(140) tdSql.query(f"select c1 , csum(c1) from {dbname}.stb partition by c1") tdSql.checkRows(100) diff --git a/tests/system-test/2-query/odbc.py b/tests/system-test/2-query/odbc.py index f7851ad04b12cfb48a08f51fabbc2afe950b4585..8fbad93995abba170c2a78d3f61f819d342de342 100644 --- a/tests/system-test/2-query/odbc.py +++ b/tests/system-test/2-query/odbc.py @@ -21,9 +21,7 @@ class TDTestCase: tdSql.execute("create table db.stb (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 tinyint unsigned, c7 smallint unsigned, c8 int unsigned, c9 bigint unsigned, c10 float, c11 double, c12 varchar(100), c13 nchar(100)) tags(t int)") tdSql.execute("insert into db.ctb using db.stb tags(1) (ts, c1) values (now, 1)") - tdSql.query("select count(*) from information_schema.ins_columns") - # enterprise version: 285, community version: 277 - tdSql.checkData(0, 0, 285) + tdSql.execute("select count(*) from information_schema.ins_columns") tdSql.query("select * from information_schema.ins_columns where table_name = 'ntb'") tdSql.checkRows(14) diff --git a/tests/system-test/2-query/orderBy.py b/tests/system-test/2-query/orderBy.py new file mode 100644 index 0000000000000000000000000000000000000000..fed1651b3ad670005adc18f48da25c32746e8f07 --- /dev/null +++ b/tests/system-test/2-query/orderBy.py @@ -0,0 +1,298 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time +import copy + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + # c3 is order + c3 = i * self.childRow + j + + value = f"({self.ts}, " + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null," + else: + value += f"{c2}," + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + # c3 + value += f"{c3}," + # ts1 same with ts + value += f"{self.ts})" + + # move next + self.ts += 1 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + + sql = "flush database db;" + tdLog.info(sql) + tdSql.execute(sql) + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1680000000000*1000 + self.childCnt = 10 + self.childRow = 100000 + self.batchSize = 5000 + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # create database db + sql = f"create database db vgroups 2 precision 'us' " + tdLog.info(sql) + tdSql.execute(sql) + sql = f"use db" + tdSql.execute(sql) + + # alter config + sql = "alter local 'querySmaOptimize 1';" + tdLog.info(sql) + tdSql.execute(sql) + + # create super talbe st + sql = f"create table st(ts timestamp, c1 int, c2 bigint, c3 bigint, ts1 timestamp) tags(area int)" + tdLog.info(sql) + tdSql.execute(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table t{i} using st tags({i}) " + tdSql.execute(sql) + + # insert data + self.insertData() + + # check data correct + def checkExpect(self, sql, expectVal): + tdSql.query(sql) + rowCnt = tdSql.getRows() + for i in range(rowCnt): + val = tdSql.getData(i,0) + if val != expectVal: + tdLog.exit(f"Not expect . query={val} expect={expectVal} i={i} sql={sql}") + return False + + tdLog.info(f"check expect ok. sql={sql} expect ={expectVal} rowCnt={rowCnt}") + return True + + # check query + def queryResultSame(self, sql1, sql2): + # sql + tdLog.info(sql1) + start1 = time.time() + rows1 = tdSql.query(sql1) + spend1 = time.time() - start1 + res1 = copy.copy(tdSql.queryResult) + + tdLog.info(sql2) + start2 = time.time() + tdSql.query(sql2) + spend2 = time.time() - start2 + res2 = tdSql.queryResult + + rowlen1 = len(res1) + rowlen2 = len(res2) + + if rowlen1 != rowlen2: + tdLog.exit(f"rowlen1={rowlen1} rowlen2={rowlen2} both not equal.") + return False + + for i in range(rowlen1): + row1 = res1[i] + row2 = res2[i] + collen1 = len(row1) + collen2 = len(row2) + if collen1 != collen2: + tdLog.exit(f"collen1={collen1} collen2={collen2} both not equal.") + return False + for j in range(collen1): + if row1[j] != row2[j]: + tdLog.exit(f"col={j} col1={row1[j]} col2={row2[j]} both col not equal.") + return False + + # warning performance + diff = (spend2 - spend1)*100/spend1 + tdLog.info("spend1=%.6fs spend2=%.6fs diff=%.1f%%"%(spend1, spend2, diff)) + if spend2 > spend1 and diff > 50: + tdLog.info("warning: the diff for performance after spliting is over 20%") + + return True + + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.clock_gettime(time.CLOCK_REALTIME) + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # check time macro + def queryBasic(self): + # check count + expectVal = self.childCnt * self.childRow + sql = f"select count(ts) from st " + self.checkExpect(sql, expectVal) + + # check diff + sql = f"select count(*) from (select diff(ts) as dif from st order by ts)" + self.checkExpect(sql, expectVal - 1) + + # check ts order count + sql = f"select count(*) from (select diff(ts) as dif from st order by ts) where dif!=1" + self.checkExpect(sql, 0) + + # check ts1 order count + sql = f"select count(*) from (select diff(ts1) as dif from st order by ts1) where dif!=1" + self.checkExpect(sql, 0) + + # check c3 order asc + sql = f"select count(*) from (select diff(c3) as dif from st order by c3) where dif!=1" + self.checkExpect(sql, 0) + + # check c3 order desc todo FIX + #sql = f"select count(*) from (select diff(c3) as dif from st order by c3 desc) where dif!=-1" + #self.checkExpect(sql, 0) + + + # advance + def queryAdvance(self): + # interval order todo FIX + #sql = f"select _wstart,count(ts),max(c2),min(c2) from st interval(100u) sliding(50u) order by _wstart limit 10" + #tdSql.query(sql) + #tdSql.checkRows(10) + + # simulate crash sql + sql = f"select _wstart,count(ts),max(c2),min(c2) from st interval(100a) sliding(10a) order by _wstart limit 10" + tdSql.query(sql) + tdSql.checkRows(10) + + # extent + sql = f"select _wstart,count(ts),max(c2),min(c2) from st interval(100a) sliding(10a) order by _wstart desc limit 5" + tdSql.query(sql) + tdSql.checkRows(5) + + # data correct checked + sql1 = "select sum(a),sum(b), max(c), min(d),sum(e) from (select _wstart,count(ts) as a,count(c2) as b ,max(c2) as c, min(c2) as d, sum(c2) as e from st interval(100a) sliding(100a) order by _wstart desc);" + sql2 = "select count(*) as a, count(c2) as b, max(c2) as c, min(c2) as d, sum(c2) as e from st;" + self.queryResultSame(sql1, sql2) + + # run + def run(self): + # prepare env + self.prepareEnv() + + # basic + self.queryBasic() + + # advance + self.queryAdvance() + + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/slimit.py b/tests/system-test/2-query/slimit.py index 48209da59a081a923884770a05cfd118f1e953b5..b5aa1879803d6564730d3865d555126193d628c7 100644 --- a/tests/system-test/2-query/slimit.py +++ b/tests/system-test/2-query/slimit.py @@ -40,7 +40,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root)-len("/build/bin")] diff --git a/tests/system-test/2-query/smaBasic.py b/tests/system-test/2-query/smaBasic.py new file mode 100644 index 0000000000000000000000000000000000000000..c221a70605064a3d23210c7a064c36657d5a9b5a --- /dev/null +++ b/tests/system-test/2-query/smaBasic.py @@ -0,0 +1,296 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + self.ts += 1 + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null)" + else: + value += f"{c2})" + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + + sql = "flush database db;" + tdLog.info(sql) + tdSql.execute(sql) + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1600000000000 + self.childCnt = 5 + self.childRow = 1000000 + self.batchSize = 5000 + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # create database db + sql = f"create database db vgroups 5 replica 3 stt_trigger 1" + tdLog.info(sql) + tdSql.execute(sql) + sql = f"use db" + tdSql.execute(sql) + + # create super talbe st + sql = f"create table st(ts timestamp, c1 int, c2 bigint) tags(area int)" + tdLog.info(sql) + tdSql.execute(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table t{i} using st tags({i}) " + tdSql.execute(sql) + + # insert data + self.insertData() + + # query sql value + def queryValue(self, sql): + tdSql.query(sql) + return tdSql.getData(0, 0) + + # sum + def checkCorrentSum(self): + # query count + sql = "select sum(c1) from st" + val = self.queryValue(sql) + # c1Sum is equal c1Cnt + if val != self.c1Cnt: + tdLog.exit(f"Sum Not Expect. expect={self.c1Cnt} query={val} sql:{sql}") + return + + # not + sql1 = "select sum(c1) from st where c2 = 8764231" + val1 = self.queryValue(sql1) + sql2 = "select sum(c1) from st where c2 != 8764231" + val2 = self.queryValue(sql2) + sql3 = "select sum(c1) from st where c2 is null" + val3 = self.queryValue(sql3) + if val != val1 + val2 + val3: + tdLog.exit(f"Sum Not Equal. val != val1 + val2 + val3. val={val} val1={val1} val2={val2} val2={val3} sql1={sql1} sql2={sql2} sql2={sql3}") + return + + # over than + sql1 = "select sum(c1) from st where c2 > 8000" + val1 = self.queryValue(sql1) + sql2 = "select sum(c1) from st where c2 <= 8000" + val2 = self.queryValue(sql2) + sql3 = "select sum(c1) from st where c2 is null" + val3 = self.queryValue(sql3) + if val != val1 + val2 + val3: + tdLog.exit(f"Sum Not Equal. val != val1 + val2 + val3. val={val} val1={val1} val2={val2} val2={val3} sql1={sql1} sql2={sql2} sql2={sql3}") + return + + tdLog.info(f"check correct sum on c1 successfully.") + + # check result + def checkResult(self, fun, val, val1, val2, sql1, sql2): + if fun == "count": + if val != val1 + val2: + tdLog.exit(f"{fun} NOT SAME. val != val1 + val2. val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + elif fun == "max": + if val != max([val1, val2]): + tdLog.exit(f"{fun} NOT SAME . val != max(val1 ,val2) val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + elif fun == "min": + if val != min([val1, val2]): + tdLog.exit(f"{fun} NOT SAME . val != min(val1 ,val2) val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + + # sum + def checkCorrentFun(self, fun, expectVal): + # query + sql = f"select {fun}(c2) from st" + val = self.queryValue(sql) + if val != expectVal: + tdLog.exit(f"{fun} Not Expect. expect={expectVal} query={val} sql:{sql}") + return + + # not + sql1 = f"select {fun}(c2) from st where c2 = 8764231" + val1 = self.queryValue(sql1) + sql2 = f"select {fun}(c2) from st where c2 != 8764231" + val2 = self.queryValue(sql2) + self.checkResult(fun, val, val1, val2, sql1, sql2) + + # over than + sql1 = f"select {fun}(c2) from st where c2 > 8000" + val1 = self.queryValue(sql1) + sql2 = f"select {fun}(c2) from st where c2 <= 8000" + val2 = self.queryValue(sql2) + self.checkResult(fun, val, val1, val2, sql1, sql2) + + # successful + tdLog.info(f"check correct {fun} on c2 successfully.") + + # check query corrent + def checkCorrect(self): + # count + self.checkCorrentFun("count", self.c2Cnt) + # max + self.checkCorrentFun("max", self.c2Max) + # min + self.checkCorrentFun("min", self.c2Min) + # sum + self.checkCorrentSum() + + # c2 sum + sql = "select sum(c2) from st" + val = self.queryValue(sql) + # c1Sum is equal c1Cnt + if val != self.c2Sum: + tdLog.exit(f"c2 Sum Not Expect. expect={self.c2Sum} query={val} sql:{sql}") + return + + def checkPerformance(self): + # have sma caculate + sql1 = "select count(*) from st" + stime = time.time() + tdSql.execute(sql1, 1) + spend1 = time.time() - stime + + + # no sma caculate + sql2 = "select count(*) from st where c2 != 8764231 or c2 is null" + stime = time.time() + tdSql.execute(sql2, 1) + spend2 = time.time() - stime + + time1 = "%.2f"%(spend1*1000) + time2 = "%.2f"%(spend2*1000) + if spend2 < spend1 * 8: + tdLog.exit(f"performance not passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") + return + tdLog.info(f"performance passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") + + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.clock_gettime(time.CLOCK_REALTIME) + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # run + def run(self): + # prepare env + self.prepareEnv() + + # query + self.checkCorrect() + + # performance + self.checkPerformance() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index 2f97118fbf0893a253499ab7c9ba9425bd9e9c5d..b3aeb7219439415cb06ac4661a524466429a105d 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -15,7 +15,7 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'fqdn': 'localhost'}, 'fqdn': 'localhost'} + updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'fqdn': 'localhost', 'smlDot2Underline': 0}, 'fqdn': 'localhost'} print("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): @@ -101,6 +101,15 @@ class TDTestCase: tdSql.query(f"desc {dbname}.macylr") tdSql.checkRows(25) + + tdSql.query(f"select * from ts3724.`.stb2`") + tdSql.checkRows(1) + + tdSql.query(f"select * from ts3724.`stb.2`") + tdSql.checkRows(1) + + tdSql.query(f"select * from ts3724.`stb2.`") + tdSql.checkRows(1) return def run(self): diff --git a/tests/system-test/2-query/sml_TS-3724.py b/tests/system-test/2-query/sml_TS-3724.py new file mode 100644 index 0000000000000000000000000000000000000000..a8b16c4662596db00a79f03eb602be7ccc33ef6c --- /dev/null +++ b/tests/system-test/2-query/sml_TS-3724.py @@ -0,0 +1,125 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'fqdn': 'localhost', 'smlTsDefaultName': "times"}, 'fqdn': 'localhost'} + print("===================: ", updatecfgDict) + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def checkContent(self, dbname="sml_db"): + simClientCfg="%s/taos.cfg"%tdDnodes.getSimCfgPath() + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/sml_test %s'%(buildPath, simClientCfg) + print("cmdStr:", cmdStr) + tdLog.info(cmdStr) + ret = os.system(cmdStr) + if ret != 0: + tdLog.info("sml_test ret != 0") + + tdSql.query(f"select * from ts3303.stb2") + tdSql.query(f"select * from ts3303.meters") + + # tdSql.execute('use sml_db') + tdSql.query(f"select * from {dbname}.t_b7d815c9222ca64cdf2614c61de8f211") + tdSql.checkRows(1) + + tdSql.checkData(0, 0, '2016-01-01 08:00:07.000') + tdSql.checkData(0, 1, 2000) + tdSql.checkData(0, 2, 200) + tdSql.checkData(0, 3, 15) + tdSql.checkData(0, 4, 24.5208) + tdSql.checkData(0, 5, 28.09377) + tdSql.checkData(0, 6, 428) + tdSql.checkData(0, 7, 0) + tdSql.checkData(0, 8, 304) + tdSql.checkData(0, 9, 0) + tdSql.checkData(0, 10, 25) + + tdSql.query(f"select * from {dbname}.readings") + tdSql.checkRows(9) + + tdSql.query(f"select distinct tbname from {dbname}.readings") + tdSql.checkRows(4) + + tdSql.query(f"select * from {dbname}.t_0799064f5487946e5d22164a822acfc8 order by times") + tdSql.checkRows(2) + tdSql.checkData(0, 3, "kk") + tdSql.checkData(1, 3, "") + + + tdSql.query(f"select distinct tbname from {dbname}.`sys_if_bytes_out`") + tdSql.checkRows(2) + + tdSql.query(f"select * from {dbname}.t_fc70dec6677d4277c5d9799c4da806da order by times") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1.300000000) + tdSql.checkData(1, 1, 13.000000000) + + tdSql.query(f"select * from {dbname}.`sys_procs_running`") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 42.000000000) + tdSql.checkData(0, 2, "web01") + + tdSql.query(f"select distinct tbname from {dbname}.`sys_cpu_nice`") + tdSql.checkRows(3) + + tdSql.query(f"select * from {dbname}.`sys_cpu_nice` order by times") + tdSql.checkRows(4) + tdSql.checkData(0, 1, 13.000000000) + tdSql.checkData(0, 2, "web01") + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 4, "lga") + + tdSql.checkData(1, 1, 9.000000000) + tdSql.checkData(1, 2, "web02") + tdSql.checkData(3, 3, "t1") + tdSql.checkData(0, 4, "lga") + + tdSql.query(f"select * from {dbname}.macylr") + tdSql.checkRows(2) + + tdSql.query(f"select * from {dbname}.qelhxo") + tdSql.checkRows(5) + + tdSql.query(f"desc {dbname}.macylr") + tdSql.checkRows(25) + + tdSql.query(f"select * from ts3724._stb2") + tdSql.checkRows(1) + + tdSql.query(f"select * from ts3724.stb_2") + tdSql.checkRows(1) + + tdSql.query(f"select * from ts3724.stb2_") + tdSql.checkRows(1) + return + + def run(self): + tdSql.prepare() + self.checkContent() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) + diff --git a/tests/system-test/2-query/systable_func.py b/tests/system-test/2-query/systable_func.py new file mode 100644 index 0000000000000000000000000000000000000000..3df641c72e15c4a3109a97f44886758e7a6dd247 --- /dev/null +++ b/tests/system-test/2-query/systable_func.py @@ -0,0 +1,60 @@ +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + tdSql.prepare() + + tdSql.query(f"select count(`columns`) from `information_schema`.`ins_tables`;") + tdSql.query(f"select sum(`columns`) from `information_schema`.`ins_tables`;") + tdSql.query(f"select min(`columns`) from `information_schema`.`ins_tables`;") + tdSql.query(f"select max(`columns`) from `information_schema`.`ins_tables`;") + tdSql.query(f"select stddev(`columns`) from `information_schema`.`ins_tables`;") + tdSql.query(f"select avg(`columns`) from `information_schema`.`ins_tables`;") + tdSql.query(f"select apercentile(`columns`, 50) from `information_schema`.`ins_tables`;") + tdSql.query(f"select top(`columns`, 3) from `information_schema`.`ins_tables`;") + tdSql.query(f"select bottom(`columns`, 3) from `information_schema`.`ins_tables`;") + tdSql.query(f"select spread(`columns`) from `information_schema`.`ins_tables`;") + tdSql.query(f"select histogram(`columns`, 'user_input', '[1, 3, 5]', 0) from `information_schema`.`ins_tables`;") + tdSql.query(f"select hyperloglog(`columns`) from `information_schema`.`ins_tables`;") + tdSql.query(f"select sample(`columns`, 3) from `information_schema`.`ins_tables`;") + tdSql.query(f"select mode(`columns`) from `information_schema`.`ins_tables`;") + + tdSql.error(f"select unique(`columns`) from `information_schema`.`ins_tables`;") + tdSql.error(f"select tail(`columns`, 3) from `information_schema`.`ins_tables`;") + tdSql.error(f"select leastsquares(`columns`, 1, 1) from `information_schema`.`ins_tables`;") + tdSql.error(f"select elapsed(`columns`) from `information_schema`.`ins_tables`;") + tdSql.error(f"select interp(`columns`) from `information_schema`.`ins_tables` range(0, 1) every(1s) fill(null);") + tdSql.error(f"select percentile(`columns`, 50) from `information_schema`.`ins_tables`;") + tdSql.error(f"select derivative(`columns`, 1s, 0) from `information_schema`.`ins_tables`;") + tdSql.error(f"select irate(`columns`) from `information_schema`.`ins_tables`;") + tdSql.error(f"select last_row(`columns`) from `information_schema`.`ins_tables`;") + tdSql.error(f"select last(`columns`) from `information_schema`.`ins_tables`;") + tdSql.error(f"select first(`columns`) from `information_schema`.`ins_tables`;") + tdSql.error(f"select twa(`columns`) from `information_schema`.`ins_tables`;") + tdSql.error(f"select diff(`columns`) from `information_schema`.`ins_tables`;") + tdSql.error(f"select statecount(`columns`, 'GE', 0) from `information_schema`.`ins_tables`;") + tdSql.error(f"select stateduration(`columns`, 'GE', 0, 1s) from `information_schema`.`ins_tables`;") + tdSql.error(f"select csum(`columns`) from `information_schema`.`ins_tables`;") + tdSql.error(f"select mavg(`columns`, 1) from `information_schema`.`ins_tables`;") + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py index 43321810d0a6672c9a8cccf5377877fd79b204e3..99791db3df1ca7b32cb2ed79197ffd12876df466 100644 --- a/tests/system-test/2-query/tail.py +++ b/tests/system-test/2-query/tail.py @@ -412,6 +412,18 @@ class TDTestCase: tdSql.checkData(0,0,4) tdSql.checkData(1,0,1) + tdSql.query(f"select tail(a, 1) from (select _rowts, first(c2) as a from {dbname}.ct1 group by c2);") + tdSql.checkRows(1) + + tdSql.query(f"select tail(a, 1) from (select _rowts, first(c2) as a from {dbname}.ct1 partition by c2);") + tdSql.checkRows(1) + + tdSql.query(f"select tail(a, 1) from (select _rowts, first(c2) as a from {dbname}.ct1 order by c2);") + tdSql.checkRows(1) + + tdSql.query(f"select tail(a, 1) from (select _rowts, first(c2) as a from {dbname}.ct1 union select _rowts, first(c2) as a from {dbname}.ct1);") + tdSql.checkRows(1) + def check_boundary_values(self, dbname="bound_test"): tdSql.execute(f"drop database if exists {dbname}") diff --git a/tests/system-test/2-query/ts_3405_3398_3423.py b/tests/system-test/2-query/ts_3405_3398_3423.py new file mode 100644 index 0000000000000000000000000000000000000000..dcd8fb5a85711feec23f96d32c39f996e135b860 --- /dev/null +++ b/tests/system-test/2-query/ts_3405_3398_3423.py @@ -0,0 +1,162 @@ +from util.log import * +from util.sql import * +from util.cases import * +from util.sqlset import * +import datetime +import random + +class TDTestCase: + """The test cases are for TS_3398, TS_3405, TS_3423 + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), False) + + def run(self): + """This test case is used to verify the query performance for the merge scans process of + multiple tables join + """ + tdLog.info("Start the test case for ts_3405") + # test case for https://jira.taosdata.com:18080/browse/TS-3405: + # create db + tdSql.execute("CREATE DATABASE IF NOT EXISTS statistics2 REPLICA {} DURATION 14400m KEEP 5256000m,5256000m,5256000m PRECISION 'ms' MINROWS 100 MAXROWS 4096 COMP 2;".format(self.replicaVar)) + tdSql.execute("use statistics2;") + + # create stable + tdSql.execute("CREATE STABLE IF NOT EXISTS statistics2.`pg`(`day` timestamp,`lt_3` int,`c3_3` int,`c6_3` int,`c9_3` int,`c12_3` int,`c15_3` int,`c18_3` int,`c21_3` int,`c24_3` int,`c27_3` int,`ge_3` int) TAGS(`vin` binary(32));") + tdSql.execute("CREATE STABLE IF NOT EXISTS statistics2.`b`(`day` timestamp, `month` int) TAGS(`group_path` binary(32),`vin` binary(32));") + tdSql.execute("CREATE STABLE IF NOT EXISTS statistics2.`g`(`day` timestamp,`run_state` tinyint) TAGS(`vin` binary(32));") + + # insert the data to table + times = 10 + insertRows = 3000 + pg_sql = "insert into d1001 using statistics2.`pg` tags('test') values" + b_sql = "insert into d2001 using statistics2.`b` tags('1#%', 'test') values" + g_sql = "insert into d3001 using statistics2.`g` tags('test') values" + for t in range(times): + for i in range(t * insertRows, t * insertRows + insertRows): + ts = datetime.datetime.strptime('2023-05-01 00:00:00.000', '%Y-%m-%d %H:%M:%S.%f') + datetime.timedelta(seconds=i) + pg_sql += " ('{}', {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {})".format(ts, i, i, i+1, i+2, i+3, i+4, i+5, i+6, i+7, i+8, i+9) + b_sql += " ('{}', {})".format(ts, 5) + g_sql += " ('{}', {})".format(ts, 1) + + tdSql.execute(pg_sql) + tdSql.execute(b_sql) + tdSql.execute(g_sql) + # reset the sql statements + pg_sql = "insert into d1001 using statistics2.`pg` tags('test') values" + b_sql = "insert into d2001 using statistics2.`b` tags('1#%', 'test') values" + g_sql = "insert into d3001 using statistics2.`g` tags('test') values" + tdLog.info("insert %d rows" % (insertRows * times)) + + # execute the sql statements + ret = tdSql.query("SELECT sum(pg.lt_3) es1,sum(pg.c3_3) es2,sum(pg.c6_3) es3,sum(pg.c9_3) es4,sum(pg.c12_3) es5,sum(pg.c15_3) es6,sum(pg.c18_3) es7,sum(pg.c21_3) es8,sum(pg.c24_3) es9,sum(pg.c27_3) es10,sum(pg.ge_3) es11 FROM statistics2.b b,statistics2.pg pg,statistics2.g g WHERE b.`day` = pg.`day` AND b.`day` = g.`day` AND b.vin = pg.vin AND b.vin = g.vin AND b.vin IS NOT NULL AND b.`group_path` LIKE '1#%';") + # check the first query result + if (449985000, 449985000, 450015000, 450045000, 450075000, 450105000, 450135000, 450165000, 450195000, 450225000, 450255000) in tdSql.queryResult: + tdLog.info("first query result is correct") + else: + tdLog.info("first query result is wrong with res: {}".format(str(tdSql.queryResult))) + + ret = tdSql.query("SELECT sum(pg.lt_3) es1, sum(pg.c3_3) es2, sum(pg.c6_3) es3, sum(pg.c9_3) es4, sum(pg.c12_3) es5, sum(pg.c15_3) es6, sum(pg.c18_3) es7, sum(pg.c21_3) es8, sum(pg.c24_3) es9, sum(pg.c27_3) es10, sum(pg.ge_3) es11 FROM (select * from statistics2.b order by day,month) b, (select * from statistics2.pg order by day,lt_3 ) pg, (select * from statistics2.g order by day,run_state) g WHERE b.`day` = pg.`day` AND b.`day` = g.`day` AND b.vin = pg.vin AND b.vin = g.vin AND b.vin IS NOT NULL;") + # check the second query result + if (449985000, 449985000, 450015000, 450045000, 450075000, 450105000, 450135000, 450165000, 450195000, 450225000, 450255000) in tdSql.queryResult: + tdLog.info("second query result is correct") + else: + tdLog.info("second query result is wrong with res: {}".format(str(tdSql.queryResult))) + tdLog.info("Finish the test case for ts_3405 successfully") + + """This test case is used to verify the aliasName of Node structure is not truncated + when sum clause is more than 65 bits. + """ + # test case for https://jira.taosdata.com:18080/browse/TS-3398: + # create db + tdLog.info("Start the test case for ts_3398") + tdSql.execute("CREATE DATABASE IF NOT EXISTS statistics1 REPLICA {} DURATION 14400m KEEP 5256000m,5256000m,5256000m PRECISION 'ms' MINROWS 100 MAXROWS 4096 COMP 2;".format(self.replicaVar)) + tdSql.execute("use statistics1;") + + # create stable + tdSql.execute("CREATE STABLE IF NOT EXISTS statistics1.`g`(`day` timestamp,`run_state` tinyint) TAGS(`vin` binary(32));") + tdSql.execute("CREATE STABLE IF NOT EXISTS statistics1.`b`(`day` timestamp, `total_heart` int) TAGS(`vin` binary(32));") + tdSql.execute("CREATE STABLE IF NOT EXISTS statistics1.`tg`(`day` timestamp,`lt_4177` int,`f30_4177` int, `f35_4177` int) TAGS(`vin` binary(32));") + + # insert the data to table + tdSql.execute("insert into d1001 using statistics1.`g` tags('NJHYNBSAS0000061') values (%s, %d)" % ("'2023-05-01'", 99)) + tdSql.execute("insert into d2001 using statistics1.`b` tags('NJHYNBSAS0000061') values (%s, %d)" % ("'2023-05-01'", 99)) + tdSql.execute("insert into d3001 using statistics1.`tg` tags('NJHYNBSAS0000061') values (%s, %d, %d, %d)" % ("'2023-05-01'", 99, 99, 99)) + + # execute the sql statements + tdSql.query("SELECT b.`day` `day`,sum(CASE WHEN tg.lt_4177 IS NULL THEN 0 ELSE tg.lt_4177 END \ + + CASE WHEN tg.f35_4177 IS NULL THEN 0 ELSE tg.f35_4177 END) / 3600 es0,sum(CASE WHEN tg.lt_4177 \ + IS NULL THEN 0 ELSE tg.lt_4177 END + CASE WHEN tg.f35_4177 IS NULL THEN 0 ELSE tg.f35_4177 \ + END + CASE WHEN tg.f30_4177 IS NULL THEN 0 ELSE tg.f30_4177 END) / 3600 es1 FROM \ + statistics1.b b,statistics1.tg tg,statistics1.g g WHERE b.`day` = tg.`day` AND g.`day` = b.`day` \ + AND b.vin = tg.vin AND b.vin = g.vin AND b.`day` BETWEEN '2023-05-01' AND '2023-05-05' \ + AND b.vin = 'NJHYNBSAS0000061' AND g.vin IS NOT NULL AND b.vin IS NOT NULL AND tg.vin IS NOT NULL \ + GROUP BY b.`day`;") + # check the result + if 0.055 in tdSql.queryResult[0] and 0.0825 in tdSql.queryResult[0]: + tdLog.info("query result is correct") + else: + tdLog.info("query result is wrong") + tdLog.info("Finish the test case for ts_3398 successfully") + + """This test case is used to verify last(*) query result is correct when the data + is group by tag for stable + """ + # test case for https://jira.taosdata.com:18080/browse/TS-3423: + # create db + tdLog.info("Start the test case for ts_3423") + tdSql.execute("CREATE DATABASE IF NOT EXISTS ts_3423 REPLICA {} DURATION 14400m KEEP 5256000m,5256000m,5256000m PRECISION 'ms' MINROWS 100 MAXROWS 4096 COMP 2;".format(self.replicaVar)) + tdSql.execute("use ts_3423;") + + # create stable + tdSql.execute("CREATE STABLE IF NOT EXISTS ts_3423.`st_last`(`ts` timestamp,`n1` int,`n2` float) TAGS(`groupname` binary(32));") + + # insert the data to table + insertRows = 10 + child_table_num = 10 + for i in range(insertRows): + ts = datetime.datetime.strptime('2023-05-01 00:00:00.000', '%Y-%m-%d %H:%M:%S.%f') + datetime.timedelta(seconds=i) + for j in range(child_table_num): + tdSql.execute("insert into {} using ts_3423.`st_last` tags('{}') values ('{}', {}, {})".format("d" + str(j), "group" + str(j), str(ts), str(i+1), random.random())) + tdLog.info("insert %d rows for every child table" % (insertRows)) + + # cache model list + cache_model = ["none", "last_row", "last_value", "both"] + query_res = [] + + # execute the sql statements first + tdSql.query("select `cachemodel` from information_schema.ins_databases where name='ts_3423'") + current_cache_model = tdSql.queryResult[0][0] + tdLog.info("query on cache model {}".format(current_cache_model)) + tdSql.query("select last(*) from st_last group by groupname;") + # save the results + query_res.append(len(tdSql.queryResult)) + # remove the current cache model + cache_model.remove(current_cache_model) + + for item in cache_model: + tdSql.execute("alter database ts_3423 cachemodel '{}';".format(item)) + # execute the sql statements + tdSql.query("select last(*) from st_last group by groupname;") + tdLog.info("query on cache model {}".format(item)) + query_res.append(len(tdSql.queryResult)) + # check the result + res = True if query_res.count(child_table_num) == 4 else False + if res: + tdLog.info("query result is correct and same among different cache model") + else: + tdLog.info("query result is wrong") + tdLog.info("Finish the test case for ts_3423 successfully") + + def stop(self): + # clear the db + tdSql.execute("drop database if exists statistics1;") + tdSql.execute("drop database if exists statistics2;") + tdSql.execute("drop database if exists ts_3423;") + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/unique.py b/tests/system-test/2-query/unique.py index 9b5da50e1f84e41d24a671e2b6efcb854268087c..b3053817d39be223e87c14b9b56b09fea26bb488 100644 --- a/tests/system-test/2-query/unique.py +++ b/tests/system-test/2-query/unique.py @@ -438,6 +438,15 @@ class TDTestCase: tdSql.checkData(0,0,4) tdSql.checkData(1,0,1) + tdSql.query(f"select unique(c1) v from (select _rowts, c1 from {dbname}.ct1 partition by c2)") + tdSql.checkRows(10) + + tdSql.query(f"select unique(c1) v from (select _rowts, c1 from {dbname}.ct1 order by c2)") + tdSql.checkRows(10) + + tdSql.query(f"select unique(c1) v from (select _rowts, c1 from {dbname}.ct1 union all select _rowts, c1 from {dbname}.ct1)") + tdSql.checkRows(10) + # TD-19911 tdSql.error("select unique(mode(12)) from (select _rowts , t1 , tbname from db.stb1 );") tdSql.error("select unique(mode(t1,1)) from (select _rowts , t1 , tbname from db.stb1 );") diff --git a/tests/system-test/3-enterprise/restore/restoreBasic.py b/tests/system-test/3-enterprise/restore/restoreBasic.py new file mode 100644 index 0000000000000000000000000000000000000000..485f896f0576709caefa35ec21cebf76c8e12cdc --- /dev/null +++ b/tests/system-test/3-enterprise/restore/restoreBasic.py @@ -0,0 +1,237 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +from util.dnodes import * +from util.autogen import * +from util.cluster import * + +import random +import os +import subprocess +import shutil +import time + + +class RestoreBasic: + # init + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdSql.init(conn.cursor()) + self.dnodes_num = 5 + + # get from global + # test + self.dnodes = cluster.dnodes + num = len(self.dnodes) + + if num < self.dnodes_num : + tdLog.exit(f" cluster dnode is less than {self.dnodes_num}. num={num}") + + # create data + self.dbname = "db" + self.stable = "st" + self.child_count = 100 + self.insert_rows = 10000 + self.create_data() + + # create data + def create_data(self): + gen = AutoGen() + gen.create_db(self.dbname, 8, 3) + gen.create_stable(self.stable, 5, 10, 8, 8) + gen.create_child(self.stable, "d", self.child_count) + gen.set_batch_size(1000) + gen.insert_data(self.insert_rows) + + tdSql.execute(f"flush database {self.dbname}") + # put some duplicate ts on wal + gen.insert_data(self.insert_rows%100) + + for i in range(self.dnodes_num): + sql = f"create qnode on dnode {i+1}" + tdSql.execute(sql) + + + # status + def check_status_corrent(self): + # query + tdSql.query(f" show {self.dbname}.vgroups") + + # check 8 vgroups + tdSql.checkRows(8) + + # check data corrent + for i in range(8): + leader = False + for j in range(3): + status = tdSql.getData(i, 4 + j*2) + if status == "leader": + leader = True + elif status == "follower": + pass + else: + tdLog.info(f" check vgroups status not leader or follower. i={i} j={j} status={status}") + return False + + # check leader + if leader == False: + tdLog.info(f" check vgroups not found leader i={i} ") + return False + + # info + tdLog.info("check vgroups status successfully.") + return True + + # check data corrent + def check_corrent(self): + # check status + status = False + for i in range(100): + if self.check_status_corrent(): + status = True + break + else: + time.sleep(0.5) + tdLog.info(f"sleep 500ms retry {i} to check status again...") + + if status == False: + tdLog.exit("check vgroups status failed, exit.") + + # check rows count + sql = f"select count(ts) from {self.dbname}.{self.stable}" + tdSql.query(sql) + tdSql.checkData(0, 0, self.child_count* self.insert_rows) + + + # restore dnode + def restore_dnode(self, index): + tdLog.info(f"start restore dnode {index}") + dnode = self.dnodes[index - 1] + + # stop dnode + tdLog.info(f"stop dnode {index}") + dnode.stoptaosd() + + # remove dnode folder + try: + shutil.rmtree(dnode.dataDir) + tdLog.info(f"delete dir {dnode.dataDir} successful") + except OSError as x: + tdLog.exit(f"remove path {dnode.dataDir} error : {x.strerror}") + + dnode.starttaosd() + + # exec restore + sql = f"restore dnode {index}" + tdLog.info(sql) + tdSql.execute(sql) + self.check_corrent() + + # restore vnode + def restore_vnode(self, index): + tdLog.info(f"start restore vnode on dnode {index}") + dnode = self.dnodes[index - 1] + del_dir = f"{dnode.dataDir}/vnode" + + # stop dnode + tdLog.info(f"stop dnode {index}") + dnode.stoptaosd() + + # remove dnode folder + try: + shutil.rmtree(del_dir) + tdLog.info(f"delete dir {del_dir} successful") + except OSError as x: + tdLog.exit(f"remove path {del_dir} error : {x.strerror}") + + dnode.starttaosd() + + # exec restore + sql = f"restore vnode on dnode {index}" + tdLog.info(sql) + tdSql.execute(sql) + + # check result + self.check_corrent() + + + # restore mnode + def restore_mnode(self, index): + tdLog.info(f"start restore mnode {index}") + dnode = self.dnodes[index - 1] + del_dir = f"{dnode.dataDir}/mnode" + + # stop dnode + tdLog.info(f"stop dnode {index}") + dnode.stoptaosd() + + # remove dnode folder + try: + shutil.rmtree(del_dir) + tdLog.info(f"delete dir {del_dir} successful") + except OSError as x: + tdLog.exit(f"remove path {del_dir} error : {x.strerror}") + + dnode.starttaosd() + + # exec restore + sql = f"restore mnode on dnode {index}" + tdLog.info(sql) + tdSql.execute(sql) + self.check_corrent() + + + # restore qnode + def restore_qnode(self, index): + tdLog.info(f"start restore qnode on dnode {index}") + dnode = self.dnodes[index - 1] + del_dir = f"{dnode.dataDir}/qnode" + + # stop dnode + tdLog.info(f"stop dnode {index}") + dnode.stoptaosd() + + # remove dnode folder + try: + shutil.rmtree(del_dir) + tdLog.info(f"delete dir {del_dir} successful") + except OSError as x: + tdLog.exit(f"remove path {del_dir} error : {x.strerror}") + + # start dnode + dnode.starttaosd() + + # exec restore + sql = f"restore qnode on dnode {index}" + tdLog.info(sql) + tdSql.execute(sql) + self.check_corrent() + + # path exist + qfile = f"{del_dir}/qnode.json" + if os.path.exists(qfile) == False: + tdLog.exit(f"qnode restore failed. qnode.json is not exist. {qfile}") + else: + tdLog.info(f"check qnode.json restore ok. {qfile}") + + # stop + def stop(self): + tdSql.close() + + diff --git a/tests/system-test/3-enterprise/restore/restoreDnode.py b/tests/system-test/3-enterprise/restore/restoreDnode.py new file mode 100644 index 0000000000000000000000000000000000000000..b92c823764734bdac659abe2381f7bdc3d50598f --- /dev/null +++ b/tests/system-test/3-enterprise/restore/restoreDnode.py @@ -0,0 +1,41 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +sys.path.append("./3-enterprise/restore") +from restoreBasic import * + + +class TDTestCase: + # init + def init(self, conn, logSql, replicaVar=1): + tdLog.debug("start to execute %s" % __file__) + self.basic = RestoreBasic() + self.basic.init(conn, logSql, replicaVar) + + # run + def run(self): + self.basic.restore_dnode(2) + + # stop + def stop(self): + self.basic.stop() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/3-enterprise/restore/restoreMnode.py b/tests/system-test/3-enterprise/restore/restoreMnode.py new file mode 100644 index 0000000000000000000000000000000000000000..3f3ccb8a5ff7d25e0df817261196032d0f69dda9 --- /dev/null +++ b/tests/system-test/3-enterprise/restore/restoreMnode.py @@ -0,0 +1,42 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +sys.path.append("./3-enterprise/restore") +from restoreBasic import * + + + +class TDTestCase: + # init + def init(self, conn, logSql, replicaVar=1): + tdLog.debug("start to execute %s" % __file__) + self.basic = RestoreBasic() + self.basic.init(conn, logSql, replicaVar) + + # run + def run(self): + self.basic.restore_mnode(3) + + # stop + def stop(self): + self.basic.stop() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/3-enterprise/restore/restoreQnode.py b/tests/system-test/3-enterprise/restore/restoreQnode.py new file mode 100644 index 0000000000000000000000000000000000000000..d2136523e8fdded5e75a61f118731fc505047eb8 --- /dev/null +++ b/tests/system-test/3-enterprise/restore/restoreQnode.py @@ -0,0 +1,41 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +sys.path.append("./3-enterprise/restore") +from restoreBasic import * + + +class TDTestCase: + # init + def init(self, conn, logSql, replicaVar=1): + tdLog.debug("start to execute %s" % __file__) + self.basic = RestoreBasic() + self.basic.init(conn, logSql, replicaVar) + + # run + def run(self): + self.basic.restore_qnode(5) + + # stop + def stop(self): + self.basic.stop() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/3-enterprise/restore/restoreVnode.py b/tests/system-test/3-enterprise/restore/restoreVnode.py new file mode 100644 index 0000000000000000000000000000000000000000..9f5b7f56068a72ea8144ef3a1f8f97f22dc6c867 --- /dev/null +++ b/tests/system-test/3-enterprise/restore/restoreVnode.py @@ -0,0 +1,41 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +sys.path.append("./3-enterprise/restore") +from restoreBasic import * + + +class TDTestCase: + # init + def init(self, conn, logSql, replicaVar=1): + tdLog.debug("start to execute %s" % __file__) + self.basic = RestoreBasic() + self.basic.init(conn, logSql, replicaVar) + + # run + def run(self): + self.basic.restore_vnode(4) + + # stop + def stop(self): + self.basic.stop() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/5-taos-tools/taosbenchmark/insertMix.py b/tests/system-test/5-taos-tools/taosbenchmark/insertMix.py new file mode 100644 index 0000000000000000000000000000000000000000..b4046b8c98b311d8f73fc811cf4e70a2d2510e38 --- /dev/null +++ b/tests/system-test/5-taos-tools/taosbenchmark/insertMix.py @@ -0,0 +1,107 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +import os +import subprocess +import time + +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def caseDescription(self): + """ + [TD-13823] taosBenchmark test cases + """ + return + + def init(self, conn, logSql, replicaVar=1): + # comment off by Shuduo for CI self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] + else: + projPath = selfPath[: selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if (tool) in files: + rootRealPath = os.path.dirname(os.path.realpath(root)) + if "packaging" not in rootRealPath: + paths.append(os.path.join(root, tool)) + break + if len(paths) == 0: + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def checkDataCorrect(self): + sql = "select count(*) from meters" + tdSql.query(sql) + allCnt = tdSql.getData(0, 0) + if allCnt < 2000000: + tdLog.exit(f"taosbenchmark insert row small. row count={allCnt} sql={sql}") + return + + # group by 10 child table + rowCnt = tdSql.query("select count(*),tbname from meters group by tbname") + tdSql.checkRows(10) + + # interval + sql = "select count(*),max(ic),min(dc),last(*) from meters interval(1s)" + rowCnt = tdSql.query(sql) + if rowCnt < 10: + tdLog.exit(f"taosbenchmark interval(1s) count small. row cout={rowCnt} sql={sql}") + return + + # nest query + tdSql.query("select count(*) from (select * from meters order by ts desc)") + tdSql.checkData(0, 0, allCnt) + + rowCnt = tdSql.query("select tbname, count(*) from meters partition by tbname slimit 11") + if rowCnt != 10: + tdLog.exit("partition by tbname should return 10 rows of table data which is " + str(rowCnt)) + return + + + def run(self): + binPath = self.getPath() + cmd = "%s -f ./5-taos-tools/taosbenchmark/json/insertMix.json" % binPath + tdLog.info("%s" % cmd) + errcode = os.system("%s" % cmd) + if errcode != 0: + tdLog.exit(f"execute taosBenchmark ret error code={errcode}") + return + + tdSql.execute("use mixdb") + self.checkDataCorrect() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/5-taos-tools/taosbenchmark/json/insertMix.json b/tests/system-test/5-taos-tools/taosbenchmark/json/insertMix.json new file mode 100644 index 0000000000000000000000000000000000000000..7f3b2103cc607c53233acb04687a209612632e5c --- /dev/null +++ b/tests/system-test/5-taos-tools/taosbenchmark/json/insertMix.json @@ -0,0 +1,81 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 3000, + "thread_count": 10, + "create_table_thread_count": 2, + "result_file": "./insert_res_mix.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "check_sql": "yes", + "continue_if_fail": "no", + "databases": [ + { + "dbinfo": { + "name": "mixdb", + "drop": "yes", + "vgroups": 6, + "replica": 3, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 10, + "insert_rows": 300000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "insert_interval": 0, + "timestamp_step": 100, + "start_timestamp":1500000000000, + "disorder_ratio": 10, + "update_ratio": 5, + "delete_ratio": 1, + "disorder_fill_interval": 300, + "update_fill_interval": 25, + "generate_row_rule": 2, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 1, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": 0 }, + { "type": "smallint", "name": "si", "max": 100, "min": 0 }, + { "type": "int", "name": "ic", "max": 100, "min": 0 }, + { "type": "bigint", "name": "bi", "max": 100, "min": 0 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 100, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 100, "min": 0 }, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + { + "type": "tinyint", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "binary", + "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/system-test/5-taos-tools/taosbenchmark/json/stt.json b/tests/system-test/5-taos-tools/taosbenchmark/json/stt.json new file mode 100644 index 0000000000000000000000000000000000000000..27f32010ed5d07647b76c7d2ea797c809101aaf3 --- /dev/null +++ b/tests/system-test/5-taos-tools/taosbenchmark/json/stt.json @@ -0,0 +1,81 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 3000, + "thread_count": 20, + "create_table_thread_count": 5, + "result_file": "./insert_res_wal.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "flush_each_batch": "yes", + "vgroups": 2, + "replica": 1, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 1000, + "insert_rows": 2850, + "childtable_prefix": "d", + "insert_mode": "taosc", + "insert_interval": 0, + "timestamp_step": 10, + "disorder_ratio": 10, + "update_ratio": 5, + "delete_ratio": 1, + "disorder_fill_interval": 30, + "update_fill_interval": 25, + "generate_row_rule": 2, + "start_timestamp":"2022-01-01 10:00:00", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 1, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": 0 }, + { "type": "smallint", "name": "si", "max": 100, "min": 0 }, + { "type": "int", "name": "ic", "max": 100, "min": 0 }, + { "type": "bigint", "name": "bi", "max": 100, "min": 0 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 100, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 100, "min": 0 }, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + { + "type": "tinyint", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "binary", + "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} + diff --git a/tests/system-test/5-taos-tools/taosbenchmark/stt.py b/tests/system-test/5-taos-tools/taosbenchmark/stt.py new file mode 100644 index 0000000000000000000000000000000000000000..9b86bd8e40b22f22959e9d4b218caf5a210b7f9e --- /dev/null +++ b/tests/system-test/5-taos-tools/taosbenchmark/stt.py @@ -0,0 +1,102 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +import os +import subprocess +import time + +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def caseDescription(self): + """ + [TD-13823] taosBenchmark test cases + """ + return + + def init(self, conn, logSql, replicaVar=1): + # comment off by Shuduo for CI self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] + else: + projPath = selfPath[: selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if (tool) in files: + rootRealPath = os.path.dirname(os.path.realpath(root)) + if "packaging" not in rootRealPath: + paths.append(os.path.join(root, tool)) + break + if len(paths) == 0: + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def checkDataCorrect(self): + sql = "select count(*) from meters" + tdSql.query(sql) + allCnt = tdSql.getData(0, 0) + if allCnt < 2000000: + tdLog.exit(f"taosbenchmark insert row small. row count={allCnt} sql={sql}") + return + + # group by 10 child table + rowCnt = tdSql.query("select count(*),tbname from meters group by tbname") + tdSql.checkRows(1000) + + # interval + sql = "select count(*),max(ic),min(dc),last(*) from meters interval(1s)" + rowCnt = tdSql.query(sql) + if rowCnt < 10: + tdLog.exit(f"taosbenchmark interval(1s) count small. row cout={rowCnt} sql={sql}") + return + + # nest query + tdSql.query("select count(*) from (select * from meters order by ts desc)") + tdSql.checkData(0, 0, allCnt) + + + def run(self): + binPath = self.getPath() + cmd = "%s -f ./5-taos-tools/taosbenchmark/json/stt.json" % binPath + tdLog.info("%s" % cmd) + errcode = os.system("%s" % cmd) + if errcode != 0: + tdLog.exit(f"execute taosBenchmark ret error code={errcode}") + return + + tdSql.execute("use db") + self.checkDataCorrect() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/5dnode2mnode.py b/tests/system-test/6-cluster/5dnode2mnode.py index 6054ef69f8738d52cd5547ed00a74537e0ddcbc1..ca7d6a58d54dc211b17a05f3ded9e6691bef58e7 100644 --- a/tests/system-test/6-cluster/5dnode2mnode.py +++ b/tests/system-test/6-cluster/5dnode2mnode.py @@ -35,7 +35,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py b/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py index 8a5f43e1f9e28dec28b495ee3a08b951dff1acb0..223b91be6f19836fb391ae678b5f509af0865311 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py +++ b/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py @@ -43,7 +43,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeDrop.py b/tests/system-test/6-cluster/5dnode3mnodeDrop.py index c4b30a16f71ba1331fd3f146aa3245375326588e..aefa7a09f8984a78ebb6e981e61e2960f613cdaf 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeDrop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeDrop.py @@ -47,7 +47,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeDropInsert.py b/tests/system-test/6-cluster/5dnode3mnodeDropInsert.py index 01d08ee8392e125e9f7ffee71b61229f8a8d43b7..db183d80c164a56e53f0bbe35271dfb9a98de478 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeDropInsert.py +++ b/tests/system-test/6-cluster/5dnode3mnodeDropInsert.py @@ -48,7 +48,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py b/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py index 41082baa3da6161d378e66923a0473d3e2c1951f..650bc347aa48f7e3c9439452b06d01d7a621a28c 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py @@ -42,7 +42,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -136,7 +136,7 @@ class TDTestCase: tdLog.info("Confirm the status of the dnode again") tdSql.error("create mnode on dnode 2") tdSql.query("select * from information_schema.ins_dnodes;") - print(tdSql.queryResult) + # print(tdSql.queryResult) clusterComCheck.checkDnodes(dnodeNumbers) # recreate mnode @@ -160,80 +160,38 @@ class TDTestCase: if tdSql.queryResult[i][0] == "%s"%username : tdLog.info("create user:%s successfully"%username) - # # create database and stable - # clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) - # tdLog.info("Take turns stopping Mnodes ") - - # tdDnodes=cluster.dnodes - # stopcount =0 - # threads=[] - - # # create stable:stb_0 - # stableName= paraDict['stbName'] - # newTdSql=tdCom.newTdSql() - # clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']) - # #create child table:ctb_0 - # for i in range(paraDict['stbNumbers']): - # stableName= '%s_%d'%(paraDict['stbName'],i) - # newTdSql=tdCom.newTdSql() - # clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum']) - # #insert date - # for i in range(paraDict['stbNumbers']): - # stableName= '%s_%d'%(paraDict['stbName'],i) - # newTdSql=tdCom.newTdSql() - # threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) - # for tr in threads: - # tr.start() - # for tr in threads: - # tr.join() - - # while stopcount < restartNumbers: - # tdLog.info(" restart loop: %d"%stopcount ) - # if stopRole == "mnode": - # for i in range(mnodeNums): - # tdDnodes[i].stoptaosd() - # # sleep(10) - # tdDnodes[i].starttaosd() - # # sleep(10) - # elif stopRole == "vnode": - # for i in range(vnodeNumbers): - # tdDnodes[i+mnodeNums].stoptaosd() - # # sleep(10) - # tdDnodes[i+mnodeNums].starttaosd() - # # sleep(10) - # elif stopRole == "dnode": - # for i in range(dnodeNumbers): - # tdDnodes[i].stoptaosd() - # # sleep(10) - # tdDnodes[i].starttaosd() - # # sleep(10) - - # # dnodeNumbers don't include database of schema - # if clusterComCheck.checkDnodes(dnodeNumbers): - # tdLog.info("dnode is ready") - # else: - # print("dnodes is not ready") - # self.stopThread(threads) - # tdLog.exit("one or more of dnodes failed to start ") - # # self.check3mnode() - # stopcount+=1 - - - # clusterComCheck.checkDnodes(dnodeNumbers) - # clusterComCheck.checkDbRows(dbNumbers) - # # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"]) - - # tdSql.execute("use %s" %(paraDict["dbName"])) - # tdSql.query("show stables") - # tdSql.checkRows(paraDict["stbNumbers"]) - # # for i in range(paraDict['stbNumbers']): - # # stableName= '%s_%d'%(paraDict['stbName'],i) - # # tdSql.query("select * from %s"%stableName) - # # tdSql.checkRows(rowsPerStb) + """ case for TS-3524 and test 'taos -h' """ + bPath = self.getBuildPath() + for i in range(6): + nodePort = 6030 + i*100 + newTdSql=tdCom.newTdSql(port=nodePort) + + dataPath = tdDnodes[1].dataDir + os.system(f"rm -rf {dataPath}/*") + os.system(f"rm -rf {dataPath}/.runing") + + tdDnodes[1].stoptaosd() + tdDnodes[1].starttaosd() + sleep(5) + for i in range(6): + nodePort = 6030 + i*100 + newTdSql=tdCom.newTdSql(port=nodePort) + + dataPath = tdDnodes[0].dataDir + os.system(f"rm -rf {dataPath}/*") + os.system(f"rm -rf {dataPath}/.runing") + + tdDnodes[0].stoptaosd() + tdDnodes[0].starttaosd() + sleep(5) + for i in range(6): + nodePort = 6030 + i*100 + newTdSql=tdCom.newTdSql(port=nodePort) + def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=1,stopRole='dnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='dnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py index 392b0d7764c4404a4e2282b7bf5708a79556acf8..b96a9b8175100a1d0307ff10c196bdd701cb4104 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py @@ -43,7 +43,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py index 04c69ad6188a657cd37651a01d417d790a9830af..da16d39ac225db38f8ae75c2de7e1d1f1f93c129 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py @@ -43,7 +43,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeRoll.py b/tests/system-test/6-cluster/5dnode3mnodeRoll.py new file mode 100644 index 0000000000000000000000000000000000000000..38ac47f777171c7ae32f600bc40c0cfe2e756ea6 --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeRoll.py @@ -0,0 +1,364 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +from numpy import row_stack +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +from util.common import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck +from pathlib import Path +from taos.tmq import Consumer + + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import json + +BASEVERSION = "3.1.0.0" + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.replicaVar = int(replicaVar) + self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300;use deldata; + create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int); + create table deldata.ct1 using deldata.stb1 tags ( 1 ); + insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); + select avg(c1) from deldata.ct1; + delete from deldata.stb1; + flush database deldata; + insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); + delete from deldata.ct1;''' + + def checkProcessPid(self,processName): + i=0 + while i<60: + print(f"wait stop {processName}") + processPid = subprocess.getstatusoutput(f'ps aux|grep {processName} |grep -v "grep"|awk \'{{print $2}}\'')[1] + print(f"times:{i},{processName}-pid:{processPid}") + if(processPid == ""): + break + i += 1 + sleep(1) + else: + print(f'this processName is not stoped in 60s') + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def getCfgPath(self): + buildPath = self.getBuildPath() + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + cfgPath = buildPath + "/../sim/dnode1/cfg/" + else: + cfgPath = buildPath + "/../sim/dnode1/cfg/" + + return cfgPath + + def installTaosd(self,bPath,cPath): + # os.system(f"rmtaos && mkdir -p {self.getBuildPath()}/build/lib/temp && mv {self.getBuildPath()}/build/lib/libtaos.so* {self.getBuildPath()}/build/lib/temp/ ") + # os.system(f" mv {bPath}/build {bPath}/build_bak ") + # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so {self.getBuildPath()}/build/lib/libtaos.so_bak ") + # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so.1 {self.getBuildPath()}/build/lib/libtaos.so.1_bak ") + + packagePath = "/usr/local/src/" + dataPath = cPath + "/../data/" + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" + packageTPath = packageName.split("-Linux-")[0] + my_file = Path(f"{packagePath}/{packageName}") + if not my_file.exists(): + print(f"{packageName} is not exists") + tdLog.info(f"cd {packagePath} && wget https://www.tdengine.com/assets-download/3.0/{packageName}") + os.system(f"cd {packagePath} && wget https://www.tdengine.com/assets-download/3.0/{packageName}") + else: + print(f"{packageName} has been exists") + os.system(f" cd {packagePath} && tar xvf {packageName} && cd {packageTPath} && ./install.sh -e no " ) + # tdDnodes.stop(1) + # print(f"start taosd: rm -rf {dataPath}/* && nohup taosd -c {cPath} & ") + # os.system(f"rm -rf {dataPath}/* && nohup taosd -c {cPath} & " ) + # sleep(5) + + + def buildTaosd(self,bPath): + # os.system(f"mv {bPath}/build_bak {bPath}/build ") + os.system(f" cd {bPath}/ && make install ") + + def is_list_same_as_ordered_list(self,unordered_list, ordered_list): + sorted_list = sorted(unordered_list) + return sorted_list == ordered_list + + def insertAllData(self,cPath,dbname,tableNumbers,recordNumbers): + tdLog.info(f"insertAllData") + # tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -d dbtest -t {tableNumbers} -c {cPath} -n {recordNumbers} -v 2 -a 3 -y -k 10 -z 5 ") + # os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -d dbtest -t {tableNumbers} -c {cPath} -n {recordNumbers} -v 2 -a 3 -y -k 10 -z 5 ") + + print(f"sed -i 's/\"cfgdir\".*/\"cfgdir\": \"{cPath}\",/' 6-cluster/rollup.json && sed -i '0,/\"name\":.*/s/\"name\":.*/\"name\": \"{dbname}\",/' 6-cluster/rollup.json && sed -i 's/\"childtable_count\":.*/\"childtable_count\": {tableNumbers},/' 6-cluster/rollup.json && sed -i 's/\"insert_rows\":.*/\"insert_rows\": {recordNumbers},/' 6-cluster/rollup.json" ) + os.system(f"sed -i 's/\"cfgdir\".*/\"cfgdir\": \"{cPath}\",/' 6-cluster/rollup.json && sed -i '0,/\"name\":.*/s/\"name\":.*/\"name\": \"{dbname}\",/' 6-cluster/rollup.json && sed -i 's/\"childtable_count\":.*/\"childtable_count\": {tableNumbers},/' 6-cluster/rollup.json && sed -i 's/\"insert_rows\":.*/\"insert_rows\": {recordNumbers},/' 6-cluster/rollup.json") + print("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 6-cluster/rollup.json -y -k 10 -z 5") + os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 6-cluster/rollup.json -y -k 10 -z 5 ") + + + def insertData(self,countstart,countstop): + # fisrt add data : db\stable\childtable\general table + + for couti in range(countstart,countstop): + tdLog.debug("drop database if exists db%d" %couti) + tdSql.execute("drop database if exists db%d" %couti) + print("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("use db%d" %couti) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db0_0', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'stbNumbers': 2, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 200, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + "rowsPerTbl": 1000, + "batchNum": 5000 + } + hostname = socket.gethostname() + dnodeNumbers=int(dnodeNumbers) + + tdLog.info("first check dnode and mnode") + tdSql=tdCom.newTdSql() + tdSql.query("select * from information_schema.ins_dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + + tdLog.printNoPrefix(f"==========step1:prepare cluster of {dnodeNumbers} dnodes whith old version-{BASEVERSION} ") + + scriptsPath = os.path.dirname(os.path.realpath(__file__)) + distro_id = distro.id() + if distro_id == "alpine": + tdLog.info(f"alpine skip Roll test") + return True + if platform.system().lower() == 'windows': + tdLog.info(f"Windows skip Roll test") + return True + + tdLog.info("====step1.1:stop all taosd and clear data dir,then start all old taosd ====") + + bPath = self.getBuildPath() + cPath = self.getCfgPath() + tdDnodes=cluster.dnodes + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + self.installTaosd(bPath,cPath) + for i in range(dnodeNumbers): + dnode_cfgPath = tdDnodes[i].cfgDir + dnode_dataPath = tdDnodes[i].dataDir + os.system(f"rm -rf {dnode_dataPath}/* && nohup taosd -c {dnode_cfgPath} & ") + + tdLog.info("====step1.2: create dnode on cluster ====") + + for i in range(1,dnodeNumbers): + dnode_id = tdDnodes[i].cfgDict["fqdn"] + ":" + tdDnodes[i].cfgDict["serverPort"] + os.system(f" LD_LIBRARY_PATH=/usr/lib taos -s 'create dnode \"{dnode_id}\" ' ") + sleep(5) + os.system(" LD_LIBRARY_PATH=/usr/lib taos -s 'show dnodes' ") + + for i in range(2,dnodeNumbers+1): + os.system(f" LD_LIBRARY_PATH=/usr/lib taos -s 'create mnode on dnode {i} ' ") + sleep(10) + os.system(" LD_LIBRARY_PATH=/usr/lib taos -s 'show mnodes' ") + + tdLog.info("====step1.3: insert data, includes time data, tmq and stream ====") + tableNumbers1=100 + recordNumbers1=100000 + recordNumbers2=1000 + + dbname = "dbtest" + stb = f"{dbname}.meters" + cPath_temp=cPath.replace("/","\/") + + # os.system("echo 'debugFlag 143' > /etc/taos/taos.cfg ") + # create database and tables + print(f"sed -i 's/\"cfgdir\".*/\"cfgdir\": \"{cPath_temp}\",/' 6-cluster/rollup_db.json && sed -i '0,/\"name\":.*/s/\"name\":.*/\"name\": \"{dbname}\",/' 6-cluster/rollup_db.json ") + os.system(f"sed -i 's/\"cfgdir\".*/\"cfgdir\": \"{cPath_temp}\",/' 6-cluster/rollup_db.json && sed -i '0,/\"name\":.*/s/\"name\":.*/\"name\": \"{dbname}\",/' 6-cluster/rollup_db.json") + print("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 6-cluster/rollup_db.json -y ") + os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 6-cluster/rollup_db.json -y") + # insert data + tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -d test -t {tableNumbers1} -c {cPath} -n {recordNumbers2} -v 2 -a 3 -y -k 10 -z 5 ") + os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -d test -t {tableNumbers1} -c {cPath} -n {recordNumbers2} -v 2 -a 3 -y -k 10 -z 5 ") + + # os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ") + # os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ') + # os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "use test;show streams;" ') + os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "alter database test WAL_RETENTION_PERIOD 1000" ') + os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists tmq_test_topic as select current,voltage,phase from test.meters where voltage <= 106 and current <= 5;" ') + os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "use test;show topics;" ') + + print(f"sed -i 's/\"cfgdir\".*/\"cfgdir\": \"{cPath_temp}\",/' 0-others/compa4096.json ") + os.system(f"sed -i 's/\"cfgdir\".*/\"cfgdir\": \"{cPath_temp}\",/'0-others/compa4096.json ") + tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y -k 10 -z 5 ") + os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y -k 10 -z 5 ") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -f 0-others/TS-3131.tsql") + # self.buildTaosd(bPath) + + # add deleted data + os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "{self.deletedDataSql}" ') + + threads=[] + threads.append(threading.Thread(target=self.insertAllData, args=(cPath_temp,dbname,tableNumbers1,recordNumbers1))) + for tr in threads: + tr.start() + # when inserting data porcess has been started up ,we can upgrade taosd + sleep(5) + tdLog.printNoPrefix("==========step2:start to rolling upgdade ") + for i in range(dnodeNumbers): + tdDnodes[i].running = 1 + tdDnodes[i].stoptaosd() + sleep(2) + tdDnodes[i].starttaosd() + + for tr in threads: + tr.join() + + tdLog.printNoPrefix(f"==========step3:check dnode status ") + # wait 10s for taosd cluster ready + sleep(10) + tdsql=tdCom.newTdSql() + tdsql.query("select * from information_schema.ins_dnodes;") + tdLog.info(tdsql.queryResult) + tdsql.checkData(2,1,'%s:6230'%self.host) + clusterComCheck.checkDnodes(dnodeNumbers) + + tdsql1=tdCom.newTdSql() + tdsql1.query(f"SELECT SERVER_VERSION();") + nowServerVersion=tdsql1.queryResult[0][0] + tdLog.printNoPrefix(f"==========step4:prepare and check data in new version-{nowServerVersion}") + + tdLog.info(f"New server version is {nowServerVersion}") + tdsql1.query(f"SELECT CLIENT_VERSION();") + nowClientVersion=tdsql1.queryResult[0][0] + tdLog.info(f"New client version is {nowClientVersion}") + + tdsql1.query(f"select count(*) from {stb}") + tdsql1.checkData(0,0,tableNumbers1*recordNumbers1) + tdsql1.query(f"select count(*) from db4096.stb0") + tdsql1.checkData(0,0,50000) + + # checkout deleted data + tdsql.execute("insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a );") + tdsql.query("flush database deldata;select avg(c1) from deldata.ct1;") + + + # tdsql1.query("show streams;") + # tdsql1.checkRows(2) + tdsql1.query("select *,tbname from d0.almlog where mcid='m0103';") + tdsql1.checkRows(6) + expectList = [0,3003,20031,20032,20033,30031] + resultList = [] + for i in range(6): + resultList.append(tdsql1.queryResult[i][3]) + print(resultList) + if self.is_list_same_as_ordered_list(resultList,expectList): + print("The unordered list is the same as the ordered list.") + else: + tdlog.error("The unordered list is not the same as the ordered list.") + tdsql1.execute(f"insert into test.d80 values (now+1s, 11, 103, 0.21);") + tdsql1.execute(f"insert into test.d9 values (now+5s, 4.3, 104, 0.4);") + + conn = taos.connect() + + consumer = Consumer( + { + "group.id": "tg75", + "client.id": "124", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "enable.auto.commit": "true", + "experimental.snapshot.enable": "true", + } + ) + consumer.subscribe(["tmq_test_topic"]) + + while True: + res = consumer.poll(10) + if not res: + break + err = res.error() + if err is not None: + raise err + val = res.value() + + for block in val: + print(block.fetchall()) + tdsql1.query("show topics;") + tdsql1.checkRows(1) + + + # #check mnode status + # tdLog.info("check mnode status") + # clusterComCheck.checkMnodeStatus(mnodeNums) + + + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=3,mnodeNums=3,restartNumbers=2,stopRole='dnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py index b55c689eee88ef6f4f19893f6d6bf18a91d6acc4..3a972ff4e951f9e51434c27baec427d835f76abf 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py index 97e61950372e2a79e8e94f0de296009351d701a0..2a8f4fd52654c75334f0c7bed232eb7bd8acefe6 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py index 296e9daecaf16d3ef4016bafa7454ff0c7d62af9..7eaf756737e38ff5e74e0172350ec30d70857f60 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py @@ -42,7 +42,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py index 06d626b77cc8d9e67f1ecbe2fc524a1d9a69decc..c1c47fd55cdefff3fd1837b16bca5aa371a6beba 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py @@ -42,7 +42,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeRCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeRCreateDb.py index 9d99980b88081607d825f8f9198bb3d0487906df..27b15d4c990e4594a9b046649938543b554611f4 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeRCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeRCreateDb.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py index 15d18d50909fde41aefb12fcfb6cb665f9a3eeb0..8fe3b24d3b2c99acc3b5d252bc5ae819b60fcecf 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py index 98842e3358fb29c19a59fcb671848168ba9a2f80..8d483919a551ec48f4df33702cdf1de41a455ad8 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py index cb160595242ae33b00feafbe6d5b57c3ae2c3e5b..9395dd2a2b74f857bf5a512a23b50bcddaab1e04 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py index 3e4dc2483f2222fe9f5cd04c2e4dc529d1e83c7d..0522a72d38c05d2f9bcab3cd092b731518144283 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py @@ -42,7 +42,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py index 21caf23ea63d641ab42d8aa2596416c6b7f01729..e39855b42ebe010970865577f08e8db9cf8e45d8 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py @@ -42,7 +42,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py index 84236529d146584b0305747ae1c93b484250b52c..2fb196635fb9fe1b7a0b304fe5b7418871bf9bed 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeSepVnodeStopDnodeCreateUser.py b/tests/system-test/6-cluster/5dnode3mnodeSepVnodeStopDnodeCreateUser.py index 94e02b77b307281975ff7393298f182f233a7c65..bcc7edf5cbbc2f4105f2b5f9eef18bf612f000c3 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSepVnodeStopDnodeCreateUser.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSepVnodeStopDnodeCreateUser.py @@ -43,7 +43,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py index 522ba4c2fcfd713853e9fe2313c818507f5fa792..8e76033c27278f7b2588eef44b3028723b7652b8 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py @@ -39,7 +39,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py b/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py index 0596dd84edf2811ca382a58c75d642f9c1336fdb..e89df638d07d3f858a74cd42d8ee81ff834a5ad9 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py @@ -39,7 +39,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeStopConnect.py b/tests/system-test/6-cluster/5dnode3mnodeStopConnect.py index 2c735ed9b6230bbc52b446fcf0feba15fb9e1a44..a87cd23b381451330861ca387668f8d5dd591d7a 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStopConnect.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStopConnect.py @@ -39,7 +39,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeStopFollowerLeader.py b/tests/system-test/6-cluster/5dnode3mnodeStopFollowerLeader.py index d7176e142fa27007ad2d4940f0f8eb69078eb815..a8ebfbace5a1d6d914d8f990b48e5fa219b3362d 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStopFollowerLeader.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStopFollowerLeader.py @@ -39,7 +39,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeStopInsert.py b/tests/system-test/6-cluster/5dnode3mnodeStopInsert.py index d08ce79a9bb7dd9f6b21f926c3860575bee93931..9d2430506fd5f3286551931468351d3b0a0368e1 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStopInsert.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStopInsert.py @@ -47,7 +47,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py b/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py index 52d61fb529b9893ba1de5f1594206eb421b4ffd4..11869f8ee4eb17cd7aa82c92fa3f622e3e01b57d 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py @@ -39,7 +39,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py index 7aa2ba06b9a38590bdb252b05891f9c6fa391640..439f0b6b8c93ee78e0fe1f2341a8426715c3ddab 100644 --- a/tests/system-test/6-cluster/clusterCommonCheck.py +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -256,12 +256,12 @@ class ClusterComCheck: if vgroup_status_first.count('leader') == 1 and vgroup_status_first.count('follower') == 2: if vgroup_status_last.count('leader') == 1 and vgroup_status_last.count('follower') == 2: ready_time= (count + 1) - tdLog.success(f"elections of {db_name} all vgroups are ready in {ready_time} s") + tdLog.success(f"elections of {db_name}.vgroups are ready in {ready_time} s") return True count+=1 else: tdLog.debug(tdSql.queryResult) - tdLog.notice(f"elections of {db_name} all vgroups are failed in{count}s ") + tdLog.notice(f"elections of {db_name} all vgroups are failed in{count} s ") caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno) tdLog.exit("%s(%d) failed " % args) diff --git a/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDataRebootAlterRep1-3.py b/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDataRebootAlterRep1-3.py index aa3ed8e3fd4596f0f9f6afa98f43d10101265b96..0d3b920bb484d8723543bae6036a56c4c8cd7d50 100644 --- a/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDataRebootAlterRep1-3.py +++ b/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDataRebootAlterRep1-3.py @@ -40,7 +40,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDataRebootModifyMetaAlterRep1to3.py b/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDataRebootModifyMetaAlterRep1to3.py index 7d46b3143d26ad99b4e02c92518a02af0d3be2db..06636c1ae9926e60e5324b74ba6b047cf4951fab 100644 --- a/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDataRebootModifyMetaAlterRep1to3.py +++ b/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDataRebootModifyMetaAlterRep1to3.py @@ -40,7 +40,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDataRebootModifyMetaAlterRep3to1.py b/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDataRebootModifyMetaAlterRep3to1.py index 5b5fb04969468e6febe91feb8263c8045b1e64eb..40b2291548086697117370176fa54b4b78251873 100644 --- a/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDataRebootModifyMetaAlterRep3to1.py +++ b/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDataRebootModifyMetaAlterRep3to1.py @@ -40,7 +40,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDatarRebootAlterRep1-3.py b/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDatarRebootAlterRep1-3.py index aa3ed8e3fd4596f0f9f6afa98f43d10101265b96..0d3b920bb484d8723543bae6036a56c4c8cd7d50 100644 --- a/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDatarRebootAlterRep1-3.py +++ b/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDatarRebootAlterRep1-3.py @@ -40,7 +40,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py b/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py index fede19ca3ae2adee02acc00b88a71b860da0fddc..fb9872a8f6bd0c49e6a57c33bee7dfe216295922 100644 --- a/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py +++ b/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py @@ -40,7 +40,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -182,7 +182,7 @@ class TDTestCase: tdLog.info(f"show transactions;alter database db0_0 replica {replica3};") TdSqlEx.execute(f'show transactions;') TdSqlEx.execute(f'alter database db0_0 replica {replica3};') - clusterComCheck.check_vgroups_status(vgroup_numbers=paraDict["vgroups"],db_replica=replica3,db_name=paraDict["dbName"],count_number=120) + clusterComCheck.check_vgroups_status(vgroup_numbers=paraDict["vgroups"],db_replica=replica3,db_name=paraDict["dbName"],count_number=180) def run(self): # print(self.master_dnode.cfgDict) diff --git a/tests/system-test/6-cluster/manually-test/6dnode3mnodeStopDnodeInsertDatatb.py b/tests/system-test/6-cluster/manually-test/6dnode3mnodeStopDnodeInsertDatatb.py index ee48b973c90e859856fb2eeea54d1003386a1273..2ada32e075ea4388c0d0986b575fa93f6624f64f 100644 --- a/tests/system-test/6-cluster/manually-test/6dnode3mnodeStopDnodeInsertDatatb.py +++ b/tests/system-test/6-cluster/manually-test/6dnode3mnodeStopDnodeInsertDatatb.py @@ -40,7 +40,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/rollup.json b/tests/system-test/6-cluster/rollup.json new file mode 100644 index 0000000000000000000000000000000000000000..02669acb937ea9e9575adda203c2a744bea66548 --- /dev/null +++ b/tests/system-test/6-cluster/rollup.json @@ -0,0 +1,77 @@ +{ + "filetype": "insert", + "cfgdir": "/home/chr/TDengine/debug/../sim/dnode1/cfg/", + "host": "localhost", + "port": 6030, + "rest_port": 6041, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "create_table_thread_count": 4, + "result_file": "taosBenchmark_result.log", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [ + { + "dbinfo": { + "name": "dbtest", + "drop": "no", + "replica": 1, + "duration": 10, + "precision": "ms", + "keep": 3650, + "comp": 2, + "vgroups": 2, + "buffer": 1000 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "yes", + "childtable_count": 100, + "childtable_prefix": "ctb", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 500, + "data_source": "rand", + "insert_mode": "taosc", + "continue_if_fail": "yes", + "keep_trying": 500, + "trying_interval": 100, + "interlace_rows": 0, + "line_protocol": null, + "tcp_transfer": "no", + "insert_rows": 100000, + "childtable_limit": 0, + "childtable_offset": 0, + "rows_per_tbl": 0, + "max_sql_len": 1048576, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2022-10-22 17:20:36", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "partial_col_num": 999, + "columns": [{"type": "TIMESTAMP","max": 10, "min": 0},{"type": "INT","max": 10, "min": 0}, {"type": "BIGINT","max": 10, "min": 0}, {"type": "FLOAT","max": 10, "min": 0}, {"type": "DOUBLE","max": 10, "min": 0}, {"type": "SMALLINT","max": 10, "min": 0}, {"type": "TINYINT","max": 10, "min": 0}, {"type": "BOOL","max": 10, "min": 0}, {"type": "NCHAR","len": 29, "count":1, + "values": ["d1", "d2"] + }, {"type": "UINT","max": 10, "min": 0}, {"type": "UBIGINT","max": 10, "min": 0}, {"type": "UTINYINT","max": 10, "min": 0}, {"type": "USMALLINT","max": 10, "min": 0}, {"type": "BINARY", "len": 23, "count":1, + "values": ["b1","b2"] + }], + "tags": [{"type": "TIMESTAMP","max": 10, "min": 0},{"type": "INT","max": 10, "min": 0}, {"type": "BIGINT","max": 10, "min": 0}, {"type": "FLOAT","max": 10, "min": 0}, {"type": "DOUBLE","max": 10, "min": 0}, {"type": "SMALLINT","max": 10, "min": 0}, {"type": "TINYINT","max": 10, "min": 0}, {"type": "BOOL","max": 10, "min": 0}, {"type": "NCHAR","len": 17, "count":1, + "values": ["d1", "d2"] + }, {"type": "UINT","max": 10, "min": 0}, {"type": "UBIGINT","max": 10, "min": 0}, {"type": "UTINYINT","max": 10, "min": 0}, {"type": "USMALLINT","max": 10, "min": 0}, {"type": "BINARY", "len": 19, "count":1, + "values": ["b1","b2"] + }] + } + ] + } + ], + "prepare_rand": 10000, + "chinese": "no", + "streams": false, + "test_log": "/root/testlog/" +} diff --git a/tests/system-test/6-cluster/rollup_db.json b/tests/system-test/6-cluster/rollup_db.json new file mode 100644 index 0000000000000000000000000000000000000000..fedc47024cc3a2dbaf9c9146c1fcef0c7e454d9d --- /dev/null +++ b/tests/system-test/6-cluster/rollup_db.json @@ -0,0 +1,77 @@ +{ + "filetype": "insert", + "cfgdir": "/home/chr/TDengine/debug/../sim/dnode1/cfg/", + "host": "localhost", + "port": 6030, + "rest_port": 6041, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "create_table_thread_count": 4, + "result_file": "taosBenchmark_result.log", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [ + { + "dbinfo": { + "name": "dbtest", + "drop": "yes", + "replica": 1, + "duration": 10, + "precision": "ms", + "keep": 3650, + "comp": 2, + "vgroups": 2, + "buffer": 1000 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 100, + "childtable_prefix": "ctb", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 500, + "data_source": "rand", + "insert_mode": "taosc", + "continue_if_fail": "yes", + "keep_trying": 500, + "trying_interval": 100, + "interlace_rows": 0, + "line_protocol": null, + "tcp_transfer": "no", + "insert_rows": 0, + "childtable_limit": 0, + "childtable_offset": 0, + "rows_per_tbl": 0, + "max_sql_len": 1048576, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2022-10-22 17:20:36", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "partial_col_num": 999, + "columns": [{"type": "TIMESTAMP","max": 10, "min": 0},{"type": "INT","max": 10, "min": 0}, {"type": "BIGINT","max": 10, "min": 0}, {"type": "FLOAT","max": 10, "min": 0}, {"type": "DOUBLE","max": 10, "min": 0}, {"type": "SMALLINT","max": 10, "min": 0}, {"type": "TINYINT","max": 10, "min": 0}, {"type": "BOOL","max": 10, "min": 0}, {"type": "NCHAR","len": 29, "count":1, + "values": ["d1", "d2"] + }, {"type": "UINT","max": 10, "min": 0}, {"type": "UBIGINT","max": 10, "min": 0}, {"type": "UTINYINT","max": 10, "min": 0}, {"type": "USMALLINT","max": 10, "min": 0}, {"type": "BINARY", "len": 23, "count":1, + "values": ["b1","b2"] + }], + "tags": [{"type": "TIMESTAMP","max": 10, "min": 0},{"type": "INT","max": 10, "min": 0}, {"type": "BIGINT","max": 10, "min": 0}, {"type": "FLOAT","max": 10, "min": 0}, {"type": "DOUBLE","max": 10, "min": 0}, {"type": "SMALLINT","max": 10, "min": 0}, {"type": "TINYINT","max": 10, "min": 0}, {"type": "BOOL","max": 10, "min": 0}, {"type": "NCHAR","len": 17, "count":1, + "values": ["d1", "d2"] + }, {"type": "UINT","max": 10, "min": 0}, {"type": "UBIGINT","max": 10, "min": 0}, {"type": "UTINYINT","max": 10, "min": 0}, {"type": "USMALLINT","max": 10, "min": 0}, {"type": "BINARY", "len": 19, "count":1, + "values": ["b1","b2"] + }] + } + ] + } + ], + "prepare_rand": 10000, + "chinese": "no", + "streams": false, + "test_log": "/root/testlog/" +} diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py index 139be74a08a964edd7457bbcb32db68d2c20aa26..52d675208babbed401ff4d02575fd92458cbd4d2 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py @@ -34,7 +34,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py index 4a0522ad352b179f5886c4e6d5c8ba92ac02d7b2..9cc97543add8c2afc8e781efd45d9f37effb4551 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py @@ -40,7 +40,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py index 82ba256122f0cd04ff0fb008f140d6b5dbeb6fdf..4ea00ff2e282a117299867ff405ede9a4cbd6627 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py index 3751391d6596944ab3b09c833f1d5f6b38d21e22..51da6fc72396ff0f5b435261ccb3028b24bd56ee 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py @@ -40,7 +40,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py index 73153c5825684ba61e4e62e4288eb74ec02341e9..6e5043940d8daaf86beb10ee8acc226a7d5f739c 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py @@ -49,7 +49,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py index 24b4ff63dd6825fba7287cb5c613dd5f3a54d20b..a111e0bab5c6623d6667265fed855045f4a13a7e 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py index 6ef239382b49bd4080526f97170a4162417b2fe2..66eca7143da7477accffbb1d84c60c2674d37cd0 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py @@ -43,7 +43,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py index 35ea3f392c0ecc095d68f5df8c3ca4f00c3e7d25..db9139dca2d358c846573edf64ec36f80e06f862 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py @@ -43,7 +43,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py index ab5d05f3627166ee451176e2e783d210de5d9436..4fc4507c3fc651195c649f4eaf483e3d3406be21 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py @@ -43,7 +43,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py index fc6d3c06830d457b3f89b9f932c91ba22686cfde..f06b539ff26e23b3a511177f07c3636147eef104 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py @@ -49,7 +49,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py index 6e9aacebc2f2ea687ecc330d567c6ba49c3c9c9e..eb77c6d0032e384ad5b289dede2b347db3cd61a9 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py @@ -49,7 +49,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py index a55bc3c39f948ea71837825d540a5445f4c1060c..9079bedb7cb9e222b34f9232744ec6a6574ca3b8 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py @@ -49,7 +49,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py index dd8b6b374a258802fd37916c735fcca7d7f28c88..35cbceb2689cbec8a6e6ce5b67823a4941029c2d 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py @@ -49,7 +49,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py index 0af157ebffea6ce8510f73b963de675b28961b17..95b099b0a14049e130c444f8ece7813a68fed664 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py @@ -46,7 +46,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py index 124bf838bbd2da210196515cddb2171c5afe570d..bf2ebadd06b9f68a958571198d1a6799a0cebea5 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py @@ -46,7 +46,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py index 791b58d28d85bf162fdb2df616ded769c03c4f6e..25aba29235b2cc2b4c8ad70e09cba3bc39f03280 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py index 4fcfbfaf08f121a943fb5a58c349adfb7a23ac87..d29ab6b74ebc4b364d94f3d4127c6e978c7feb9c 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py @@ -51,7 +51,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py index 42d9e944f9430a16a5d356a8a7019b3c3b31090f..16ac90c31d210b695fa3c2130c2045e5858a1bf2 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py @@ -51,7 +51,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py index 6b87bee5a3c7a018336c46ca1f65eccddbe6bdce..a5f86a1e31a7a835ce145d8e88f360623af7a7d2 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py @@ -51,7 +51,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py index c53e909417451b90701d04c43b023aacd372ea1a..c272da0c2b040057b2a4ba376221fdfe66e705ca 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py @@ -51,7 +51,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py index 53a9463d64e0e045c1556b6550ded0e4dd4464ca..45ceb73059dd334ec79347ba51287af54d04b329 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py @@ -40,7 +40,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py index ddb765085a7f586dcae3f0c6452756af580e72b5..3f72f33951bd7a78121292edb936778bbbae0c5c 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py @@ -43,7 +43,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index 080b431ffe54f43e725e22d84a3f365f8bfc4cea..a16f2c348fdd925e8eadd1558ef12de926e64f92 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -149,7 +149,6 @@ class TDTestCase: topicFromStb = 'topic_stb_column' topicFromCtb = 'topic_ctb_column' - tdSql.execute("alter database %s wal_retention_period 3600" % (parameterDict['dbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName'])) diff --git a/tests/system-test/7-tmq/checkOffsetRowParams.py b/tests/system-test/7-tmq/checkOffsetRowParams.py new file mode 100644 index 0000000000000000000000000000000000000000..f7e4c61c9c115b2e3048748d0fe3965e5e6dc1d2 --- /dev/null +++ b/tests/system-test/7-tmq/checkOffsetRowParams.py @@ -0,0 +1,317 @@ + +import taos +import sys +import time +import socket +import os +import threading +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class actionType(Enum): + CREATE_DATABASE = 0 + CREATE_STABLE = 1 + CREATE_CTABLE = 2 + INSERT_DATA = 3 + +class TDTestCase: + hostname = socket.gethostname() + #rpcDebugFlagVal = '143' + #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #print ("===================: ", updatecfgDict) + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def newcur(self,cfg,host,port): + user = "root" + password = "taosdata" + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def initConsumerTable(self,cdbName='cdb'): + tdLog.info("create consume database, and consume info table, and consume result table") + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) + + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + def initConsumerInfoTable(self,cdbName='cdb'): + tdLog.info("drop consumeinfo table") + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + + def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + tdLog.info("consume info sql: %s"%sql) + tdSql.query(sql) + + def selectConsumeResult(self,expectRows,cdbName='cdb'): + resultList=[] + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == expectRows: + break + else: + time.sleep(5) + + for i in range(expectRows): + tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + resultList.append(tdSql.getData(i , 3)) + + return resultList + + def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): + if valgrind == 1: + logFile = cfgPath + '/../log/valgrind-tmq.log' + shellCmd = 'nohup valgrind --log-file=' + logFile + shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' + + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, dbName,stbName): + tsql.execute("create table if not exists %s.%s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%(dbName, stbName)) + tdLog.debug("complete to create %s.%s" %(dbName, stbName)) + return + + def create_ctables(self,tsql, dbName,stbName,ctbNum): + tsql.execute("use %s" %dbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=0): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs == 0: + t = time.time() + startTs = int(round(t * 1000)) + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + rowsOfSql = 0 + for i in range(ctbNum): + sql += " %s_%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + rowsOfSql += 1 + if (j > 0) and ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsOfSql = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s_%d values " %(stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareEnv(self, **parameterDict): + # create new connector for my thread + tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030) + + if parameterDict["actionType"] == actionType.CREATE_DATABASE: + self.create_database(tsql, parameterDict["dbName"]) + elif parameterDict["actionType"] == actionType.CREATE_STABLE: + self.create_stable(tsql, parameterDict["dbName"], parameterDict["stbName"]) + elif parameterDict["actionType"] == actionType.CREATE_CTABLE: + self.create_ctables(tsql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + elif parameterDict["actionType"] == actionType.INSERT_DATA: + self.insert_data(tsql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"], \ + parameterDict["rowsPerTbl"],parameterDict["batchNum"]) + else: + tdLog.exit("not support's action: ", parameterDict["actionType"]) + + return + + def tmqCase1(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 1: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db1', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:2000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 20 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start show subscriptions 1") + while(1): + tdSql.query("show subscriptions") + if (tdSql.getRows() == 0): + tdLog.info("sleep") + time.sleep(1) + elif (tdSql.queryResult[0][4] != None): + # tdSql.checkData(0, 4, "earliest") + tdSql.checkData(0, 5, 0) + break + + tdSql.query("show consumers") + tdSql.checkRows(1) + tdSql.checkData(0, 8, "tbname:1,commit:1,interval:2000ms,reset:earliest") + + time.sleep(2) + tdLog.info("start insert data") + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + time.sleep(2) + tdLog.info("start show subscriptions 2") + tdSql.query("show subscriptions") + tdSql.checkRows(4) + print(tdSql.queryResult) + # tdSql.checkData(0, 4, 'offset(log) ver:103') + tdSql.checkData(0, 5, 10000) + # tdSql.checkData(1, 4, 'offset(log) ver:103') + tdSql.checkData(1, 5, 10000) + # tdSql.checkData(2, 4, 'offset(log) ver:303') + tdSql.checkData(2, 5, 50000) + # tdSql.checkData(3, 4, 'offset(log) ver:239') + tdSql.checkData(3, 5, 30000) + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + + time.sleep(2) + tdLog.info("start show subscriptions 3") + tdSql.query("show subscriptions") + tdSql.checkRows(4) + print(tdSql.queryResult) + tdSql.checkData(0, 3, None) + # tdSql.checkData(0, 4, 'offset(log) ver:103') + tdSql.checkData(0, 5, 10000) + # tdSql.checkData(1, 4, 'offset(log) ver:103') + tdSql.checkData(1, 5, 10000) + # tdSql.checkData(2, 4, 'offset(log) ver:303') + tdSql.checkData(2, 5, 50000) + # tdSql.checkData(3, 4, 'offset(log) ver:239') + tdSql.checkData(3, 5, 30000) + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + self.tmqCase1(cfgPath, buildPath) + # self.tmqCase2(cfgPath, buildPath) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/create_wrong_topic.py b/tests/system-test/7-tmq/create_wrong_topic.py index 77d43149cd611225d2ee8f7b017013b0fc784bfb..66814d22ea05578408793132a9ba3cd3df16a4db 100644 --- a/tests/system-test/7-tmq/create_wrong_topic.py +++ b/tests/system-test/7-tmq/create_wrong_topic.py @@ -44,7 +44,6 @@ class TDTestCase: def wrong_topic(self): tdSql.prepare() tdSql.execute('use db') - tdSql.execute('alter database db wal_retention_period 3600') stbname = f'db.{tdCom.getLongName(5, "letters")}' tag_dict = { 't0':'int' diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py index 44f58279be2df9cc97ee504babebbbfda1793f44..808a4935e30ccec4d06863e54e5e2ff3e1ba635d 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py @@ -67,7 +67,6 @@ class TDTestCase: tdLog.info("flush db to let data falls into the disk") tdSql.query("flush database %s"%(paraDict['dbName'])) - tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal.py b/tests/system-test/7-tmq/dataFromTsdbNWal.py index 0f4f1694c17701d9d4361c788980215ae418db5e..8386c22cd0430128a1b873f3f5150ef51704b245 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal.py @@ -67,7 +67,6 @@ class TDTestCase: tdLog.info("flush db to let data falls into the disk") tdSql.query("flush database %s"%(paraDict['dbName'])) - tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py index 7d11684ed81fd79584253e032f67cae1bd833de8..e25fb412af44231e0a0a5deb672ed44a15b1e879 100644 --- a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py +++ b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py @@ -106,7 +106,6 @@ class TDTestCase: paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl - tdSql.execute("alter database dbt wal_retention_period 3600") tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha' "%(paraDict['dbName'], paraDict['stbName']) diff --git a/tests/system-test/7-tmq/ins_topics_test.py b/tests/system-test/7-tmq/ins_topics_test.py new file mode 100644 index 0000000000000000000000000000000000000000..8bf0a7e91af7de0a9cb59443aa2cd1c50b01410e --- /dev/null +++ b/tests/system-test/7-tmq/ins_topics_test.py @@ -0,0 +1,67 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + # rpcDebugFlagVal = '143' + #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + # updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #print ("===================: ", updatecfgDict) + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def test(self): + tdLog.info("create database, stb, ctb") + tdSql.execute("create database if not exists db1 vgroups 4 wal_retention_period 3600") + tdSql.execute("create table if not exists db1.st(ts timestamp, c1 int, c2 bool, c3 tinyint, c4 double, c5 nchar(8)) tags(t1 int, t2 float, t3 binary(4))") + tdSql.execute("create table if not exists db1.nt(ts timestamp, c1 smallint, c2 float, c3 binary(64), c4 bigint)") + tdSql.execute("create table if not exists db1.st1 using db1.st tags(1, 9.3, \"st1\")") + + tdLog.info("create topic") + tdSql.execute("create topic topic_1 as database db1") + tdSql.execute("create topic topic_2 with meta as stable db1.st") + tdSql.execute("create topic topic_3 as select * from db1.nt") + tdSql.execute("create topic topic_4 as select ts,c3,c5,t2 from db1.st") + + tdSql.query("select * from information_schema.ins_topics order by topic_name") + tdSql.checkRows(4) + tdSql.checkData(0, 4, "NULL") + tdSql.checkData(0, 5, "no") + tdSql.checkData(0, 6, "db") + tdSql.checkData(1, 4, "[{\"name\":\"ts\",\"type\":\"TIMESTAMP\",\"length\":8},{\"name\":\"c1\",\"type\":\"INT\",\"length\":4},{\"name\":\"c2\",\"type\":\"BOOL\",\"length\":1},{\"name\":\"c3\",\"type\":\"TINYINT\",\"length\":1},{\"name\":\"c4\",\"type\":\"DOUBLE\",\"length\":8},{\"name\":\"c5\",\"type\":\"NCHAR\",\"length\":8}]") + tdSql.checkData(1, 5, "yes") + tdSql.checkData(1, 6, "stable") + tdSql.checkData(2, 4, "[{\"name\":\"ts\",\"type\":\"TIMESTAMP\",\"length\":8},{\"name\":\"c1\",\"type\":\"SMALLINT\",\"length\":2},{\"name\":\"c2\",\"type\":\"FLOAT\",\"length\":4},{\"name\":\"c3\",\"type\":\"VARCHAR\",\"length\":64},{\"name\":\"c4\",\"type\":\"BIGINT\",\"length\":8}]") + tdSql.checkData(2, 5, "no") + tdSql.checkData(2, 6, "column") + tdSql.checkData(3, 4, "[{\"name\":\"ts\",\"type\":\"TIMESTAMP\",\"length\":8},{\"name\":\"c3\",\"type\":\"TINYINT\",\"length\":1},{\"name\":\"c5\",\"type\":\"NCHAR\",\"length\":8},{\"name\":\"t2\",\"type\":\"FLOAT\",\"length\":4}]") + tdSql.checkData(3, 5, "no") + tdSql.checkData(3, 6, "column") + + tdLog.printNoPrefix("======== test case end ...... ") + + def run(self): + self.test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/stbFilter.py b/tests/system-test/7-tmq/stbFilter.py index 3f862ae0472f6a1312ac5f2ea680adbd4b95f172..6b48a6d57076b4baf54d7c08db7cd8a625fab29c 100644 --- a/tests/system-test/7-tmq/stbFilter.py +++ b/tests/system-test/7-tmq/stbFilter.py @@ -45,7 +45,6 @@ class TDTestCase: tmqCom.initConsumerTable() tmqCom.create_database(tsql=tdSql, dbName=paraDict["dbName"],dropFlag=paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=paraDict['replica']) - tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict["dbName"])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/stbFilterWhere.py b/tests/system-test/7-tmq/stbFilterWhere.py new file mode 100644 index 0000000000000000000000000000000000000000..9b3e8b77109c7064f8920cbc2b964e17e6ed39dd --- /dev/null +++ b/tests/system-test/7-tmq/stbFilterWhere.py @@ -0,0 +1,226 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1} + + tmqCom.initConsumerTable() + tmqCom.create_database(tsql=tdSql, dbName=paraDict["dbName"],dropFlag=paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=paraDict['replica']) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + return + + def tmqCase_columnError(self, topicName, condition): + tdLog.printNoPrefix("======== test case error: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1} + + tdLog.info("create topics from stb with column filter") + topicString = "create topic %s as stable %s.%s where %s" %(topicName, paraDict['dbName'], paraDict['stbName'], condition) + tdLog.info("create topic sql: %s"%topicString) + tdSql.error(topicString) + + def tmqCase(self, topicName, condition): + tdLog.printNoPrefix("======== test case: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1} + + expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with tag filter") + topicString = "create topic %s as stable %s.%s where %s" %(topicName, paraDict['dbName'], paraDict['stbName'], condition) + tdLog.info("create topic sql: %s"%topicString) + tdSql.execute(topicString) + + queryString = "select * from %s.%s where %s" %(paraDict['dbName'], paraDict['stbName'], condition) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicName + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + tdLog.printNoPrefix("======== test case end ...... ") + + def tmqCase_addNewTable_dropTag(self, topicName, condition): + tdLog.printNoPrefix("======== test case1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1} + + expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with tag filter") + topicString = "create topic %s as stable %s.%s where %s" %(topicName, paraDict['dbName'], paraDict['stbName'], condition) + tdLog.info("create topic sql: %s"%topicString) + tdSql.execute(topicString) + + queryString = "select * from %s.%s where %s" %(paraDict['dbName'], paraDict['stbName'], condition) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows() + 1) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicName + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + #add new table with one data + tdLog.info("start insert data") + insertString = "insert into %s.tmp using %s.%s tags(1, 1, 1, 't4', 't5') values(now, 1, 1, 1, 'c4', 'c5', now)" %(paraDict['dbName'], paraDict['dbName'], paraDict['stbName']) + tdSql.execute(insertString) + + #test drop tag + tdSql.error("alter stable %s.%s drop tag t1" %(paraDict['dbName'], paraDict['stbName'])) + tdSql.execute("alter stable %s.%s drop tag t2" %(paraDict['dbName'], paraDict['stbName'])) + tdSql.execute("alter stable %s.%s drop column c2" %(paraDict['dbName'], paraDict['stbName'])) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + tdLog.printNoPrefix("======== test case1 end ...... ") + + def run(self): + tdSql.prepare() + self.prepareTestEnv() + self.tmqCase_columnError("t1", "c1 = 4 and t1 = 3") + self.tmqCase("t2", "2 > 1") + self.tmqCase("t3", "t4 = 'beijing'") + self.tmqCase("t4", "t4 > t3") + self.tmqCase("t5", "t3 = t4") + self.tmqCase_addNewTable_dropTag("t6", "t1 = 1") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/stbTagFilter-1ctb.py b/tests/system-test/7-tmq/stbTagFilter-1ctb.py index 1867dc54cb3744077f477f976f88edf59f744dc3..1e6011d5db14f98b8bf29b60812a0c3eb880cb62 100644 --- a/tests/system-test/7-tmq/stbTagFilter-1ctb.py +++ b/tests/system-test/7-tmq/stbTagFilter-1ctb.py @@ -106,7 +106,6 @@ class TDTestCase: # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - tdSql.execute("alter database dbt wal_retention_period 3600") tdLog.info("create topics from stb1") topicFromStb1 = 'topic_UpperCase_stb1' # queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) diff --git a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py index 67cc60d196bfd31f75e71978145f58ac40b698c7..5043c46f00dfda378d750357659504f2ecc542b6 100644 --- a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py +++ b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/subscribeDb4.py b/tests/system-test/7-tmq/subscribeDb4.py index 764362c70882e0851d3cd5eb80ac4f0096a71113..c14d3b27b11fa8e36bd216f23a2371018358aaf0 100644 --- a/tests/system-test/7-tmq/subscribeDb4.py +++ b/tests/system-test/7-tmq/subscribeDb4.py @@ -65,7 +65,6 @@ class TDTestCase: tmqCom.initConsumerTable(self.cdbName) tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"]) - tdSql.execute("alter database %s wal_retention_period 3600" % (self.paraDict['dbName'])) self.paraDict["stbName"] = 'stb1' tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) diff --git a/tests/system-test/7-tmq/subscribeStb3.py b/tests/system-test/7-tmq/subscribeStb3.py index 6f3230e68791c9c1806abf576936cd5878bd7b06..ed44ab1fb1940969e0011dd69211f76d58ece449 100644 --- a/tests/system-test/7-tmq/subscribeStb3.py +++ b/tests/system-test/7-tmq/subscribeStb3.py @@ -546,7 +546,7 @@ class TDTestCase: keyList = 'group.id:cgrp1,\ enable.auto.commit:false,\ auto.commit.interval.ms:6000,\ - auto.offset.reset:none' + auto.offset.reset:earliest' self.insertConsumerInfo(consumerId, expectrowcnt/2,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("again start consume processor") @@ -569,7 +569,7 @@ class TDTestCase: keyList = 'group.id:cgrp1,\ enable.auto.commit:false,\ auto.commit.interval.ms:6000,\ - auto.offset.reset:none' + auto.offset.reset:earliest' self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("again start consume processor") diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index 0740830696987a6009db020de14bdc768d56c69a..8c5dc5e693a43fd6d7a4ea724db5bdfa0fd7b8a4 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -172,7 +172,6 @@ class TDTestCase: tdLog.info("async insert data") pThread = tmqCom.asyncInsertData(paraDict) - tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create topics from stb with filter") # queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) diff --git a/tests/system-test/7-tmq/tmqAlterSchema.py b/tests/system-test/7-tmq/tmqAlterSchema.py index 1a8b0693b86cae8d12eaeb11f3af91f587a23a6f..a70678219f15ba94b7f9ceb1f81b9aa3881660d4 100644 --- a/tests/system-test/7-tmq/tmqAlterSchema.py +++ b/tests/system-test/7-tmq/tmqAlterSchema.py @@ -65,7 +65,6 @@ class TDTestCase: queryStringList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") @@ -176,7 +175,6 @@ class TDTestCase: queryStringList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ntb") diff --git a/tests/system-test/7-tmq/tmqCheckData.py b/tests/system-test/7-tmq/tmqCheckData.py index cb5a40642aab7ba2053780b898fa498e2c8b49a3..4d5edf87f1bb6b88ed9fe3dd8402f94e6c915744 100644 --- a/tests/system-test/7-tmq/tmqCheckData.py +++ b/tests/system-test/7-tmq/tmqCheckData.py @@ -53,7 +53,6 @@ class TDTestCase: tdLog.info("insert data") tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create topics from stb with filter") queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s" %(topicNameList[0], queryString) diff --git a/tests/system-test/7-tmq/tmqCheckData1.py b/tests/system-test/7-tmq/tmqCheckData1.py index b4fec94dcc3a2a266a35400dc789ee13aa463caf..1209c2812c536f9b3b6d23c8ad64ac1bbad033bc 100644 --- a/tests/system-test/7-tmq/tmqCheckData1.py +++ b/tests/system-test/7-tmq/tmqCheckData1.py @@ -45,7 +45,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqClientConsLog.py b/tests/system-test/7-tmq/tmqClientConsLog.py new file mode 100644 index 0000000000000000000000000000000000000000..7f755726cee55a18400e58f9c5b0b58837cdb3bb --- /dev/null +++ b/tests/system-test/7-tmq/tmqClientConsLog.py @@ -0,0 +1,227 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 3 + self.ctbNum = 10 + self.rowsPerTbl = 1000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 2, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def updateRowsOfConsumer(self, consumerDict, consumerId, totalRowsOfConsumer): + for key in consumerDict: + if key == consumerId: + consumerDict[key] = totalRowsOfConsumer + return + + consumerDict[consumerId] = totalRowsOfConsumer + return + + def checkClientLog(self, actConsumeTotalRows, numOfConsumer): + # 01931245 TSC consumer:0x5ee20f124420000c process poll rsp, vgId:5, offset:log:3399, blocks:2, rows:6000 vg total:330000 total:654000, reqId:0xa77d2245ae20112 + # 01931245 TSC consumer:0x5ee20f124420000c process poll rsp, vgId:7, offset:log:3384, blocks:1, rows:2000 vg total:326000 total:656000, reqId:0xa77d2245b050113 + # 01931246 TSC consumer:0x5ee20f124420000d process poll rsp, vgId:6, offset:log:3400, blocks:2, rows:6000 vg total:330000 total:330000, reqId:0xa77d2245b380116 + # 01931246 TSC consumer:0x5ee20f124420000d process poll rsp, vgId:6, offset:log:3460, blocks:2, rows:6000 vg total:336000 total:336000, reqId:0xa77d2245b8f011a + # 01931246 TSC consumer:0x5ee20f124420000d process poll rsp, vgId:6, offset:log:3520, blocks:2, rows:6000 vg total:342000 total:342000, reqId:0xa77d2245beb011f + # 01931246 TSC consumer:0x5ee20f124420000d process poll rsp, vgId:6, offset:log:3567, blocks:1, rows:2000 vg total:344000 total:344000, reqId:0xa77d2245c430121 + # filter key: process poll rsp, vgId + + tdLog.printNoPrefix("======== start filter key info from client log file") + + cfgPath = tdCom.getClientCfgPath() + taosLogFile = '%s/../log/taoslog*'%(cfgPath) + filterResultFile = '%s/../log/filter'%(cfgPath) + cmdStr = 'grep "process poll rsp, vgId:" %s >> %s'%(taosLogFile, filterResultFile) + tdLog.info(cmdStr) + os.system(cmdStr) + + consumerDict = {} + for index, line in enumerate(open(filterResultFile,'r')): + # tdLog.info("row[%d]: %s"%(index, line)) + valueList = line.split(',') + # for i in range(len(valueList)): + # tdLog.info("index[%d]: %s"%(i, valueList[i])) + # get consumer id + list2 = valueList[0].split(':') + list3 = list2[4].split() + consumerId = list3[0] + print("consumerId: %s"%(consumerId)) + + # # get vg id + # list2 = valueList[1].split(':') + # vgId = list2[1] + # print("vgId: %s"%(vgId)) + + # get total rows of a certain consuer + list2 = valueList[6].split(':') + totalRowsOfConsumer = list2[1] + print("totalRowsOfConsumer: %s"%(totalRowsOfConsumer)) + + # update a certain info + self.updateRowsOfConsumer(consumerDict, consumerId, totalRowsOfConsumer) + + # print(consumerDict) + if numOfConsumer != len(consumerDict): + tdLog.info("expect consumer num: %d, act consumer num: %d"%(numOfConsumer, len(consumerDict))) + tdLog.exit("act consumer error!") + + # total rows of all consumers + totalRows = 0 + for key in consumerDict: + totalRows += int(consumerDict[key]) + + if totalRows < actConsumeTotalRows: + tdLog.info("expect consume total rows: %d, act consume total rows: %d"%(actConsumeTotalRows, totalRows)) + tdLog.exit("act consume rows error!") + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 2, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + totalRowsInserted = expectRowsList[0] + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:500, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + consumerId = 1 + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor 0") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + actConsumeTotalRows = resultList[0] + resultList[1] + + tdLog.info("two consumers poll rows: %d， %d"%(resultList[0], resultList[1])) + + tdLog.info("the consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) + if not (totalRowsInserted <= actConsumeTotalRows): + tdLog.exit("%d tmq consume rows error!"%consumerId) + + self.checkClientLog(actConsumeTotalRows, 2) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.prepareTestEnv() + self.tmqCase1() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 6b633fa19384dbd1a8afcbb088e9660800969b65..3ea8273e7f79bc2ae05358825d36f2fd59037386 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -37,6 +37,9 @@ from util.common import * # INSERT_DATA = 3 class TMQCom: + def __init__(self): + self.g_end_insert_flag = 0 + def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdSql.init(conn.cursor()) @@ -330,8 +333,11 @@ class TMQCom: ctbDict[i] = 0 #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) - rowsOfCtb = 0 + rowsOfCtb = 0 while rowsOfCtb < rowsPerTbl: + if (0 != self.g_end_insert_flag): + tdLog.debug("get signal to stop insert data") + break for i in range(ctbNum): sql += " %s.%s%d values "%(dbName,ctbPrefix,i+ctbStartIdx) rowsBatched = 0 @@ -571,6 +577,20 @@ class TMQCom: tdLog.info(tsql.queryResult) tdLog.info("wait subscriptions exit for %d s"%wait_cnt) + def killProcesser(self, processerName): + killCmd = ( + "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" + % processerName + ) + + psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % processerName + processID = subprocess.check_output(psCmd, shell=True) + + while processID: + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output(psCmd, shell=True) + def close(self): self.cursor.close() diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py index 3ad1d097e11b85f04f1d0cc9b2cd75a9ea1bdb0a..f372a2b7428a77b02b8dcdb5c4c546756ef7e1f7 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py index fdd683d08daba5625017029a3363b379d2a206b1..c7f95f6f41cbf72e8d64c3a9bd8218dfd7a1c6d5 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py index f05f0abeff3f3f6dabde18ac3d3543d9df032f4e..26dacf514d4f66273d36f6ac3fe49ee17602747a 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py index 75b49a34fc583217e5ee0bb81324abc8421ed31a..d6f100041b15c205e8294cca964353d880b9ecc4 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index 26f7a9fb4d1300907807651a5e2fd9dd80579725..11fc7dbcc0587b20fd65bc71047ba04c1adb3f91 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -56,7 +56,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index cca29c178db281b86d145b8623f72a2dd8df1366..8ed4a6df973b57f7302d5a2c193debffbf7286a1 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py index 00d2491c97b0a178fbfd289cf44e2617107fdfe9..6a03f0f75192dda0b8b8394b1733bdec02222bda 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py index 3b1dbae443f0e8d6f71dc3c8127975790c23401a..c11159c6e551d60b90aab16bb7c90e4b26021ac2 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py index a799fa57195fe84065c7bb0b295dc7c0f14cdd09..a717c4966de9d5813545c68099a7a0cd58effa20 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -223,9 +222,9 @@ class TDTestCase: actConsumeTotalRows = resultList[0] - if not (actConsumeTotalRows > 0 and actConsumeTotalRows < totalRowsInserted): + if not (actConsumeTotalRows >= 0 and actConsumeTotalRows <= totalRowsInserted): tdLog.info("act consume rows: %d"%(actConsumeTotalRows)) - tdLog.info("and second consume rows should be between 0 and %d"%(totalRowsInserted)) + tdLog.info("and second consume rows should be between [0 and %d]"%(totalRowsInserted)) tdLog.exit("%d tmq consume rows error!"%consumerId) time.sleep(10) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py index f0bedbb187b9987e8f367f1afed7b69fb4c41001..439845aa54a6d29c1cd7633e9949da3eaa40cfee 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py index a63927dd8bdee0483cc943ececaac1855248ef57..53ff020b08abf2b758a6de00984b0cb799528d45 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1.py b/tests/system-test/7-tmq/tmqConsFromTsdb1.py index 8fcc991d4e3904da04480c7bbc28cac49cf5c068..4bb6cf463f59519565f43e04e18d7d7902b73108 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py b/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py new file mode 100644 index 0000000000000000000000000000000000000000..3dabca4cd11293b338f02279d759342c114cb10c --- /dev/null +++ b/tests/system-test/7-tmq/tmqConsumeDiscontinuousData.py @@ -0,0 +1,248 @@ + +import sys +import time +import datetime +import threading +from taos.tmq import Consumer +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135} + + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 100 + self.tmqMaxTopicNum = 1 + self.tmqMaxGroups = 1 + self.walRetentionPeriod = 3 + self.actConsumeTotalRows = 0 + self.retryPoll = 0 + self.lock = threading.Lock() + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10, + 'batchNum': 1, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period %d" % (paraDict['dbName'], self.walRetentionPeriod)) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + # tdLog.info("insert data") + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def tmqSubscribe(self, **inputDict): + consumer_dict = { + "group.id": inputDict['group_id'], + "client.id": "client", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.commit.interval.ms": "100", + "enable.auto.commit": "true", + "auto.offset.reset": "earliest", + "experimental.snapshot.enable": "false", + "msg.with.table.name": "false" + } + + consumer = Consumer(consumer_dict) + consumer.subscribe([inputDict['topic_name']]) + onceFlag = 0 + try: + while True: + if (1 == self.retryPoll): + time.sleep(2) + continue + res = consumer.poll(inputDict['pollDelay']) + if not res: + break + err = res.error() + if err is not None: + raise err + + val = res.value() + for block in val: + # print(block.fetchall()) + data = block.fetchall() + for row in data: + # print("===================================") + # print(row) + self.actConsumeTotalRows += 1 + if (0 == onceFlag): + onceFlag = 1 + with self.lock: + self.retryPoll = 1 + currentTime = datetime.now() + print("%s temp stop consume"%(str(currentTime))) + + currentTime = datetime.now() + print("%s already consume rows: %d, and sleep for a while"%(str(currentTime), self.actConsumeTotalRows)) + # time.sleep(self.walRetentionPeriod * 3) + finally: + consumer.unsubscribe() + consumer.close() + + return + + def asyncSubscribe(self, inputDict): + pThread = threading.Thread(target=self.tmqSubscribe, kwargs=inputDict) + pThread.start() + return pThread + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 100, + 'batchNum': 1, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + # create topic + topicNameList = ['dbtstb_0001'] + tdLog.info("create topics from stb") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + for i in range(len(topicNameList)): + sqlString = "create topic %s as %s" %(topicNameList[i], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + + + # start consumer + inputDict = {'group_id': "grpid_0001", + 'topic_name': topicNameList[0], + 'pollDelay': 10 + } + + pThread2 = self.asyncSubscribe(inputDict) + + pThread1 = tmqCom.asyncInsertDataByInterlace(paraDict) + pThread1.join() + tdLog.info("firstly call to flash database") + tdSql.query("flush database %s"%(paraDict['dbName'])) + time.sleep(self.walRetentionPeriod + 1) + tdLog.info("secondely call to flash database") + tdSql.query("flush database %s"%(paraDict['dbName'])) + + # wait the consumer to complete one poll + while (0 == self.retryPoll): + time.sleep(1) + continue + + with self.lock: + self.retryPoll = 0 + currentTime = datetime.now() + print("%s restart consume"%(str(currentTime))) + + paraDict["startTs"] = 1640966400000 + paraDict["ctbNum"] * paraDict["rowsPerTbl"] + pThread3 = tmqCom.asyncInsertDataByInterlace(paraDict) + + + tdLog.debug("wait sub-thread to end insert data") + pThread3.join() + + totalInsertRows = paraDict["ctbNum"] * paraDict["rowsPerTbl"] * 2 + tdLog.debug("wait sub-thread to end consume data") + pThread2.join() + + tdLog.info("act consume total rows: %d, act insert total rows: %d"%(self.actConsumeTotalRows, totalInsertRows)) + + if (self.actConsumeTotalRows >= totalInsertRows): + tdLog.exit("act consume rows: %d not equal expect: %d"%(self.actConsumeTotalRows, totalInsertRows)) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqConsumerGroup.py b/tests/system-test/7-tmq/tmqConsumerGroup.py index f05f600f27b1d1c3194c85a03edfbcac1aeee3b7..e64d8552340ba9c348ee637e1f676556fd6beabb 100644 --- a/tests/system-test/7-tmq/tmqConsumerGroup.py +++ b/tests/system-test/7-tmq/tmqConsumerGroup.py @@ -45,7 +45,6 @@ class TDTestCase: queryRowsList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index 648d629e5cc2e5cbcc8d60cf1599ba93c37d080a..74aba317264cea673cba4ed459165c20caec1cf3 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -187,7 +187,6 @@ class TDTestCase: tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) # tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqDropConsumer.json b/tests/system-test/7-tmq/tmqDropConsumer.json new file mode 100644 index 0000000000000000000000000000000000000000..538e93ea5ce13980c655959335bb91fc0d43ffe7 --- /dev/null +++ b/tests/system-test/7-tmq/tmqDropConsumer.json @@ -0,0 +1,28 @@ +{ + "filetype": "subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "result_file": "tmq_res.txt", + "tmq_info": { + "concurrent": 2, + "poll_delay": 100000, + "group.id": "", + "group_mode": "independent", + "create_mode": "parallel", + "client.id": "cliid_0001", + "auto.offset.reset": "earliest", + "enable.manual.commit": "false", + "enable.auto.commit": "false", + "auto.commit.interval.ms": 1000, + "experimental.snapshot.enable": "false", + "msg.with.table.name": "false", + "rows_file": "", + "topic_list": [ + {"name": "dbtstb_0001", "sql": "select * from dbt.stb;"}, + {"name": "dbtstb_0002", "sql": "select * from dbt.stb;"} + ] + } +} diff --git a/tests/system-test/7-tmq/tmqDropConsumer.py b/tests/system-test/7-tmq/tmqDropConsumer.py new file mode 100644 index 0000000000000000000000000000000000000000..06ce4c0fd7b411d5118797ece846fa90f5afab2e --- /dev/null +++ b/tests/system-test/7-tmq/tmqDropConsumer.py @@ -0,0 +1,293 @@ + +import sys +import time +import threading +from taos.tmq import Consumer +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135} + + def __init__(self): + self.vgroups = 2 + self.ctbNum = 10 + self.rowsPerTbl = 10 + self.tmqMaxTopicNum = 2 + self.tmqMaxGroups = 2 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 2, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 360000" % (paraDict['dbName'])) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def tmqSubscribe(self, topicName, newGroupId, expectResult): + # create new connector for new tdSql instance in my thread + # newTdSql = tdCom.newTdSql() + # topicName = inputDict['topic_name'] + # group_id = inputDict['group_id'] + + consumer_dict = { + "group.id": newGroupId, + "client.id": "client", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.commit.interval.ms": "1000", + "enable.auto.commit": "true", + "auto.offset.reset": "earliest", + "experimental.snapshot.enable": "false", + "msg.with.table.name": "false" + } + + ret = 'success' + consumer = Consumer(consumer_dict) + # print("======%s"%(inputDict['topic_name'])) + try: + consumer.subscribe([topicName]) + except Exception as e: + tdLog.info("consumer.subscribe() fail ") + tdLog.info("%s"%(e)) + if (expectResult == "fail"): + consumer.close() + return 'success' + else: + consumer.close() + return 'fail' + + tdLog.info("consumer.subscribe() success ") + if (expectResult == "success"): + consumer.close() + return 'success' + else: + consumer.close() + return 'fail' + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 100000000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + topicNameList = ['dbtstb_0001','dbtstb_0002'] + tdLog.info("create topics from stb") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + for i in range(len(topicNameList)): + sqlString = "create topic %s as %s" %(topicNameList[i], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # tdSql.query('show topics;') + # topicNum = tdSql.queryRows + # tdLog.info(" topic count: %d"%(topicNum)) + # if topicNum != len(topicNameList): + # tdLog.exit("show topics %d not equal expect num: %d"%(topicNum, len(topicNameList))) + + pThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + # use taosBenchmark to subscribe + binPath = self.getPath() + cmd = "nohup %s -f ./7-tmq/tmqDropConsumer.json > /dev/null 2>&1 & " % binPath + tdLog.info("%s"%(cmd)) + os.system(cmd) + + expectTopicNum = len(topicNameList) + consumerThreadNum = 2 + expectConsumerNUm = expectTopicNum * consumerThreadNum + expectSubscribeNum = self.vgroups * expectTopicNum * consumerThreadNum + + tdSql.query('show topics;') + topicNum = tdSql.queryRows + tdLog.info(" get topic count: %d"%(topicNum)) + if topicNum != expectTopicNum: + tdLog.exit("show topics %d not equal expect num: %d"%(topicNum, expectTopicNum)) + + flag = 0 + while (1): + tdSql.query('show consumers;') + consumerNUm = tdSql.queryRows + tdLog.info(" get consumers count: %d"%(consumerNUm)) + if consumerNUm == expectConsumerNUm: + flag = 1 + break + else: + time.sleep(1) + + if (0 == flag): + tmqCom.g_end_insert_flag = 1 + tdLog.exit("show consumers %d not equal expect num: %d"%(topicNum, expectConsumerNUm)) + + flag = 0 + for i in range(10): + tdSql.query('show subscriptions;') + subscribeNum = tdSql.queryRows + tdLog.info(" get subscriptions count: %d"%(subscribeNum)) + if subscribeNum == expectSubscribeNum: + flag = 1 + break + else: + time.sleep(1) + + if (0 == flag): + tmqCom.g_end_insert_flag = 1 + tdLog.exit("show subscriptions %d not equal expect num: %d"%(subscribeNum, expectSubscribeNum)) + + # get all consumer group id + tdSql.query('show consumers;') + consumerNUm = tdSql.queryRows + groupIdList = [] + for i in range(consumerNUm): + groupId = tdSql.getData(i,1) + existFlag = 0 + for j in range(len(groupIdList)): + if (groupId == groupIdList[j]): + existFlag = 1 + break + if (0 == existFlag): + groupIdList.append(groupId) + + # kill taosBenchmark + tmqCom.killProcesser("taosBenchmark") + tdLog.info("kill taosBenchmak end") + + # wait the status to "lost" + while (1): + exitFlag = 1 + tdSql.query('show consumers;') + consumerNUm = tdSql.queryRows + for i in range(consumerNUm): + status = tdSql.getData(i,3) + if (status != "lost"): + exitFlag = 0 + time.sleep(2) + break + if (1 == exitFlag): + break + + tdLog.info("all consumers status into 'lost'") + + # drop consumer groups + tdLog.info("drop all consumers") + for i in range(len(groupIdList)): + for j in range(len(topicNameList)): + sqlCmd = f"drop consumer group `%s` on %s"%(groupIdList[i], topicNameList[j]) + tdLog.info("drop consumer cmd: %s"%(sqlCmd)) + tdSql.execute(sqlCmd) + + tmqCom.g_end_insert_flag = 1 + tdLog.debug("notify sub-thread to stop insert data") + pThread.join() + + tdSql.query('show consumers;') + consumerNUm = tdSql.queryRows + + tdSql.query('show subscriptions;') + subscribeNum = tdSql.queryRows + + if (0 != consumerNUm or 0 != subscribeNum): + tdLog.exit("drop consumer fail! consumerNUm %d, subscribeNum: %d"%(consumerNUm, subscribeNum)) + + tdLog.info("drop consuer success, there is no consumers and subscribes") + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py index 7a9c1bbb8c808fe62e8fede76640ac78e19f2f30..c8bcdd6235782f4a29cc18a4f243e8e53ed5ffe5 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py @@ -57,7 +57,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") @@ -144,7 +143,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py index 0b9cb7e66a2d481f6162f1e5e92d8c1f68dcb300..3fc5a2fdc70b64ec95324c370625f5397287ae7d 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py @@ -57,7 +57,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") @@ -144,7 +143,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") diff --git a/tests/system-test/7-tmq/tmqDropStb.py b/tests/system-test/7-tmq/tmqDropStb.py index 0b252a733495041464a3504a3bc8d94e54fcf48c..00affabafcb8c729e2f3ca9f988a0f7930eb030b 100644 --- a/tests/system-test/7-tmq/tmqDropStb.py +++ b/tests/system-test/7-tmq/tmqDropStb.py @@ -64,7 +64,6 @@ class TDTestCase: tmqCom.initConsumerTable(self.cdbName) tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"]) - tdSql.execute("alter database %s wal_retention_period 3600" % (self.paraDict['dbName'])) self.paraDict["stbName"] = 'stb1' tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) diff --git a/tests/system-test/7-tmq/tmqDropStbCtb.py b/tests/system-test/7-tmq/tmqDropStbCtb.py index 587baf12aab02a3ae3e7d4d9783cafab6ac0ceca..c9e34136cc1e59af173205a8e0cb2796acb1a14a 100644 --- a/tests/system-test/7-tmq/tmqDropStbCtb.py +++ b/tests/system-test/7-tmq/tmqDropStbCtb.py @@ -54,7 +54,6 @@ class TDTestCase: # tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqMaxGroupIds.json b/tests/system-test/7-tmq/tmqMaxGroupIds.json new file mode 100644 index 0000000000000000000000000000000000000000..beb16576b0a821d5c03efe739b98d688006d46bc --- /dev/null +++ b/tests/system-test/7-tmq/tmqMaxGroupIds.json @@ -0,0 +1,27 @@ +{ + "filetype": "subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "result_file": "tmq_res.txt", + "tmq_info": { + "concurrent": 99, + "poll_delay": 100000, + "group.id": "", + "group_mode": "independent", + "create_mode": "parallel", + "client.id": "cliid_0001", + "auto.offset.reset": "earliest", + "enable.manual.commit": "false", + "enable.auto.commit": "false", + "auto.commit.interval.ms": 1000, + "experimental.snapshot.enable": "false", + "msg.with.table.name": "false", + "rows_file": "", + "topic_list": [ + {"name": "dbtstb_0001", "sql": "select * from dbt.stb;"} + ] + } +} diff --git a/tests/system-test/7-tmq/tmqMaxGroupIds.py b/tests/system-test/7-tmq/tmqMaxGroupIds.py new file mode 100644 index 0000000000000000000000000000000000000000..d22b79a44cfbae0cf35584503ce3976f94b2b787 --- /dev/null +++ b/tests/system-test/7-tmq/tmqMaxGroupIds.py @@ -0,0 +1,246 @@ + +import sys +import time +import threading +from taos.tmq import Consumer +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135} + + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10 + self.tmqMaxTopicNum = 20 + self.tmqMaxGroups = 100 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 360000" % (paraDict['dbName'])) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def tmqSubscribe(self, topicName, newGroupId, expectResult): + # create new connector for new tdSql instance in my thread + # newTdSql = tdCom.newTdSql() + # topicName = inputDict['topic_name'] + # group_id = inputDict['group_id'] + + consumer_dict = { + "group.id": newGroupId, + "client.id": "client", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.commit.interval.ms": "1000", + "enable.auto.commit": "true", + "auto.offset.reset": "earliest", + "experimental.snapshot.enable": "false", + "msg.with.table.name": "false" + } + + ret = 'success' + consumer = Consumer(consumer_dict) + # print("======%s"%(inputDict['topic_name'])) + try: + consumer.subscribe([topicName]) + except Exception as e: + tdLog.info("consumer.subscribe() fail ") + tdLog.info("%s"%(e)) + if (expectResult == "fail"): + consumer.close() + return 'success' + else: + consumer.close() + return 'fail' + + tdLog.info("consumer.subscribe() success ") + if (expectResult == "success"): + consumer.close() + return 'success' + else: + consumer.close() + return 'fail' + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 100000000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + topicNameList = ['dbtstb_0001'] + tdLog.info("create topics from stb") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + for i in range(len(topicNameList)): + sqlString = "create topic %s as %s" %(topicNameList[i], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # tdSql.query('show topics;') + # topicNum = tdSql.queryRows + # tdLog.info(" topic count: %d"%(topicNum)) + # if topicNum != len(topicNameList): + # tdLog.exit("show topics %d not equal expect num: %d"%(topicNum, len(topicNameList))) + + pThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + # use taosBenchmark to subscribe + binPath = self.getPath() + cmd = "nohup %s -f ./7-tmq/tmqMaxGroupIds.json > /dev/null 2>&1 & " % binPath + tdLog.info("%s"%(cmd)) + os.system(cmd) + + expectTopicNum = 1 + expectConsumerNUm = 99 + expectSubscribeNum = 99 + + tdSql.query('show topics;') + topicNum = tdSql.queryRows + tdLog.info(" get topic count: %d"%(topicNum)) + if topicNum != expectTopicNum: + tdLog.exit("show topics %d not equal expect num: %d"%(topicNum, expectTopicNum)) + + flag = 0 + while (1): + tdSql.query('show consumers;') + consumerNUm = tdSql.queryRows + tdLog.info(" get consumers count: %d"%(consumerNUm)) + if consumerNUm == expectConsumerNUm: + flag = 1 + break + else: + time.sleep(1) + + if (0 == flag): + tdLog.exit("show consumers %d not equal expect num: %d"%(topicNum, expectConsumerNUm)) + + flag = 0 + for i in range(10): + tdSql.query('show subscriptions;') + subscribeNum = tdSql.queryRows + tdLog.info(" get subscriptions count: %d"%(subscribeNum)) + if subscribeNum == expectSubscribeNum: + flag = 1 + break + else: + time.sleep(1) + + if (0 == flag): + tdLog.exit("show subscriptions %d not equal expect num: %d"%(subscribeNum, expectSubscribeNum)) + + res = self.tmqSubscribe(topicNameList[0], "newGroupId_001", "success") + if res != 'success': + tdLog.exit("limit max groupid fail") + + res = self.tmqSubscribe(topicNameList[0], "newGroupId_002", "fail") + if res != 'success': + tdLog.exit("limit max groupid fail") + + tmqCom.g_end_insert_flag = 1 + tdLog.debug("notify sub-thread to stop insert data") + pThread.join() + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqMaxTopic.py b/tests/system-test/7-tmq/tmqMaxTopic.py new file mode 100644 index 0000000000000000000000000000000000000000..05b699ca00de81b0f41b0fe33b7fa7ea4c7fb5b6 --- /dev/null +++ b/tests/system-test/7-tmq/tmqMaxTopic.py @@ -0,0 +1,362 @@ + +import sys +import time +import threading +from taos.tmq import Consumer +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135} + + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10 + self.tmqMaxTopicNum = 20 + self.tmqMaxGroups = 100 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def modifyMaxTopics(self, tmqMaxTopicNum): + # single dnode + cfgDir = tdDnodes.dnodes[0].cfgDir + + # cluster dnodes + # tdDnodes[1].dataDir + # tdDnodes[1].logDir + # tdDnodes[1].cfgDir + + cfgFile = f"%s/taos.cfg"%(cfgDir) + shellCmd = 'echo tmqMaxTopicNum %d >> %s'%(tmqMaxTopicNum, cfgFile) + tdLog.info(" shell cmd: %s"%(shellCmd)) + os.system(shellCmd) + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) + time.sleep(5) + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def tmqSubscribe(self, **inputDict): + # create new connector for new tdSql instance in my thread + # newTdSql = tdCom.newTdSql() + # topicName = inputDict['topic_name'] + # group_id = inputDict['group_id'] + + consumer_dict = { + "group.id": inputDict['group_id_prefix'], + "client.id": "client", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.commit.interval.ms": "1000", + "enable.auto.commit": "true", + "auto.offset.reset": "earliest", + "experimental.snapshot.enable": "false", + "msg.with.table.name": "false" + } + + for j in range(self.tmqMaxGroups): + consumer_dict["group.id"] = f"%s_%d"%(inputDict['group_id_prefix'], j) + consumer_dict["client.id"] = f"%s_%d"%(inputDict['group_id_prefix'], j) + print("======grpid: %s"%(consumer_dict["group.id"])) + consumer = Consumer(consumer_dict) + # print("======%s"%(inputDict['topic_name'])) + consumer.subscribe([inputDict['topic_name']]) + # res = consumer.poll(inputDict['pollDelay']) + return + + def asyncSubscribe(self, inputDict): + pThread = threading.Thread(target=self.tmqSubscribe, kwargs=inputDict) + pThread.start() + return pThread + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNamePrefix = 'topicname_' + tdLog.info("create topics from stb") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + for i in range(self.tmqMaxTopicNum): + sqlString = "create topic %s%d as %s" %(topicNamePrefix, i, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + sqlString = "create topic %s%s as %s" %(topicNamePrefix, 'xyz', queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.error(sqlString) + + tdSql.query('show topics;') + topicNum = tdSql.queryRows + tdLog.info(" topic count: %d"%(topicNum)) + if topicNum != self.tmqMaxTopicNum: + tdLog.exit("show topics %d not equal expect num: %d"%(topicNum, self.tmqMaxTopicNum)) + + # self.updatecfgDict = {'tmqMaxTopicNum': 22} + # tdDnodes.stoptaosd(1) + # tdDnodes.deploy(1, self.updatecfgDict) + # tdDnodes.starttaosd(1) + # time.sleep(5) + + newTmqMaxTopicNum = 22 + self.modifyMaxTopics(newTmqMaxTopicNum) + + sqlString = "create topic %s%s as %s" %(topicNamePrefix, 'x', queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + sqlString = "create topic %s%s as %s" %(topicNamePrefix, 'y', queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + sqlString = "create topic %s%s as %s" %(topicNamePrefix, 'xyz', queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.error(sqlString) + + tdSql.query('show topics;') + topicNum = tdSql.queryRows + tdLog.info(" topic count: %d"%(topicNum)) + if topicNum != newTmqMaxTopicNum: + tdLog.exit("show topics %d not equal expect num: %d"%(topicNum, newTmqMaxTopicNum)) + + newTmqMaxTopicNum = 18 + self.modifyMaxTopics(newTmqMaxTopicNum) + + i = 0 + sqlString = "drop topic %s%d" %(topicNamePrefix, i) + tdLog.info("drop topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + i = 1 + sqlString = "drop topic %s%d" %(topicNamePrefix, i) + tdLog.info("drop topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + sqlString = "drop topic %s%s" %(topicNamePrefix, "x") + tdLog.info("drop topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + sqlString = "drop topic %s%s" %(topicNamePrefix, "y") + tdLog.info("drop topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + sqlString = "create topic %s%s as %s" %(topicNamePrefix, 'xyz', queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.error(sqlString) + + tdLog.info("drop database when there are topic") + sqlString = "drop database %s" %(paraDict['dbName']) + tdLog.info("drop database sql: %s"%sqlString) + tdSql.error(sqlString) + + tdLog.info("drop all topic for re-create") + tdSql.query('show topics;') + topicNum = tdSql.queryRows + tdLog.info(" topic count: %d"%(topicNum)) + for i in range(topicNum): + sqlString = "drop topic %s" %(tdSql.getData(i, 0)) + tdLog.info("drop topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + time.sleep(1) + + tdLog.info("re-create topics") + topicNamePrefix = 'newTopic_' + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + for i in range(topicNum): + sqlString = "create topic %s%d as %s" %(topicNamePrefix, i, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + #=================================================# + tdLog.info("drop all topic for testcase2") + tdSql.query('show topics;') + topicNum = tdSql.queryRows + tdLog.info(" topic count: %d"%(topicNum)) + for i in range(topicNum): + sqlString = "drop topic %s" %(tdSql.getData(i, 0)) + tdLog.info("drop topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: test topic name len") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + totalTopicNum = 0 + + topicName = 'a' + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.query(sqlString) + totalTopicNum += 1 + + topicName = '3' + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.error(sqlString) + totalTopicNum += 0 + + topicName = '_1' + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.query(sqlString) + totalTopicNum += 1 + + topicName = 'a\\' + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.error(sqlString) + totalTopicNum += 0 + + topicName = 'a\*\&\^' + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.error(sqlString) + totalTopicNum += 0 + + + str191char = 'a' + for i in range(190): + str191char = ('%s%d'%(str191char, 1)) + + topicName = str191char + 'a' + + if (192 != len(topicName)): + tdLog.exit("topicName len error") + + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.query(sqlString) + totalTopicNum += 1 + + topicName = str191char + '12' + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.error(sqlString) + totalTopicNum += 0 + + # topicName = str192char + '12' + # sqlString = "create topic %s as %s" %(topicName, queryString) + # tdLog.info("create topic sql: %s"%sqlString) + # tdSql.error(sqlString) + # totalTopicNum += 0 + + # check topic count + tdSql.query('show topics;') + topicNum = tdSql.queryRows + tdLog.info(" topic count: %d"%(topicNum)) + if topicNum != totalTopicNum: + tdLog.exit("show topics %d not equal expect num: %d"%(topicNum, totalTopicNum)) + + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1() + self.tmqCase2() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqOffset.py b/tests/system-test/7-tmq/tmqOffset.py new file mode 100644 index 0000000000000000000000000000000000000000..500c6f53e4efc591c9592db37a990509c23dd7e3 --- /dev/null +++ b/tests/system-test/7-tmq/tmqOffset.py @@ -0,0 +1,399 @@ + +import sys +import re +import time +import threading +from taos.tmq import Consumer +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135} + + def __init__(self): + self.vgroups = 2 + self.ctbNum = 1 + self.rowsPerTbl = 10000 + self.tmqMaxTopicNum = 10 + self.tmqMaxGroups = 10 + + self.TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE = '0x4007' + self.TSDB_CODE_TMQ_INVALID_VGID = '0x4008' + self.TSDB_CODE_TMQ_INVALID_TOPIC = '0x4009' + + + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + # 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + # 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + + 'colSchema': [{'type': 'INT', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}], + + + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1, + 'rowsPerTbl': 10, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + # tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1,wal_retention_period=36000) + # tdSql.execute("alter database %s wal_retention_period 360000" % (paraDict['dbName'])) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def tmqPollAllRows(self, consumer): + totalRows = 0 + + res = consumer.poll(10) + while (res): + if not res: + break + err = res.error() + if err is not None: + raise err + + val = res.value() + # print(len(val)) + for block in val: + # print(block.fetchall()) + # print(len(block.fetchall())) + totalRows += len(block.fetchall()) + + res = consumer.poll(10) + + tdLog.info("poll total rows: %d"%(totalRows)) + return totalRows + + def tmqPollRowsByOne(self, consumer): + rows = 0 + res = consumer.poll(3) + if not res: + return rows + err = res.error() + if err is not None: + raise err + val = res.value() + + # print(len(val)) + + for block in val: + # print(block.fetchall()) + # print(len(block.fetchall())) + rows += len(block.fetchall()) + + return rows + + def tmqOffsetTest(self, consumer): + # get topic assignment + tdLog.info("before poll get offset status:") + assignments = consumer.assignment() + for assignment in assignments: + print(assignment) + + # poll + # consumer.poll(5) + rows = self.tmqPollRowsByOne(consumer) + tdLog.info("poll rows: %d"%(rows)) + + # get topic assignment + tdLog.info("after first poll get offset status:") + assignments = consumer.assignment() + for assignment in assignments: + print(assignment) + + + rows = self.tmqPollRowsByOne(consumer) + tdLog.info("poll rows: %d"%(rows)) + + # get topic assignment + tdLog.info("after second poll get offset status:") + assignments = consumer.assignment() + for assignment in assignments: + print(assignment) + + + return + + def tmqSubscribe(self, inputDict): + consumer_dict = { + "group.id": inputDict['group_id'], + "client.id": "client", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.commit.interval.ms": "1000", + "enable.auto.commit": inputDict['auto_commit'], + "auto.offset.reset": inputDict['offset_reset'], + "experimental.snapshot.enable": "false", + "msg.with.table.name": "false" + } + + consumer = Consumer(consumer_dict) + try: + consumer.subscribe([inputDict['topic_name']]) + except Exception as e: + tdLog.info("consumer.subscribe() fail ") + tdLog.info("%s"%(e)) + + # rows = self.tmqPollAllRows(consumer) + tdLog.info("create consumer success!") + return consumer + + def tmqConsumer(self, **inputDict): + consumer = self.tmqSubscribe(inputDict) + self.tmqPollAllRows(consumer) + # consumer.unsubscribe() + # consumer.close() + return + + def asyncSubscribe(self, inputDict): + pThread = threading.Thread(target=self.tmqConsumer, kwargs=inputDict) + pThread.start() + return pThread + + def seekErrorVgid(self, consumer, assignment): + ####################### test1: error vgid + assignmentNew = assignment + # assignment.topic + assignmentNew.partition = assignment.partition + self.vgroups + self.vgroups + # assignment.offset + # consumer.seek(assignment) + + errCodeStr = '' + try: + print("seek parameters:", assignmentNew) + consumer.seek(assignmentNew) + except Exception as e: + tdLog.info("error: %s"%(e)) + + rspString = str(e) + start = "[" + end = "]" + + start_index = rspString.index(start) + len(start) + end_index = rspString.index(end) + + errCodeStr = rspString[start_index:end_index] + # print(errCodeStr) + tdLog.info("error code: %s"%(errCodeStr)) + + if (self.TSDB_CODE_TMQ_INVALID_VGID != errCodeStr): + tdLog.exit("tmq seek should return error code: %s"%(self.TSDB_CODE_TMQ_INVALID_VGID)) + + def seekErrorTopic(self, consumer, assignment): + assignmentNew = assignment + assignmentNew.topic = 'errorToipcName' + # assignment.partition + # assignment.offset + # consumer.seek(assignment) + + errCodeStr = '' + try: + print("seek parameters:", assignmentNew) + consumer.seek(assignmentNew) + except Exception as e: + tdLog.info("error: %s"%(e)) + + rspString = str(e) + start = "[" + end = "]" + + start_index = rspString.index(start) + len(start) + end_index = rspString.index(end) + + errCodeStr = rspString[start_index:end_index] + # print(errCodeStr) + tdLog.info("error code: %s"%(errCodeStr)) + + if (self.TSDB_CODE_TMQ_INVALID_TOPIC != errCodeStr): + tdLog.exit("tmq seek should return error code: %s"%(self.TSDB_CODE_TMQ_INVALID_TOPIC)) + + def seekErrorVersion(self, consumer, assignment): + assignmentNew = assignment + # print(assignment.topic, assignment.partition, assignment.offset) + # assignment.topic + # assignment.partition + assignmentNew.offset = assignment.offset + self.rowsPerTbl * 100000 + # consumer.seek(assignment) + + errCodeStr = '' + try: + # print(assignmentNew.topic, assignmentNew.partition, assignmentNew.offset) + print("seek parameters:", assignmentNew) + consumer.seek(assignmentNew) + except Exception as e: + tdLog.info("error: %s"%(e)) + + rspString = str(e) + start = "[" + end = "]" + + start_index = rspString.index(start) + len(start) + end_index = rspString.index(end) + + errCodeStr = rspString[start_index:end_index] + # print(errCodeStr) + tdLog.info("error code: %s"%(errCodeStr)) + + if (self.TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE != errCodeStr): + tdLog.exit("tmq seek should return error code: %s"%(self.TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE)) + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1, + 'rowsPerTbl': 100000000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + # ntbName = 'ntb' + # sqlString = "create table %s.%s (ts timestamp, c int)"%(paraDict['dbName'], ntbName) + # tdLog.info("create ntb sql: %s"%sqlString) + # tdSql.execute(sqlString) + + topicName = 'offset_tp' + # queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName) + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + inputDict = { + "topic_name": topicName, + "group_id": "offsetGrp", + "auto_commit": "true", + "offset_reset": "earliest" + } + + pThread = self.asyncSubscribe(inputDict) + # pThread.join() + + consumer = self.tmqSubscribe(inputDict) + # get topic assignment + assignments = consumer.assignment() + # print(type(assignments)) + for assignment in assignments: + print(assignment) + + assignment = assignments[0] + topic = assignment.topic + partition = assignment.partition + offset = assignment.offset + + tdLog.info("======== test error vgid =======") + print("current assignment: ", assignment) + self.seekErrorVgid(consumer, assignment) + + tdLog.info("======== test error topic =======") + assignment.topic = topic + assignment.partition = partition + assignment.offset = offset + print("current assignment: ", assignment) + self.seekErrorTopic(consumer, assignment) + + tdLog.info("======== test error version =======") + assignment.topic = topic + assignment.partition = partition + assignment.offset = offset + print("current assignment: ", assignment) + self.seekErrorVersion(consumer, assignment) + + pThread.join() + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqParamsTest.py b/tests/system-test/7-tmq/tmqParamsTest.py new file mode 100644 index 0000000000000000000000000000000000000000..ff7c70bcd285c7ce9340576b584756d8d2950d3e --- /dev/null +++ b/tests/system-test/7-tmq/tmqParamsTest.py @@ -0,0 +1,179 @@ +import sys +import time +import threading +from taos.tmq import Consumer +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135} + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.wal_retention_period1 = 3600 + self.wal_retention_period2 = 1 + self.commit_value_list = ["true", "false"] + self.offset_value_list = ["", "earliest", "latest", "none"] + self.tbname_value_list = ["true", "false"] + self.snapshot_value_list = ["false"] + + # self.commit_value_list = ["true"] + # self.offset_value_list = [""] + # self.tbname_value_list = ["true"] + # self.snapshot_value_list = ["false"] + + def tmqParamsTest(self): + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'vgroups': 4, + 'stbName': 'stb', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'auto_commit_interval': "100"} + + + start_group_id = 1 + for snapshot_value in self.snapshot_value_list: + for commit_value in self.commit_value_list: + for offset_value in self.offset_value_list: + for tbname_value in self.tbname_value_list: + topic_name = 'topic1' + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + tdLog.info("insert data") + tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + + + tdLog.info("create topics from stb with filter") + queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topic_name, queryString) + tdSql.query(f'select * from information_schema.ins_databases') + db_wal_retention_period_list = list(map(lambda x:x[-6] if x[0] == paraDict['dbName'] else None, tdSql.queryResult)) + for i in range(len(db_wal_retention_period_list)): + if db_wal_retention_period_list[0] is None or db_wal_retention_period_list[-1] is None: + db_wal_retention_period_list.remove(None) + if snapshot_value =="true": + if db_wal_retention_period_list[0] != self.wal_retention_period2: + tdSql.execute(f"alter database {paraDict['dbName']} wal_retention_period {self.wal_retention_period2}") + time.sleep(self.wal_retention_period2+1) + tdSql.execute(f'flush database {paraDict["dbName"]}') + else: + if db_wal_retention_period_list[0] != self.wal_retention_period1: + tdSql.execute(f"alter database {paraDict['dbName']} wal_retention_period {self.wal_retention_period1}") + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expected_res = tdSql.queryRows + group_id = "csm_" + str(start_group_id) + consumer_dict = { + "group.id": group_id, + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.commit.interval.ms": paraDict["auto_commit_interval"], + "enable.auto.commit": commit_value, + "auto.offset.reset": offset_value, + "experimental.snapshot.enable": snapshot_value, + "msg.with.table.name": tbname_value + } + consumer_commit = 1 if consumer_dict["enable.auto.commit"] == "true" else 0 + consumer_tbname = 1 if consumer_dict["msg.with.table.name"] == "true" else 0 + consumer_ret = "earliest" if offset_value == "" else offset_value + expected_parameters=f'tbname:{consumer_tbname},commit:{consumer_commit},interval:{paraDict["auto_commit_interval"]}ms,reset:{consumer_ret}' + if len(offset_value) == 0: + del consumer_dict["auto.offset.reset"] + consumer = Consumer(consumer_dict) + consumer.subscribe([topic_name]) + tdLog.info(f"enable.auto.commit: {commit_value}, auto.offset.reset: {offset_value}, experimental.snapshot.enable: {snapshot_value}, msg.with.table.name: {tbname_value}") + stop_flag = 0 + try: + while True: + res = consumer.poll(1) + tdSql.query('show consumers;') + consumer_info = tdSql.queryResult[0][-1] + if offset_value == "latest": + if not res and stop_flag == 1: + break + else: + if not res: + break + # err = res.error() + # if err is not None: + # raise err + # val = res.value() + # for block in val: + # print(block.fetchall()) + if offset_value == "latest" and stop_flag == 0: + tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],int(round(time.time()*1000))) + stop_flag = 1 + finally: + consumer.unsubscribe() + consumer.close() + tdSql.checkEqual(consumer_info, expected_parameters) + start_group_id += 1 + tdSql.query('show subscriptions;') + subscription_info = tdSql.queryResult + tdLog.info(f"---------- subscription_info: {subscription_info}") + if snapshot_value == "true": + if offset_value != "earliest" and offset_value != "": + if offset_value == "latest": + offset_value_list = list(map(lambda x: int(x[-2].replace("wal:", "").replace("earliest", "0").replace("latest", "0").replace(offset_value, "0")), subscription_info)) + tdSql.checkEqual(sum(offset_value_list) >= 0, True) + rows_value_list = list(map(lambda x: int(x[-1]), subscription_info)) + tdSql.checkEqual(sum(rows_value_list), expected_res) + elif offset_value == "none": + offset_value_list = list(map(lambda x: x[-2], subscription_info)) + tdSql.checkEqual(offset_value_list, ['none']*len(subscription_info)) + rows_value_list = list(map(lambda x: x[-1], subscription_info)) + tdSql.checkEqual(rows_value_list, [0]*len(subscription_info)) + else: + if offset_value != "none": + offset_value_str = ",".join(list(map(lambda x: x[-2], subscription_info))) + tdLog.info("checking tsdb in offset_value_str") + # tdSql.checkEqual("tsdb" in offset_value_str, True) + # rows_value_list = list(map(lambda x: int(x[-1]), subscription_info)) + # tdSql.checkEqual(sum(rows_value_list), expected_res) + else: + offset_value_list = list(map(lambda x: x[-2], subscription_info)) + tdSql.checkEqual(offset_value_list, [None]*len(subscription_info)) + rows_value_list = list(map(lambda x: x[-1], subscription_info)) + tdSql.checkEqual(rows_value_list, [None]*len(subscription_info)) + else: + if offset_value != "none": + offset_value_list = list(map(lambda x: int(x[-2].replace("wal:", "").replace("earliest", "0").replace("latest", "0").replace(offset_value, "0")), subscription_info)) + tdSql.checkEqual(sum(offset_value_list) >= 0, True) + rows_value_list = list(map(lambda x: int(x[-1]), subscription_info)) + tdSql.checkEqual(sum(rows_value_list), expected_res) + else: + offset_value_list = list(map(lambda x: x[-2], subscription_info)) + tdSql.checkEqual(offset_value_list, ['none']*len(subscription_info)) + rows_value_list = list(map(lambda x: x[-1], subscription_info)) + tdSql.checkEqual(rows_value_list, [0]*len(subscription_info)) + tdSql.execute(f"drop topic if exists {topic_name}") + tdSql.execute(f'drop database if exists {paraDict["dbName"]}') + + def run(self): + self.tmqParamsTest() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/7-tmq/tmqSeekAndCommit.py b/tests/system-test/7-tmq/tmqSeekAndCommit.py new file mode 100644 index 0000000000000000000000000000000000000000..2d837ef7a441e16dbbe8051d739da2877bf13643 --- /dev/null +++ b/tests/system-test/7-tmq/tmqSeekAndCommit.py @@ -0,0 +1,127 @@ +import sys +import re +import time +import threading +from taos.tmq import * +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + self.db_name = "tmq_db" + self.topic_name = "tmq_topic" + self.stable_name = "tmqst" + + + def prepareData(self): + # create database + tdSql.execute("create database if not exists %s;"%(self.db_name)) + tdSql.execute("use %s;"%(self.db_name)) + # create stable + tdSql.execute("create table %s.tmqst (ts timestamp, col0 int) tags(groupid int);"%(self.db_name)) + # create child tables + tdSql.execute("create table tmqct_1 using %s.%s tags(1);"%(self.db_name, self.stable_name)) + tdSql.execute("create table tmqct_2 using %s.%s tags(2);"%(self.db_name, self.stable_name)) + tdSql.execute("create table tmqct_3 using %s.%s tags(3);"%(self.db_name, self.stable_name)) + tdSql.execute("create table tmqct_4 using %s.%s tags(4);"%(self.db_name, self.stable_name)) + tdSql.execute("create table tmqct_5 using %s.%s tags(5);"%(self.db_name, self.stable_name)) + # insert into data + ctb_list = ["tmqct_1", "tmqct_2", "tmqct_3", "tmqct_4", "tmqct_5"] + for i in range(5): + sql = "insert into %s "%(ctb_list[i]) + sql_values = "values" + for j in range(1000 * i, 1000 * (i+1)): + sql_values += "(%s, %s)"%("now" if j == 0 else "now+%s"%(str(j) + "s"), str(j)) + sql += sql_values + ";" + tdLog.info(sql) + tdSql.execute(sql) + tdLog.info("Insert data into child tables successfully") + # create topic + tdSql.execute("create topic %s as select * from %s;"%(self.topic_name, self.stable_name)) + + def tmqSubscribe(self, inputDict): + consumer_dict = { + "group.id": inputDict['group_id'], + "client.id": "client", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.commit.interval.ms": "1000", + "enable.auto.commit": inputDict['auto_commit'], + "auto.offset.reset": inputDict['offset_reset'], + "experimental.snapshot.enable": "false", + "msg.with.table.name": "false" + } + + consumer = Consumer(consumer_dict) + try: + consumer.subscribe([inputDict['topic_name']]) + except Exception as e: + tdLog.info("consumer.subscribe() fail ") + tdLog.info("%s"%(e)) + + tdLog.info("create consumer success!") + return consumer + + def test_seek_and_committed_position_with_autocommit(self): + try: + self.prepareData() + inputDict = { + "topic_name": self.topic_name, + "group_id": "1", + "auto_commit": "true", + "offset_reset": "earliest" + } + consumer = self.tmqSubscribe(inputDict) + while(True): + res = consumer.poll(1) + if not res: + break + err = res.error() + if err is not None: + raise err + val = res.value() + for block in val: + tdLog.info("block.fetchall() number: %s"%(len(block.fetchall()))) + + partitions = consumer.assignment() + position_partitions = consumer.position(partitions) + tdLog.info("position_partitions: %s"%(position_partitions)) + for i in range(len(position_partitions)): + tdLog.info("position_partitions[%s].offset: %s"%(i, position_partitions[i].offset)) + committed_partitions = consumer.committed(partitions) + tdLog.info("committed_partitions: %s"%(committed_partitions)) + for i in range(len(committed_partitions)): + tdLog.info("committed_partitions[%s].offset: %s"%(i, committed_partitions[i].offset)) + assert(len(position_partitions) == len(committed_partitions)) + for i in range(len(position_partitions)): + assert(position_partitions[i].offset == committed_partitions[i].offset) + # seek to the beginning of the topic + + except Exception as ex: + raise Exception("Failed to test seek and committed position with autocommit with error: {}".format(str(ex))) + finally: + consumer.unsubscribe() + consumer.close() + + def test_commit_by_offset(self): + pass + + def run(self): + self.test_seek_and_committed_position_with_autocommit() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py index e9234f6c7aea99cf339be76ed799b34d3ec44721..31ddc1b0f831df5d6e10f4337536488ddeaa892e 100644 --- a/tests/system-test/7-tmq/tmqShow.py +++ b/tests/system-test/7-tmq/tmqShow.py @@ -51,7 +51,6 @@ class TDTestCase: consumerIdList = [0, 1, 2, 3] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py index 7f322dc2583201bce47f40a9c1883daabadf8e42..85222a941bf466631feedd1ead0387ef5f984096 100644 --- a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py +++ b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py @@ -94,7 +94,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replica) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py index bee38ca8ee192c542ebc2c09718d4e4224904c1e..5f9fb42c453ed49d16461ec7ec597bff69c3ed46 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py @@ -88,7 +88,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -136,7 +135,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -239,7 +237,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py index d3b64d2b2150521c1a3bc98fe41734390609084f..6278527c64061806013e64367c65bc685d76386b 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py @@ -88,7 +88,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -136,7 +135,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -239,7 +237,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUdf.py b/tests/system-test/7-tmq/tmqUdf.py index 5da1625cb125091dddc6ddbe3e635dc404352bbb..8af4406f780916087cbf5c6a8fffe549d90b42af 100644 --- a/tests/system-test/7-tmq/tmqUdf.py +++ b/tests/system-test/7-tmq/tmqUdf.py @@ -88,7 +88,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -136,7 +135,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -240,7 +238,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUpdate-1ctb.py b/tests/system-test/7-tmq/tmqUpdate-1ctb.py index 8fdf7748a3425b1efe7966949a1098e5867b6ee2..920e8e77e466eaf27c99fd9ac462bbd27a18ed94 100644 --- a/tests/system-test/7-tmq/tmqUpdate-1ctb.py +++ b/tests/system-test/7-tmq/tmqUpdate-1ctb.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py index 8b67f6f8252e5e3682284c5c2d5b4295308db26c..2f1d3e263175e2adedc6fad6f4693a0fcda0d4bb 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py @@ -55,7 +55,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py index 5a35c4f5ee84c950ee7a611e5be93fe13371b031..6b8c10de271b28726639baad8648e2586c02f9a1 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py @@ -55,7 +55,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py index 84617efae4d78cf57cf6d522a7b3fbf50321e5f2..3975013e747f09d8cc0548b5e32f9f01c63fb9a4 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py @@ -55,7 +55,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmq_offset.py b/tests/system-test/7-tmq/tmq_offset.py new file mode 100644 index 0000000000000000000000000000000000000000..6453f452c61be3cbf55f3e518bbc8d8e5d61f604 --- /dev/null +++ b/tests/system-test/7-tmq/tmq_offset.py @@ -0,0 +1,47 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + buildPath = tdCom.getBuildPath() + cmdStr1 = '%s/build/bin/taosBenchmark -i 50 -B 1 -t 1000 -n 100000 -y &'%(buildPath) + tdLog.info(cmdStr1) + os.system(cmdStr1) + time.sleep(15) + + cmdStr2 = '%s/build/bin/tmq_offset_test &'%(buildPath) + tdLog.info(cmdStr2) + os.system(cmdStr2) + + time.sleep(20) + + os.system("kill -9 `pgrep taosBenchmark`") + result = os.system("kill -9 `pgrep tmq_offset_test`") + if result != 0: + tdLog.exit("tmq_offset_test error!") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index c3ec4875ceeeb25a0ee3b9f921a8e7d5c40fdc07..86c40fdc727fbba7d0167209c14dbb8714a318b4 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -220,6 +220,17 @@ class TDTestCase: return + def checkWal1VgroupOnlyMeta(self): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + cmdStr = '%s/build/bin/tmq_taosx_ci -c %s -sv 1 -dv 1 -d -onlymeta'%(buildPath, cfgPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + self.checkJson(cfgPath, "tmq_taosx_tmp") + + return + def checkWal1VgroupTable(self): buildPath = tdCom.getBuildPath() cfgPath = tdCom.getClientCfgPath() @@ -301,6 +312,8 @@ class TDTestCase: def run(self): tdSql.prepare() + self.checkWal1VgroupOnlyMeta() + self.checkWal1Vgroup() self.checkSnapshot1Vgroup() diff --git a/tests/system-test/99-TDcase/TD-16821.py b/tests/system-test/99-TDcase/TD-16821.py index 2e23002059c5d157b3e1f6edf2f21a5d291739bc..26b41e6afc9a7e90b8993ef9ca1b8cf33eb27e05 100644 --- a/tests/system-test/99-TDcase/TD-16821.py +++ b/tests/system-test/99-TDcase/TD-16821.py @@ -45,7 +45,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") diff --git a/tests/system-test/99-TDcase/TD-17255.py b/tests/system-test/99-TDcase/TD-17255.py index 5f68a5b7389d3ea095a8886e2801dd82fb6af8d9..0f83468754783ee5adca78113c7a8df5c78bbe99 100644 --- a/tests/system-test/99-TDcase/TD-17255.py +++ b/tests/system-test/99-TDcase/TD-17255.py @@ -53,7 +53,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -98,7 +97,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -183,7 +181,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -270,7 +267,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("insert data by auto create ctb") diff --git a/tests/system-test/99-TDcase/TD-17699.py b/tests/system-test/99-TDcase/TD-17699.py index 6956e88aec7d0c8adb84c8af9aff31cb6d80fa1f..2862f4a78d5391098f0493afa35e482c5a032817 100644 --- a/tests/system-test/99-TDcase/TD-17699.py +++ b/tests/system-test/99-TDcase/TD-17699.py @@ -65,7 +65,6 @@ class TDTestCase: tmqCom.initConsumerTable(self.cdbName) tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"]) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) self.paraDict["stbName"] = 'stb1' tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) diff --git a/tests/system-test/99-TDcase/TS-3311.py b/tests/system-test/99-TDcase/TS-3311.py new file mode 100644 index 0000000000000000000000000000000000000000..ce39597a7b5dd20ad76818c4078b02a3736e219e --- /dev/null +++ b/tests/system-test/99-TDcase/TS-3311.py @@ -0,0 +1,121 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def create_tables(self): + tdSql.execute("create database if not exists dbus precision 'us'") + tdSql.execute("create database if not exists dbns precision 'ns'") + + tdSql.execute("use dbus") + + tdSql.execute(f"CREATE STABLE `stb_us` (`ts` TIMESTAMP, `ip_value` FLOAT, `ip_quality` INT) TAGS (`t1` INT)") + tdSql.execute(f"CREATE TABLE `ctb1_us` USING `stb_us` (`t1`) TAGS (1)") + tdSql.execute(f"CREATE TABLE `ctb2_us` USING `stb_us` (`t1`) TAGS (2)") + + tdSql.execute("use dbns") + + tdSql.execute(f"CREATE STABLE `stb_ns` (`ts` TIMESTAMP, `ip_value` FLOAT, `ip_quality` INT) TAGS (`t1` INT)") + tdSql.execute(f"CREATE TABLE `ctb1_ns` USING `stb_ns` (`t1`) TAGS (1)") + tdSql.execute(f"CREATE TABLE `ctb2_ns` USING `stb_ns` (`t1`) TAGS (2)") + + def insert_data(self): + tdLog.debug("start to insert data ............") + + tdSql.execute(f"INSERT INTO `dbus`.`ctb1_us` VALUES ('2023-07-01 00:00:00.000', 10.30000, 100)") + tdSql.execute(f"INSERT INTO `dbus`.`ctb2_us` VALUES ('2023-08-01 00:00:00.000', 20.30000, 200)") + + tdSql.execute(f"INSERT INTO `dbns`.`ctb1_ns` VALUES ('2023-07-01 00:00:00.000', 10.30000, 100)") + tdSql.execute(f"INSERT INTO `dbns`.`ctb2_ns` VALUES ('2023-08-01 00:00:00.000', 20.30000, 200)") + + tdLog.debug("insert data ............ [OK]") + + def run(self): + tdSql.prepare() + self.create_tables() + self.insert_data() + tdLog.printNoPrefix("======== test TS-3311") + + # test ns + tdSql.query(f"select _wstart, _wend, count(*) from `dbns`.`stb_ns` interval(1n)") + tdSql.checkRows(2) + + tdSql.checkData(0, 0, '2023-07-01 00:00:00.000000000') + tdSql.checkData(1, 0, '2023-08-01 00:00:00.000000000') + + tdSql.checkData(0, 1, '2023-08-01 00:00:00.000000000') + tdSql.checkData(1, 1, '2023-09-01 00:00:00.000000000') + + tdSql.query(f"select _wstart, _wend, count(*) from `dbns`.`stb_ns` interval(12n)") + tdSql.checkRows(1) + + tdSql.checkData(0, 0, '2023-01-01 00:00:00.000000000') + tdSql.checkData(0, 1, '2024-01-01 00:00:00.000000000') + + tdSql.query(f"select _wstart, _wend, count(*) from `dbns`.`stb_ns` interval(1y)") + tdSql.checkRows(1) + + tdSql.checkData(0, 0, '2023-01-01 00:00:00.000000000') + tdSql.checkData(0, 1, '2024-01-01 00:00:00.000000000') + + + ## test us + tdSql.query(f"select _wstart, _wend, count(*) from `dbus`.`stb_us` interval(1n)") + tdSql.checkRows(2) + + tdSql.checkData(0, 0, '2023-07-01 00:00:00.000000') + tdSql.checkData(1, 0, '2023-08-01 00:00:00.000000') + + tdSql.checkData(0, 1, '2023-08-01 00:00:00.000000') + tdSql.checkData(1, 1, '2023-09-01 00:00:00.000000') + + tdSql.query(f"select _wstart, _wend, count(*) from `dbus`.`stb_us` interval(12n)") + tdSql.checkRows(1) + + tdSql.checkData(0, 0, '2023-01-01 00:00:00.000000') + tdSql.checkData(0, 1, '2024-01-01 00:00:00.000000') + + tdSql.query(f"select _wstart, _wend, count(*) from `dbus`.`stb_us` interval(1y)") + tdSql.checkRows(1) + + tdSql.checkData(0, 0, '2023-01-01 00:00:00.000000') + tdSql.checkData(0, 1, '2024-01-01 00:00:00.000000') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/99-TDcase/TS-3404.py b/tests/system-test/99-TDcase/TS-3404.py new file mode 100644 index 0000000000000000000000000000000000000000..3fdfb9a19378747b276678fe3c350bae5de18bc2 --- /dev/null +++ b/tests/system-test/99-TDcase/TS-3404.py @@ -0,0 +1,101 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def create_tables(self): + tdSql.execute(f"CREATE STABLE `stb5` (`ts` TIMESTAMP, `ip_value` FLOAT, `ip_quality` INT) TAGS (`t1` INT)") + tdSql.execute(f"CREATE TABLE `t_11` USING `stb5` (`t1`) TAGS (1)") + + def insert_data(self): + tdLog.debug("start to insert data ............") + + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-10 09:30:47.722', 10.30000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-10 09:30:56.383', 12.30000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-10 09:48:55.778', 13.30000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-10 09:51:50.821', 9.30000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-10 09:58:07.162', 9.30000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-10 13:41:16.075', 9.30000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-13 14:12:58.318', 21.00000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-13 14:13:21.328', 1.10000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-13 14:35:24.258', 1.30000, 100)") + tdSql.execute(f"INSERT INTO `t_11` VALUES ('2023-05-13 16:56:49.033', 1.80000, 100)") + + tdLog.debug("insert data ............ [OK]") + + def run(self): + tdSql.prepare() + self.create_tables() + self.insert_data() + tdLog.printNoPrefix("======== test TS-3404") + + tdSql.query(f"select _irowts, interp(ip_value) from t_11 range('2023-05-13 14:00:00', '2023-05-13 15:00:00') every(300s) fill(linear);") + tdSql.checkRows(13) + + tdSql.checkData(0, 0, '2023-05-13 14:00:00.000') + tdSql.checkData(1, 0, '2023-05-13 14:05:00.000') + tdSql.checkData(2, 0, '2023-05-13 14:10:00.000') + tdSql.checkData(3, 0, '2023-05-13 14:15:00.000') + tdSql.checkData(4, 0, '2023-05-13 14:20:00.000') + tdSql.checkData(5, 0, '2023-05-13 14:25:00.000') + tdSql.checkData(6, 0, '2023-05-13 14:30:00.000') + tdSql.checkData(7, 0, '2023-05-13 14:35:00.000') + tdSql.checkData(8, 0, '2023-05-13 14:40:00.000') + tdSql.checkData(9, 0, '2023-05-13 14:45:00.000') + tdSql.checkData(10, 0, '2023-05-13 14:50:00.000') + tdSql.checkData(11, 0, '2023-05-13 14:55:00.000') + tdSql.checkData(12, 0, '2023-05-13 15:00:00.000') + + tdSql.checkData(0, 1, 20.96512) + tdSql.checkData(1, 1, 20.97857) + tdSql.checkData(2, 1, 20.99201) + tdSql.checkData(3, 1, 1.114917) + tdSql.checkData(4, 1, 1.160271) + tdSql.checkData(5, 1, 1.205625) + tdSql.checkData(6, 1, 1.250978) + tdSql.checkData(7, 1, 1.296333) + tdSql.checkData(8, 1, 1.316249) + tdSql.checkData(9, 1, 1.333927) + tdSql.checkData(10, 1, 1.351607) + tdSql.checkData(11, 1, 1.369285) + tdSql.checkData(12, 1, 1.386964) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/99-TDcase/TS-3581.py b/tests/system-test/99-TDcase/TS-3581.py new file mode 100644 index 0000000000000000000000000000000000000000..18488af0a6a4a8c2d0650156d468f589c93325a7 --- /dev/null +++ b/tests/system-test/99-TDcase/TS-3581.py @@ -0,0 +1,79 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def create_tables(self): + tdSql.execute(f'''CREATE STABLE `dwd_log_master` (`ts` TIMESTAMP, `dim_ip` NCHAR(64)) TAGS (`group_id` BIGINT, `st_hour` NCHAR(2), `org_id` NCHAR(32), + `dev_manufacturer_name` NCHAR(64), `dev_manufacturer_id` INT, `dev_category_name` NCHAR(64), `dev_category_id` INT, `dev_feature_name` NCHAR(64), + `dev_feature_id` INT, `dev_ip` NCHAR(64), `black_list` TINYINT, `white_list` TINYINT)''') + tdSql.execute(f'''CREATE TABLE `dwd_log_master_475021043` USING `dwd_log_master` (`group_id`, `st_hour`, `org_id`, `dev_manufacturer_name`, `dev_manufacturer_id`, + `dev_category_name`, `dev_category_id`, `dev_feature_name`, `dev_feature_id`, `dev_ip`, `black_list`, `white_list`) TAGS + (475021043, "14", NULL, NULL, NULL, NULL, NULL, NULL, NULL, "172.18.22.230", NULL, NULL)''') + + def insert_data(self): + tdLog.debug("start to insert data ............") + + tdSql.execute(f"INSERT INTO `dwd_log_master_475021043` VALUES ('2023-06-26 14:38:30.000','192.168.192.102')") + tdSql.execute(f"INSERT INTO `dwd_log_master_475021043` VALUES ('2023-06-26 14:38:31.000','172.18.23.249')") + tdSql.execute(f"INSERT INTO `dwd_log_master_475021043` VALUES ('2023-06-26 14:38:32.000','192.168.200.231')") + tdSql.execute(f"INSERT INTO `dwd_log_master_475021043` VALUES ('2023-06-26 14:38:33.000','172.18.22.231')") + tdSql.execute(f"INSERT INTO `dwd_log_master_475021043` VALUES ('2023-06-26 14:38:34.000','192.168.210.231')") + tdSql.execute(f"INSERT INTO `dwd_log_master_475021043` VALUES ('2023-06-26 14:38:35.000','192.168.192.100')") + tdSql.execute(f"INSERT INTO `dwd_log_master_475021043` VALUES ('2023-06-26 14:38:36.000','192.168.192.231')") + tdSql.execute(f"INSERT INTO `dwd_log_master_475021043` VALUES ('2023-06-26 14:38:37.000','172.18.23.231')") + + tdLog.debug("insert data ............ [OK]") + + def run(self): + tdSql.prepare() + self.create_tables() + self.insert_data() + tdLog.printNoPrefix("======== test TS-3581") + + for i in range(100): + tdSql.query(f"select first(ts), last(ts), count(*) from dwd_log_master;") + tdSql.checkRows(1) + print(tdSql.queryResult) + tdSql.checkData(0, 0, '2023-06-26 14:38:30.000') + return + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/eco-system/main.py b/tests/system-test/eco-system/main.py new file mode 100644 index 0000000000000000000000000000000000000000..a1f72147a02273676283a3af18e2b7aa45c1cda1 --- /dev/null +++ b/tests/system-test/eco-system/main.py @@ -0,0 +1,37 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import re +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + + def run(self): + tdLog.info(" ------ eco-system main -------") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/eco-system/manager/cmul.py b/tests/system-test/eco-system/manager/cmul.py new file mode 100644 index 0000000000000000000000000000000000000000..ac2fa5e4f28743513e7c78c3a9c3ecbc5c7592e9 --- /dev/null +++ b/tests/system-test/eco-system/manager/cmul.py @@ -0,0 +1,104 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# The option for wal_retetion_period and wal_retention_size is work well +# + +import taos +from taos.tmq import Consumer + +import os +import sys +import threading +import json +import time +import random +from datetime import date +from datetime import datetime +from datetime import timedelta +from os import path + + +topicName = "topic" +topicNum = 100 + +# consume topic +def consume_topic(topic_name, group,consume_cnt, index, wait): + consumer = Consumer( + { + "group.id": group, + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "enable.auto.commit": "true", + } + ) + + print(f"start consumer topic:{topic_name} group={group} index={index} ...") + consumer.subscribe([topic_name]) + cnt = 0 + try: + while True and cnt < consume_cnt: + res = consumer.poll(1) + if not res: + if wait: + continue + else: + break + err = res.error() + if err is not None: + raise err + val = res.value() + cnt += 1 + print(f" consume {cnt} ") + for block in val: + datas = block.fetchall() + data = datas[0][:50] + + print(f" {topic_name}_{group}_{index} {cnt} {data}") + + finally: + consumer.unsubscribe() + consumer.close() + +def consumerThread(index): + global topicName, topicNum + print(f' thread {index} start...') + while True: + idx = random.randint(0, topicNum - 1) + name = f"{topicName}{idx}" + group = f"group_{index}_{idx}" + consume_topic(name, group, 100, index, True) + + + +if __name__ == "__main__": + print(sys.argv) + threadCnt = 10 + + if len(sys.argv) == 1: + threadCnt = int(sys.argv[1]) + + + threads = [] + print(f'consumer with {threadCnt} threads...') + for i in range(threadCnt): + x = threading.Thread(target=consumerThread, args=(i,)) + x.start() + threads.append(x) + + # wait + for i, thread in enumerate(threads): + thread.join() + print(f'join thread {i} end.') + diff --git a/tests/system-test/eco-system/manager/drop_table.py b/tests/system-test/eco-system/manager/drop_table.py new file mode 100644 index 0000000000000000000000000000000000000000..d4cdf4a5410fecb7dc8f339494d2a16b178ff411 --- /dev/null +++ b/tests/system-test/eco-system/manager/drop_table.py @@ -0,0 +1,146 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(0, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + # show tables + def show_tables(self): + sql = "show tables;" + row_cnt = tdSql.query(sql) + tables = [] + for i in range(0, row_cnt): + tb_name = tdSql.queryResult[i][0] + tables.append(tb_name) + + + # execute sql + def execute(self, sql): + try: + tdSql.execute(sql, 3) + tdLog.info(f" exec ok. {sql}") + except: + tdLog.info(f" exe failed. {sql}") + traceback.print_exc() + + + # query + def query_table(self, columns, tags): + if len(columns) < 5 : + return + if len(tags) < 5: + return + + sel_cols = random.sample(columns, random.randint(1,int(len(columns)-1))) + sel_tags = random.sample(tags, random.randint(1, int(len(tags)-1))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def drop_table(self, change_cnt): + # init + + tables = self.show_tables() + + + for i in range(change_cnt): + col_idx = random.randint(0, ncol - 1) + tag_idx = random.randint(0, ntag - 1) + + cols = list(self.column_dict.keys()) + tags = list(self.tag_dict.keys()) + + # column + key = cols[col_idx] + value = self.column_dict[key] + sql = f'alter table meters drop column {key}' + self.execute(sql) + sql = f'alter table meters add column {key} {value}' + self.execute(sql) + + + # column + key = tags[col_idx] + value = self.tag_dict[key] + sql = f'alter table meters drop tag {key}' + self.execute(sql) + sql = f'alter table meters add tag {key} {value}' + self.execute(sql) + + # drop and rename + if i % 5 == 0: + # update columns + #columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(cols)} tags= {len(tags)}======") + self.query_table(cols, tags) + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.drop_table(1000000) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/manager/mul.py b/tests/system-test/eco-system/manager/mul.py new file mode 100644 index 0000000000000000000000000000000000000000..d78b63d386209d06a78ff9f77aa8552d08f1a181 --- /dev/null +++ b/tests/system-test/eco-system/manager/mul.py @@ -0,0 +1,114 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import sys +import random +import time + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + + # prepareEnv + def prepareEnv(self): + self.dbName = "mullevel" + self.stbName = "meters" + self.topicName = "topic" + self.topicNum = 100 + self.loop = 50000 + + sql = f"use {self.dbName}" + tdSql.execute(sql) + + # generate topic sql + self.sqls = [ + f"select * from {self.stbName}", + f"select * from {self.stbName} where ui < 200", + f"select * from {self.stbName} where fc > 20.1", + f"select * from {self.stbName} where nch like '%%a%%'", + f"select * from {self.stbName} where fc > 20.1", + f"select lower(bin) from {self.stbName} where length(bin) < 10;", + f"select upper(bin) from {self.stbName} where length(nch) > 10;", + f"select upper(bin) from {self.stbName} where ti > 10 or ic < 40;", + f"select * from {self.stbName} where ic < 100 " + ] + + + + # prepareEnv + def createTopics(self): + for i in range(self.topicNum): + topicName = f"{self.topicName}{i}" + sql = random.choice(self.sqls) + createSql = f"create topic if not exists {topicName} as {sql}" + try: + tdSql.execute(createSql, 3, True) + except: + tdLog.info(f" create topic {topicName} failed.") + + + # random del topic + def managerTopics(self): + + for i in range(self.loop): + tdLog.info(f"start modify loop={i}") + idx = random.randint(0, self.topicNum - 1) + # delete + topicName = f"{self.topicName}{idx}" + sql = f"drop topic if exist {topicName}" + try: + tdSql.execute(sql, 3, True) + except: + tdLog.info(f" drop topic {topicName} failed.") + + + # create topic + sql = random.choice(self.sqls) + createSql = f"create topic if not exists {topicName} as {sql}" + try: + tdSql.execute(createSql, 3, True) + except: + tdLog.info(f" create topic {topicName} failed.") + + seconds = [0.1, 0.5, 3, 2.5, 1.5, 0.4, 5.2, 2.6, 0.4, 0.2] + time.sleep(random.choice(seconds)) + + + # run + def run(self): + # prepare env + self.prepareEnv() + + # create topic + self.createTopics() + + # modify topic + self.managerTopics() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/eco-system/manager/same_column.py b/tests/system-test/eco-system/manager/same_column.py new file mode 100644 index 0000000000000000000000000000000000000000..beaf4e449e76686b8c7240b21d0dcaae8f4eb9e0 --- /dev/null +++ b/tests/system-test/eco-system/manager/same_column.py @@ -0,0 +1,181 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.column_dict = { + 'bc': 'bool', + 'fc': 'float', + 'dc': 'double', + 'ti': 'tinyint', + 'si': 'smallint', + 'ic': 'int', + 'bi': 'bigint', + 'uit': 'tinyint unsigned', + 'usi': 'smallint unsigned', + 'ui': 'int unsigned', + 'ubi': 'bigint unsigned', + 'bin': 'binary(32)', + 'nch': 'nchar(64)' + } + self.tag_dict = { + 'groupid': 'tinyint', + 'location': 'binary(16)', + 'tfc': 'float', + 'tdc': 'double', + 'tti': 'tinyint', + 'tsi': 'smallint', + 'tic': 'int', + 'tbi': 'bigint', + 'tuit': 'tinyint unsigned', + 'tusi': 'smallint unsigned', + 'tui': 'int unsigned', + 'tubi': 'bigint unsigned', + 'tbin': 'binary(32)', + 'tnch': 'nchar(64)' + } + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(20, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + def drop_tag(self, tags, cnt): + for i in range(cnt): + tag_cnt = len(tags) + sel = random.randint(1, tag_cnt-1) + sql = f"alter table meters drop tag `{tags[sel]}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + del tags[sel] + except: + tdLog.info(f" drop tags failed. {sql}") + traceback.print_exc() + + # execute sql + def execute(self, sql): + try: + tdSql.execute(sql, 3) + tdLog.info(f" exec ok. {sql}") + except: + tdLog.info(f" exe failed. {sql}") + traceback.print_exc() + + + # query + def query_table(self, columns, tags): + if len(columns) < 5 : + return + if len(tags) < 5: + return + + sel_cols = random.sample(columns, random.randint(1,int(len(columns)-1))) + sel_tags = random.sample(tags, random.randint(1, int(len(tags)-1))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def change_columns(self, change_cnt): + # init + + ncol = len(self.column_dict) + ntag = len(self.tag_dict) + + for i in range(change_cnt): + col_idx = random.randint(0, ncol - 1) + tag_idx = random.randint(0, ntag - 1) + + cols = list(self.column_dict.keys()) + tags = list(self.tag_dict.keys()) + + # column + key = cols[col_idx] + value = self.column_dict[key] + sql = f'alter table meters drop column {key}' + self.execute(sql) + sql = f'alter table meters add column {key} {value}' + self.execute(sql) + + + # column + key = tags[col_idx] + value = self.tag_dict[key] + sql = f'alter table meters drop tag {key}' + self.execute(sql) + sql = f'alter table meters add tag {key} {value}' + self.execute(sql) + + # drop and rename + if i % 5 == 0: + # update columns + #columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(cols)} tags= {len(tags)}======") + self.query_table(cols, tags) + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.change_columns(1000000) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/manager/schema_change.py b/tests/system-test/eco-system/manager/schema_change.py new file mode 100644 index 0000000000000000000000000000000000000000..400d2b100b4d99c3f0c3455e047caa7e4d45573f --- /dev/null +++ b/tests/system-test/eco-system/manager/schema_change.py @@ -0,0 +1,239 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.column_dict = { + 'col0': 'int', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': 'varchar(20)', + 'col13': 'nchar(20)' + } + self.tag_dict = { + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': 'varchar(20)', + 't13': 'nchar(20)', + 't14': 'timestamp' + } + + + # delete + def delete_col(self, columns, cnt, max_col): + # delte for random + for i in range(cnt): + col_cnt = len(columns) + if col_cnt == 0: + return + sel = random.randint(0, col_cnt - 1) + sql = f"alter table meters drop column `{columns[sel]}`" + try: + tdSql.execute(sql) + tdLog.info(f" drop cur col={len(columns)} max_col={max_col} {sql}") + del columns[sel] + except: + tdLog.info(f" drop column failed. {sql}") + traceback.print_exc() + + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(20, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + def renames(self, tags, cnt): + col_cnt = len(tags) + if col_cnt < 10: + return + for i in range(cnt): + sel = random.randint(1, col_cnt-3) + new_name = tags[sel] + "n" + sql = f"alter table meters rename tag `{tags[sel]}` `{new_name}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + tags[sel] = new_name + except: + tdLog.info(f" rename tag failed. {sql}") + traceback.print_exc() + + + def drop_tag(self, tags, cnt): + for i in range(cnt): + tag_cnt = len(tags) + sel = random.randint(1, tag_cnt-1) + sql = f"alter table meters drop tag `{tags[sel]}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + del tags[sel] + except: + tdLog.info(f" drop tags failed. {sql}") + traceback.print_exc() + + # query + def query_table(self, columns, tags): + if len(columns) < 10 : + return + if len(tags) < 10: + return + + sel_cols = random.sample(columns, random.randint(2,int(len(columns)/10))) + sel_tags = random.sample(tags, random.randint(1,int(len(tags)/10))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def change_schema(self, change_cnt): + # init + columns, tags = self.describe_table("meters") + max_col = random.randint(200, 2000) + tdLog.info(f" ----------- set max column = {max_col} -------------") + for i in range(change_cnt): + col_cnt = len(self.column_dict) + icol = random.randint(0, col_cnt-1) + key = f"col{icol}" + col_name = key + f"_{i}_{random.randint(1,100)}" + col_type = self.column_dict[key] + sql = f'alter table meters add column `{col_name}` {col_type}' + sql_tag = f'alter table meters add tag `t_{col_name}` {col_type}' + + try: + tdSql.execute(sql) + tdLog.info(f" add cur col={len(columns)} max_col={max_col} {sql}") + columns.append(col_name) + if random.randint(1, 4) == 2: + tdSql.execute(sql_tag) + tdLog.info(f" add tag tag_cnt={len(tags)} {sql_tag}") + + except: + tdLog.info(f" add column failed. {sql}") + traceback.print_exc() + + + col_cnt = len(columns) + # delete + if col_cnt > max_col + 100: + self.delete_col(columns, random.randint(1, 30), max_col) + elif col_cnt >= max_col + 30: + self.delete_col(columns, random.randint(1, 4), max_col) + max_col = random.randint(200, 2000) + tdLog.info(f" ----------- set max column = {max_col} -------------") + elif col_cnt > max_col: + self.delete_col(columns, random.randint(1, 3), max_col) + + + + if i % 50 == 0: + sql = f"flush database {self.dbname};" + tdSql.execute(sql) + tdLog.info(f" ***** {sql} *****") + + # query + if i % 70 == 0: + self.query_table(columns, tags) + + # drop and rename + if i % 10 == 0: + # update columns + columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(columns)} tags= {len(tags)}======") + + if random.randint(1,3) == 2: + self.query_table(columns, tags) + + if len(tags) > 50: + self.drop_tag(tags, random.randint(1, 30)) + + self.renames(tags, random.randint(1, 10)) + + + # sleep + #time.sleep(0.3) + + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.change_schema(1000000) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/schemaless/insert.py b/tests/system-test/eco-system/schemaless/insert.py new file mode 100644 index 0000000000000000000000000000000000000000..901196ebfd45232d462612f48bd89f0bf0d312e3 --- /dev/null +++ b/tests/system-test/eco-system/schemaless/insert.py @@ -0,0 +1,151 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback +import taos +import string +from taos import schemaless + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.conn = conn + self.schema = {} + + def random_string(self, count): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(count)) + + def genCol(self, col_name, isTag): + col_types = ["str","f64","f32","i8","u8","i16","u16","i32","u32","i64","u64"] + if self.schema.get(col_name) == None: + col_type = random.choice(col_types) + self.schema[col_name] = col_type + else: + col_type = self.schema[col_name] + + is_num = True + val = "" + if col_type == "str": + val = self.random_string(random.randint(1, 10)) + is_num = False + elif col_type == "f64": + val = random.randrange(-100000000000000, 1000000000000)/3*2.25678 + elif col_type == "f32": + val = random.randrange(-100000000, 1000000000)/3*1.2345 + elif col_type == "i8": + val = random.randint(-128, 127) + elif col_type == "u8": + val = random.randint(0, 256) + elif col_type == "i16": + val = random.randint(-32768, 32767) + elif col_type == "u16": + val = random.randint(0, 256*256) + elif col_type == "i32": + val = random.randint(-256*256*256*128, 256*256*256*128) + elif col_type == "u32": + val = random.randint(0, 256*256*256*256) + elif col_type == "i64": + val = random.randint(-256*256*256*256*256*256*256*128, 256*256*256*256*256*256*256*128) + elif col_type == "u64": + val = random.randint(0, 256*256*256*256*256*256*256*256) + else: + val = 100 + + if isTag: + col_val = val + elif is_num: + col_val = f'{val}{col_type}' + else: + col_val = '"' + val + '"' + + return f'{col_name}={col_val}' + + + # cols + def genCols(self, pre, max, index, isTag): + col_cnt = random.randint(1, max) + cols = [] + for i in range(col_cnt): + col_name = f'{pre}_{index}_{i}' + cols.append(self.genCol(col_name, isTag)) + + return ",".join(cols) + + + # execute sql + def insert(self,sql,i): + print("schema less insert") + try: + self.conn.schemaless_insert([sql], schemaless.SmlProtocol.LINE_PROTOCOL, schemaless.SmlPrecision.MILLI_SECONDS) + tdLog.info(f" exec ok i={i} {sql}") + except: + tdLog.info(f" exe failed. i={i} {sql}") + traceback.print_exc() + + def genTags(self, i): + tags = f"t1={i},t2=abc,t3=work" + return tags + + # change table schema + def schemaless_insert(self, change_cnt): + # init + ts = 1683194263000 + for i in range(change_cnt): + index = int(i/10000) % 600 + cols = self.genCols("c", 5, index, False) + tags = self.genTags(index) + sql = f'{self.stable},{tags} {cols} {ts + i}' + self.insert(sql, i) + + # run + + def run(self): + # seed + #random.seed(int(time.time())) + self.dbname = "eco_system" + self.stable = "sml_stb" + + # switch db + tdSql.execute(f"use {self.dbname};") + tdSql.execute(f"drop table if exists {self.stable};") + + + + # change meters + try: + self.schemaless_insert(1000000) + except: + traceback.print_exc() + + print(self.schema) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/eco-system/util/Consumer.py b/tests/system-test/eco-system/util/Consumer.py new file mode 100644 index 0000000000000000000000000000000000000000..b483253a9582efe7202ff90de22c855f87fcee3b --- /dev/null +++ b/tests/system-test/eco-system/util/Consumer.py @@ -0,0 +1,82 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# The option for wal_retetion_period and wal_retention_size is work well +# + +import taos +from taos.tmq import Consumer + +import os +import sys +import threading +import json +import time +from datetime import date +from datetime import datetime +from datetime import timedelta +from os import path + + +# consume topic +def consume_topic(topic_name, consume_cnt, wait): + print("start consume...") + consumer = Consumer( + { + "group.id": "tg2", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "enable.auto.commit": "true", + } + ) + print("start subscrite...") + consumer.subscribe([topic_name]) + + cnt = 0 + try: + while True and cnt < consume_cnt: + res = consumer.poll(1) + if not res: + if wait: + continue + else: + break + err = res.error() + if err is not None: + raise err + val = res.value() + cnt += 1 + print(f" consume {cnt} ") + for block in val: + print(block.fetchall()) + finally: + consumer.unsubscribe() + consumer.close() + + +if __name__ == "__main__": + print(sys.argv) + if len(sys.argv) < 2: + + print(" please input topic name for consume . -c for wait") + else: + wait = False + if "-c" == sys.argv[1]: + wait = True + topic = sys.argv[2] + else: + topic = sys.argv[1] + + print(f' wait={wait} topic={topic}') + consume_topic(topic, 10000000, wait) \ No newline at end of file diff --git a/tests/system-test/eco-system/util/restartDnodes.py b/tests/system-test/eco-system/util/restartDnodes.py new file mode 100644 index 0000000000000000000000000000000000000000..feee260fdf49957577a2204831de54e8183b222c --- /dev/null +++ b/tests/system-test/eco-system/util/restartDnodes.py @@ -0,0 +1,84 @@ +import time +import os +import subprocess +import random +import platform + +class dnode(): + def __init__(self, pid, path): + self.pid = pid + self.path = path + +# run exePath no wait finished +def runNoWait(exePath): + if platform.system().lower() == 'windows': + cmd = f"mintty -h never {exePath}" + else: + cmd = f"nohup {exePath} > /dev/null 2>&1 & " + + if os.system(cmd) != 0: + return False + else: + return True + +# get online dnodes +def getDnodes(): + cmd = "ps aux | grep taosd | awk '{{print $2,$11,$12,$13}}'" + result = os.system(cmd) + result=subprocess.check_output(cmd,shell=True) + strout = result.decode('utf-8').split("\n") + dnodes = [] + + for line in strout: + cols = line.split(' ') + if len(cols) != 4: + continue + exepath = cols[1] + if len(exepath) < 5 : + continue + if exepath[-5:] != 'taosd': + continue + + # add to list + path = cols[1] + " " + cols[2] + " " + cols[3] + dnodes.append(dnode(cols[0], path)) + + print(" show dnodes cnt=%d...\n"%(len(dnodes))) + for dn in dnodes: + print(f" pid={dn.pid} path={dn.path}") + + return dnodes + +def restartDnodes(dnodes, cnt, seconds): + print(f"start dnode cnt={cnt} wait={seconds}s") + selects = random.sample(dnodes, cnt) + for select in selects: + print(f" kill -9 {select.pid}") + cmd = f"kill -9 {select.pid}" + os.system(cmd) + print(f" restart {select.path}") + if runNoWait(select.path) == False: + print(f"run {select.path} failed.") + raise Exception("exe failed.") + print(f" sleep {seconds}s ...") + time.sleep(seconds) + +def run(): + # kill seconds interval + killLoop = 10 + minKill = 1 + maxKill = 10 + for i in range(killLoop): + dnodes = getDnodes() + killCnt = 0 + if len(dnodes) > 0: + killCnt = random.randint(1, len(dnodes)) + restartDnodes(dnodes, killCnt, random.randint(1, 5)) + + seconds = random.randint(minKill, maxKill) + print(f"----------- kill loop i={i} killCnt={killCnt} done. do sleep {seconds}s ... \n") + time.sleep(seconds) + + +if __name__ == '__main__': + run() \ No newline at end of file diff --git a/tests/system-test/empty.py b/tests/system-test/empty.py new file mode 100644 index 0000000000000000000000000000000000000000..fc44d0164f8059344ce09ea9cc31cdfce140d19f --- /dev/null +++ b/tests/system-test/empty.py @@ -0,0 +1,40 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + # init + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # run + def run(self): + # check two db query result same + tdLog.info(f"hello world.") + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/output.txt b/tests/system-test/output.txt new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index 96ffa63707494139a156b1d8b8a276bd7cf03a5b..0f644666cb4b27a4e808213a676a42deb012d1df 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -1,4 +1,125 @@ +python3 ./test.py -f 2-query/nestedQuery.py +python3 ./test.py -f 2-query/nestedQuery_str.py +python3 ./test.py -f 2-query/nestedQuery_math.py +python3 ./test.py -f 2-query/nestedQuery_time.py +python3 ./test.py -f 2-query/nestedQuery_26.py +python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 +python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 +python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 +python3 ./test.py -f 2-query/nestedQuery.py -Q 2 +python3 ./test.py -f 2-query/nestedQuery_26.py -Q 2 +python3 ./test.py -f 2-query/columnLenUpdated.py +python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2 +python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3 +python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 +python3 ./test.py -f 2-query/nestedQuery.py -Q 4 +python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 +python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 +python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 +python3 ./test.py -f 2-query/nestedQuery_26.py -Q 4 +python3 ./test.py -f 7-tmq/tmqShow.py +python3 ./test.py -f 7-tmq/tmqDropStb.py +python3 ./test.py -f 7-tmq/subscribeStb0.py +python3 ./test.py -f 7-tmq/subscribeStb1.py +python3 ./test.py -f 7-tmq/subscribeStb2.py +python3 ./test.py -f 7-tmq/subscribeStb3.py +python3 ./test.py -f 7-tmq/subscribeDb0.py -N 3 -n 3 +python3 ./test.py -f 7-tmq/ins_topics_test.py +python3 ./test.py -f 7-tmq/tmqMaxTopic.py +python3 ./test.py -f 7-tmq/tmqParamsTest.py +python3 ./test.py -f 7-tmq/tmqClientConsLog.py +python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py +python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py +python3 ./test.py -f 7-tmq/tmqOffset.py +python3 ./test.py -f 7-tmq/tmqDropConsumer.py +python3 ./test.py -f 1-insert/delete_stable.py +python3 ./test.py -f 2-query/out_of_order.py -Q 3 +python3 ./test.py -f 2-query/out_of_order.py +python3 ./test.py -f 2-query/insert_null_none.py +python3 ./test.py -f 2-query/insert_null_none.py -R +python3 ./test.py -f 2-query/insert_null_none.py -Q 2 +python3 ./test.py -f 2-query/insert_null_none.py -Q 3 +python3 ./test.py -f 2-query/insert_null_none.py -Q 4 python3 ./test.py -f 1-insert/database_pre_suf.py +python3 ./test.py -f 2-query/concat.py -Q 3 +python3 ./test.py -f 2-query/out_of_order.py -Q 2 +python3 ./test.py -f 2-query/out_of_order.py -Q 4 +python3 ./test.py -f 2-query/nestedQuery.py -Q 3 +python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 +python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 +python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +python3 ./test.py -f 2-query/nestedQuery_26.py -Q 3 +python3 ./test.py -f 2-query/select_null.py +python3 ./test.py -f 2-query/select_null.py -R +python3 ./test.py -f 2-query/select_null.py -Q 2 +python3 ./test.py -f 2-query/select_null.py -Q 3 +python3 ./test.py -f 2-query/select_null.py -Q 4 +python3 ./test.py -f 2-query/slimit.py +python3 ./test.py -f 2-query/slimit.py -R +python3 ./test.py -f 2-query/slimit.py -Q 2 +python3 ./test.py -f 2-query/slimit.py -Q 3 +python3 ./test.py -f 2-query/slimit.py -Q 4 +python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3 +python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3 +python3 ./test.py -f 3-enterprise/restore/restoreMnode.py -N 5 -M 3 +python3 ./test.py -f 3-enterprise/restore/restoreQnode.py -N 5 -M 3 +python3 ./test.py -f 7-tmq/create_wrong_topic.py +python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 +python3 ./test.py -f 7-tmq/basic5.py +python3 ./test.py -f 7-tmq/subscribeDb.py -N 3 -n 3 +python3 ./test.py -f 7-tmq/subscribeDb1.py +python3 ./test.py -f 7-tmq/subscribeDb2.py +python3 ./test.py -f 7-tmq/subscribeDb3.py +python3 ./test.py -f 7-tmq/subscribeDb4.py +python3 ./test.py -f 7-tmq/subscribeStb.py +python3 ./test.py -f 7-tmq/subscribeStb4.py +python3 ./test.py -f 7-tmq/db.py +python3 ./test.py -f 7-tmq/tmqError.py +python3 ./test.py -f 7-tmq/schema.py +python3 ./test.py -f 7-tmq/stbFilterWhere.py +python3 ./test.py -f 7-tmq/stbFilter.py +python3 ./test.py -f 7-tmq/tmqCheckData.py +python3 ./test.py -f 7-tmq/tmqCheckData1.py +python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +python3 ./test.py -f 7-tmq/tmqAlterSchema.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -N 3 -n 3 +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -N 3 -n 3 +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py +python3 ./test.py -f 7-tmq/tmqDnodeRestart.py +python3 ./test.py -f 7-tmq/tmqDnodeRestart1.py +python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py +python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3 +python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py +python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py +python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py +python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3 +python3 ./test.py -f 7-tmq/tmqDropStbCtb.py +python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py +python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py +python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py +python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py +python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py +python3 ./test.py -f 7-tmq/tmq_taosx.py +python3 ./test.py -f 7-tmq/raw_block_interface_test.py +python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py +python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 +python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 +python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 +python3 ./test.py -f 99-TDcase/TD-19201.py +python3 ./test.py -f 99-TDcase/TD-21561.py +python3 ./test.py -f 99-TDcase/TS-3404.py +python3 ./test.py -f 99-TDcase/TS-3581.py +python3 ./test.py -f 99-TDcase/TS-3311.py python3 ./test.py -f 0-others/balance_vgroups_r1.py -N 6 python3 ./test.py -f 0-others/taosShell.py python3 ./test.py -f 0-others/taosShellError.py @@ -11,10 +132,15 @@ python3 ./test.py -f 0-others/cachemodel.py python3 ./test.py -f 0-others/sysinfo.py python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/user_manage.py +python3 ./test.py -f 0-others/user_privilege.py python3 ./test.py -f 0-others/fsync.py -#python3 ./test.py -f 0-others/compatibility.py +python3 ./test.py -f 0-others/multilevel.py +python3 ./test.py -f 0-others/compatibility.py python3 ./test.py -f 0-others/tag_index_basic.py -# python3 ./test.py -f 1-insert/alter_database.py +python3 ./test.py -N 3 -f 0-others/walRetention.py +python3 ./test.py -f 0-others/timeRangeWise.py -N 3 +python3 ./test.py -f 1-insert/alter_database.py +python3 ./test.py -f 1-insert/alter_replica.py -N 3 python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py @@ -25,16 +151,43 @@ python3 ./test.py -f 1-insert/alter_table.py python3 ./test.py -f 1-insert/boundary.py python3 ./test.py -f 1-insert/insertWithMoreVgroup.py python3 ./test.py -f 1-insert/table_comment.py -#python3 ./test.py -f 1-insert/time_range_wise.py -#python3 ./test.py -f 1-insert/block_wise.py -#python3 ./test.py -f 1-insert/create_retentions.py python3 ./test.py -f 1-insert/mutil_stage.py python3 ./test.py -f 1-insert/table_param_ttl.py python3 ./test.py -f 1-insert/table_param_ttl.py -R python3 ./test.py -f 1-insert/update_data_muti_rows.py python3 ./test.py -f 1-insert/db_tb_name_check.py python3 ./test.py -f 1-insert/InsertFuturets.py +python3 ./test.py -f 1-insert/insert_wide_column.py +python3 ./test.py -f 1-insert/rowlength64k_benchmark.py +python3 ./test.py -f 1-insert/rowlength64k.py +python3 ./test.py -f 1-insert/rowlength64k.py -R +python3 ./test.py -f 1-insert/rowlength64k.py -Q 2 +python3 ./test.py -f 1-insert/rowlength64k.py -Q 3 +python3 ./test.py -f 1-insert/rowlength64k.py -Q 4 +python3 ./test.py -f 1-insert/rowlength64k_1.py +python3 ./test.py -f 1-insert/rowlength64k_1.py -R +python3 ./test.py -f 1-insert/rowlength64k_1.py -Q 2 +python3 ./test.py -f 1-insert/rowlength64k_1.py -Q 3 +python3 ./test.py -f 1-insert/rowlength64k_1.py -Q 4 +python3 ./test.py -f 1-insert/rowlength64k_2.py +python3 ./test.py -f 1-insert/rowlength64k_2.py -R +python3 ./test.py -f 1-insert/rowlength64k_2.py -Q 2 +python3 ./test.py -f 1-insert/rowlength64k_2.py -Q 3 +python3 ./test.py -f 1-insert/rowlength64k_2.py -Q 4 +python3 ./test.py -f 1-insert/rowlength64k_3.py +python3 ./test.py -f 1-insert/rowlength64k_3.py -R +python3 ./test.py -f 1-insert/rowlength64k_3.py -Q 2 +python3 ./test.py -f 1-insert/rowlength64k_3.py -Q 3 +python3 ./test.py -f 1-insert/rowlength64k_3.py -Q 4 +python3 ./test.py -f 1-insert/rowlength64k_4.py +python3 ./test.py -f 1-insert/rowlength64k_4.py -R +python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 2 +python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 3 +python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 4 +python3 ./test.py -f 1-insert/precisionUS.py +python3 ./test.py -f 1-insert/precisionNS.py python3 ./test.py -f 0-others/show.py +python3 ./test.py -f 0-others/show_tag_index.py python3 ./test.py -f 0-others/information_schema.py python3 ./test.py -f 2-query/abs.py python3 ./test.py -f 2-query/abs.py -R @@ -138,6 +291,7 @@ python3 ./test.py -f 2-query/mavg.py -R python3 ./test.py -f 2-query/max_partition.py python3 ./test.py -f 2-query/max_partition.py -R python3 ./test.py -f 2-query/max_min_last_interval.py +python3 ./test.py -f 2-query/last_row_interval.py python3 ./test.py -f 2-query/max.py python3 ./test.py -f 2-query/max.py -R python3 ./test.py -f 2-query/min.py @@ -146,6 +300,7 @@ python3 ./test.py -f 2-query/mode.py python3 ./test.py -f 2-query/mode.py -R python3 ./test.py -f 2-query/Now.py python3 ./test.py -f 2-query/Now.py -R +python3 ./test.py -f 2-query/orderBy.py -N 5 python3 ./test.py -f 2-query/percentile.py python3 ./test.py -f 2-query/percentile.py -R python3 ./test.py -f 2-query/pow.py @@ -160,8 +315,11 @@ python3 ./test.py -f 2-query/sample.py python3 ./test.py -f 2-query/sample.py -R python3 ./test.py -f 2-query/sin.py python3 ./test.py -f 2-query/sin.py -R +python3 ./test.py -f 2-query/smaBasic.py -N 3 python3 ./test.py -f 2-query/smaTest.py python3 ./test.py -f 2-query/smaTest.py -R +python3 ./test.py -f 0-others/sma_index.py +python3 ./test.py -f 2-query/sml_TS-3724.py python3 ./test.py -f 2-query/sml.py python3 ./test.py -f 2-query/sml.py -R python3 ./test.py -f 2-query/spread.py @@ -212,11 +370,13 @@ python3 ./test.py -f 2-query/case_when.py python3 ./test.py -f 2-query/case_when.py -R python3 ./test.py -f 2-query/blockSMA.py python3 ./test.py -f 2-query/blockSMA.py -R +python3 ./test.py -f 2-query/projectionDesc.py +python3 ./test.py -f 2-query/projectionDesc.py -R python3 ./test.py -f 1-insert/update_data.py python3 ./test.py -f 1-insert/tb_100w_data_order.py -python3 ./test.py -f 1-insert/delete_stable.py python3 ./test.py -f 1-insert/delete_childtable.py python3 ./test.py -f 1-insert/delete_normaltable.py +python3 ./test.py -f 1-insert/delete_systable.py python3 ./test.py -f 1-insert/keep_expired.py python3 ./test.py -f 1-insert/stmt_error.py python3 ./test.py -f 1-insert/drop.py @@ -225,17 +385,16 @@ python3 ./test.py -f 2-query/join2.py python3 ./test.py -f 2-query/union1.py python3 ./test.py -f 2-query/concat2.py python3 ./test.py -f 2-query/json_tag.py -python3 ./test.py -f 2-query/nestedQuery.py python3 ./test.py -f 2-query/nestedQueryInterval.py -python3 ./test.py -f 2-query/nestedQuery_str.py -python3 ./test.py -f 2-query/nestedQuery_math.py -python3 ./test.py -f 2-query/nestedQuery_time.py +python3 ./test.py -f 2-query/systable_func.py python3 ./test.py -f 2-query/stablity.py python3 ./test.py -f 2-query/stablity_1.py python3 ./test.py -f 2-query/elapsed.py python3 ./test.py -f 2-query/csum.py python3 ./test.py -f 2-query/function_diff.py python3 ./test.py -f 2-query/tagFilter.py +python3 ./test.py -f 2-query/projectionDesc.py +python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3 python3 ./test.py -f 2-query/queryQnode.py python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 @@ -247,7 +406,7 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -#python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -N 6 -M 3 @@ -260,9 +419,11 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 - python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 -n 3 python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py -N 6 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 6 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1 @@ -271,65 +432,6 @@ python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_que python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -N 4 -M 1 python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 -python3 ./test.py -f 7-tmq/create_wrong_topic.py -python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 -python3 ./test.py -f 7-tmq/basic5.py -python3 ./test.py -f 7-tmq/subscribeDb.py -N 3 -n 3 -python3 ./test.py -f 7-tmq/subscribeDb0.py -N 3 -n 3 -python3 ./test.py -f 7-tmq/subscribeDb1.py -python3 ./test.py -f 7-tmq/subscribeDb2.py -python3 ./test.py -f 7-tmq/subscribeDb3.py -python3 ./test.py -f 7-tmq/subscribeDb4.py -python3 ./test.py -f 7-tmq/subscribeStb.py -python3 ./test.py -f 7-tmq/subscribeStb0.py -python3 ./test.py -f 7-tmq/subscribeStb1.py -python3 ./test.py -f 7-tmq/subscribeStb2.py -python3 ./test.py -f 7-tmq/subscribeStb3.py -python3 ./test.py -f 7-tmq/subscribeStb4.py -python3 ./test.py -f 7-tmq/db.py -python3 ./test.py -f 7-tmq/tmqError.py -python3 ./test.py -f 7-tmq/schema.py -python3 ./test.py -f 7-tmq/stbFilter.py -python3 ./test.py -f 7-tmq/tmqCheckData.py -python3 ./test.py -f 7-tmq/tmqCheckData1.py -python3 ./test.py -f 7-tmq/tmqConsumerGroup.py -python3 ./test.py -f 7-tmq/tmqShow.py -python3 ./test.py -f 7-tmq/tmqAlterSchema.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -N 3 -n 3 -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -N 3 -n 3 -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py -python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py -python3 ./test.py -f 7-tmq/tmqDnodeRestart.py -python3 ./test.py -f 7-tmq/tmqDnodeRestart1.py -python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py -python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3 -python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py -python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py -python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py -python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3 -python3 ./test.py -f 7-tmq/tmqDropStb.py -python3 ./test.py -f 7-tmq/tmqDropStbCtb.py -python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py -python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py -python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py -python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py -python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -python3 ./test.py -f 7-tmq/tmq_taosx.py -python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py -python3 ./test.py -f 99-TDcase/TD-19201.py -python3 ./test.py -f 99-TDcase/TD-21561.py -python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 -python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 python3 ./test.py -f 2-query/between.py -Q 2 python3 ./test.py -f 2-query/distinct.py -Q 2 python3 ./test.py -f 2-query/varchar.py -Q 2 @@ -389,11 +491,7 @@ python3 ./test.py -f 2-query/arccos.py -Q 2 python3 ./test.py -f 2-query/arctan.py -Q 2 python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 python3 ./test.py -f 2-query/interp.py -Q 2 -python3 ./test.py -f 2-query/nestedQuery.py -Q 2 python3 ./test.py -f 2-query/nestedQueryInterval.py -Q 2 -python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 -python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 -python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 python3 ./test.py -f 2-query/stablity.py -Q 2 python3 ./test.py -f 2-query/stablity_1.py -Q 2 python3 ./test.py -f 2-query/avg.py -Q 2 @@ -422,11 +520,13 @@ python3 ./test.py -f 2-query/function_null.py -Q 2 python3 ./test.py -f 2-query/count_partition.py -Q 2 python3 ./test.py -f 2-query/max_partition.py -Q 2 python3 ./test.py -f 2-query/max_min_last_interval.py -Q 2 +python3 ./test.py -f 2-query/last_row_interval.py -Q 2 python3 ./test.py -f 2-query/last_row.py -Q 2 python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 python3 ./test.py -f 2-query/sml.py -Q 2 python3 ./test.py -f 2-query/case_when.py -Q 2 python3 ./test.py -f 2-query/blockSMA.py -Q 2 +python3 ./test.py -f 2-query/projectionDesc.py -Q 2 python3 ./test.py -f 99-TDcase/TD-21561.py -Q 2 python3 ./test.py -f 2-query/between.py -Q 3 python3 ./test.py -f 2-query/distinct.py -Q 3 @@ -443,7 +543,6 @@ python3 ./test.py -f 2-query/cast.py -Q 3 python3 ./test.py -f 2-query/substr.py -Q 3 python3 ./test.py -f 2-query/union.py -Q 3 python3 ./test.py -f 2-query/union1.py -Q 3 -python3 ./test.py -f 2-query/concat.py -Q 3 python3 ./test.py -f 2-query/concat2.py -Q 3 python3 ./test.py -f 2-query/concat_ws.py -Q 3 python3 ./test.py -f 2-query/concat_ws2.py -Q 3 @@ -515,12 +614,14 @@ python3 ./test.py -f 2-query/function_null.py -Q 3 python3 ./test.py -f 2-query/count_partition.py -Q 3 python3 ./test.py -f 2-query/max_partition.py -Q 3 python3 ./test.py -f 2-query/max_min_last_interval.py -Q 3 +python3 ./test.py -f 2-query/last_row_interval.py -Q 3 python3 ./test.py -f 2-query/last_row.py -Q 3 python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 python3 ./test.py -f 2-query/sml.py -Q 3 python3 ./test.py -f 2-query/interp.py -Q 3 python3 ./test.py -f 2-query/case_when.py -Q 3 python3 ./test.py -f 2-query/blockSMA.py -Q 3 +python3 ./test.py -f 2-query/projectionDesc.py -Q 3 python3 ./test.py -f 99-TDcase/TD-21561.py -Q 3 python3 ./test.py -f 2-query/between.py -Q 4 python3 ./test.py -f 2-query/distinct.py -Q 4 @@ -560,6 +661,7 @@ python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 python3 ./test.py -f 2-query/timetruncate.py -Q 4 python3 ./test.py -f 2-query/diff.py -Q 4 python3 ./test.py -f 2-query/Timediff.py -Q 4 +python3 ./test.py -f 2-query/json_tag.py -Q 4 python3 ./test.py -f 2-query/top.py -Q 4 python3 ./test.py -f 2-query/bottom.py -Q 4 python3 ./test.py -f 2-query/percentile.py -Q 4 @@ -578,11 +680,9 @@ python3 ./test.py -f 2-query/arcsin.py -Q 4 python3 ./test.py -f 2-query/arccos.py -Q 4 python3 ./test.py -f 2-query/arctan.py -Q 4 python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 -python3 ./test.py -f 2-query/nestedQuery.py -Q 4 python3 ./test.py -f 2-query/nestedQueryInterval.py -Q 4 -python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 -python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 -python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 +python3 ./test.py -f 2-query/stablity.py -Q 4 +python3 ./test.py -f 2-query/stablity_1.py -Q 4 python3 ./test.py -f 2-query/avg.py -Q 4 python3 ./test.py -f 2-query/elapsed.py -Q 4 python3 ./test.py -f 2-query/csum.py -Q 4 @@ -607,6 +707,7 @@ python3 ./test.py -f 2-query/function_null.py -Q 4 python3 ./test.py -f 2-query/count_partition.py -Q 4 python3 ./test.py -f 2-query/max_partition.py -Q 4 python3 ./test.py -f 2-query/max_min_last_interval.py -Q 4 +python3 ./test.py -f 2-query/last_row_interval.py -Q 4 python3 ./test.py -f 2-query/last_row.py -Q 4 python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 python3 ./test.py -f 2-query/sml.py -Q 4 @@ -617,22 +718,11 @@ python3 ./test.py -f 2-query/insert_select.py -R python3 ./test.py -f 2-query/insert_select.py -Q 2 python3 ./test.py -f 2-query/insert_select.py -Q 3 python3 ./test.py -f 2-query/insert_select.py -Q 4 -python3 ./test.py -f 2-query/insert_null_none.py -python3 ./test.py -f 2-query/insert_null_none.py -R -python3 ./test.py -f 2-query/insert_null_none.py -Q 2 -python3 ./test.py -f 2-query/insert_null_none.py -Q 3 -python3 ./test.py -f 2-query/insert_null_none.py -Q 4 -python3 ./test.py -f 2-query/out_of_order.py python3 ./test.py -f 2-query/out_of_order.py -R -python3 ./test.py -f 2-query/out_of_order.py -Q 2 -python3 ./test.py -f 2-query/out_of_order.py -Q 3 -python3 ./test.py -f 2-query/out_of_order.py -Q 4 -python3 ./test.py -f 2-query/max_min_data.py -python3 ./test.py -f 2-query/max_min_data.py -R -python3 ./test.py -f 2-query/max_min_data.py -Q 2 -python3 ./test.py -f 2-query/max_min_data.py -Q 3 -python3 ./test.py -f 2-query/max_min_data.py -Q 4 python3 ./test.py -f 2-query/blockSMA.py -Q 4 +python3 ./test.py -f 2-query/projectionDesc.py -Q 4 python3 ./test.py -f 2-query/odbc.py python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 python3 ./test.py -f 99-TDcase/TD-20582.py +python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 +python3 ./test.py -f 5-taos-tools/taosbenchmark/stt.py -N 3 diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 1461a7b373e3395ed3f4afc111a2df8ca0e5ed60..ea8b5e61693003cb8d850d5a9f8233a6c41bf6a3 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -3,8 +3,6 @@ IF (TD_WEBSOCKET) SET(websocket_lib_file "libtaosws.so") ELSEIF (TD_DARWIN) SET(websocket_lib_file "libtaosws.dylib") - ELSEIF (TD_WINDOWS) - SET(websocket_lib_file "{taosws.dll,taosws.dll.lib}") ENDIF () MESSAGE("${Green} use libtaos-ws${ColourReset}") IF (TD_ALPINE) @@ -26,6 +24,26 @@ IF (TD_WEBSOCKET) COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include COMMAND cmake -E copy target/release/taosws.h ${CMAKE_BINARY_DIR}/build/include ) + ELSEIF (TD_WINDOWS) + include(ExternalProject) + ExternalProject_Add(taosws-rs + PREFIX "taosws-rs" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs + BUILD_ALWAYS off + DEPENDS taos + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" + PATCH_COMMAND + COMMAND git clean -f -d + BUILD_COMMAND + COMMAND cargo update + COMMAND cargo build --release -p taos-ws-sys --features native-tls-vendored + INSTALL_COMMAND + COMMAND cp target/release/taosws.dll ${CMAKE_BINARY_DIR}/build/lib + COMMAND cp target/release/taosws.dll.lib ${CMAKE_BINARY_DIR}/build/lib/taosws.lib + COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include + COMMAND cmake -E copy target/release/taosws.h ${CMAKE_BINARY_DIR}/build/include + ) ELSE() include(ExternalProject) ExternalProject_Add(taosws-rs @@ -121,22 +139,48 @@ ELSE () BUILD_COMMAND COMMAND set CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client COMMAND set CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib - COMMAND go build -a -o taosadapter.exe -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" -# COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" -# COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + # COMMAND go build -a -o taosadapter.exe -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" INSTALL_COMMAND -# COMMAND cmake -E echo "Comparessing taosadapter.exe" -# COMMAND cmake -E time upx taosadapter.exe + COMMAND cmake -E echo "Comparessing taosadapter.exe" + COMMAND cmake -E time upx taosadapter.exe COMMAND cmake -E echo "Copy taosadapter.exe" COMMAND cmake -E copy taosadapter.exe ${CMAKE_BINARY_DIR}/build/bin/taosadapter.exe COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E echo "Copy taosadapter.toml" COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ -# COMMAND cmake -E echo "Copy taosadapter-debug.exe" -# COMMAND cmake -E copy taosadapter-debug.exe ${CMAKE_BINARY_DIR}/build/bin + COMMAND cmake -E echo "Copy taosadapter-debug.exe" + COMMAND cmake -E copy taosadapter-debug.exe ${CMAKE_BINARY_DIR}/build/bin ) - ELSE (TD_WINDOWS) + ELSEIF (TD_DARWIN) + MESSAGE("Building taosAdapter on MACOS") + INCLUDE(ExternalProject) + ExternalProject_Add(taosadapter + PREFIX "taosadapter" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosadapter + BUILD_ALWAYS off + DEPENDS taos + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND cmake -E echo "taosadapter no need cmake to config" + PATCH_COMMAND + COMMAND git clean -f -d + BUILD_COMMAND + # COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + INSTALL_COMMAND + COMMAND cmake -E echo "Copy taosadapter" + COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin + COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ + COMMAND cmake -E echo "Copy taosadapter.toml" + COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ + COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/ + COMMAND cmake -E echo "Copy taosadapter-debug" + COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin + ) + ELSE () MESSAGE("Building taosAdapter on non-Windows") INCLUDE(ExternalProject) ExternalProject_Add(taosadapter @@ -149,20 +193,20 @@ ELSE () PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" -# COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" -# COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + # COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/v3/version.Version=${taos_version} -X github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}" INSTALL_COMMAND -# COMMAND cmake -E echo "Comparessing taosadapter.exe" -# COMMAND upx taosadapter || : + COMMAND cmake -E echo "Comparessing taosadapter.exe" + COMMAND upx taosadapter || : COMMAND cmake -E echo "Copy taosadapter" COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E echo "Copy taosadapter.toml" COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/ -# COMMAND cmake -E echo "Copy taosadapter-debug" -# COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin + COMMAND cmake -E echo "Copy taosadapter-debug" + COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin ) - ENDIF (TD_WINDOWS) + ENDIF () ENDIF () diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 7b1dc3a541416335dca761aba8634679473ecdd2..0ce181808fb8e98dc047f3bf34747d09217ea768 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -20,7 +20,7 @@ ELSEIF (TD_DARWIN AND TD_WEBSOCKET) ADD_DEPENDENCIES(shell taosws-rs) ELSEIF (TD_WINDOWS AND TD_WEBSOCKET) ADD_DEFINITIONS(-DWEBSOCKET -I${CMAKE_BINARY_DIR}/build/include) - SET(LINK_WEBSOCKET "${CMAKE_BINARY_DIR}/build/lib/taosws.dll.lib") + SET(LINK_WEBSOCKET "${CMAKE_BINARY_DIR}/build/lib/taosws.lib") ADD_DEPENDENCIES(shell taosws-rs) ELSE () SET(LINK_WEBSOCKET "") @@ -40,7 +40,7 @@ endif () target_link_libraries( shell - PRIVATE os common transport util + PRIVATE os common transport geometry util ) target_include_directories( diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index 6345647e2f24b1a87cee8a6a0c60275907946dfe..57415f833578ae9ab003fa829d70716985af1d9a 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -45,6 +45,8 @@ #define SHELL_MAX_PKG_NUM 1 * 1024 * 1024 #define SHELL_MIN_PKG_NUM 1 #define SHELL_DEF_PKG_NUM 100 +#define SHELL_FLOAT_WIDTH 20 +#define SHELL_DOUBLE_WIDTH 25 typedef struct { char* hist[SHELL_MAX_HISTORY_SIZE]; diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 140720af815cf317bf7665c40750b2d0fb2ebd37..41cdb0f9286b84502da146937123c493dfb90331 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -71,7 +71,6 @@ SWords shellCommands[] = { {"alter all dnodes \"monitor\" \"0\";", 0, 0, NULL}, {"alter all dnodes \"monitor\" \"1\";", 0, 0, NULL}, {"alter table ;", 0, 0, NULL}, - {"alter table modify column", 0, 0, NULL}, {"alter local \"resetlog\";", 0, 0, NULL}, {"alter local \"DebugFlag\" \"143\";", 0, 0, NULL}, {"alter local \"cDebugFlag\" \"143\";", 0, 0, NULL}, @@ -92,9 +91,14 @@ SWords shellCommands[] = { {"create stream into as select", 0, 0, NULL}, // 26 append sub sql {"create topic as select", 0, 0, NULL}, // 27 append sub sql {"create function as outputtype language ", 0, 0, NULL}, + {"create or replace as outputtype language ", 0, 0, NULL}, {"create aggregate function as outputtype bufsize language ", 0, 0, NULL}, + {"create or replace aggregate function as outputtype bufsize language ", 0, 0, NULL}, {"create user pass sysinfo 0;", 0, 0, NULL}, {"create user pass sysinfo 1;", 0, 0, NULL}, +#ifdef TD_ENTERPRISE + {"compact database ", 0, 0, NULL}, +#endif {"describe ", 0, 0, NULL}, {"delete from where ", 0, 0, NULL}, {"drop database ", 0, 0, NULL}, @@ -118,8 +122,16 @@ SWords shellCommands[] = { {"kill connection ;", 0, 0, NULL}, {"kill query ", 0, 0, NULL}, {"kill transaction ", 0, 0, NULL}, +#ifdef TD_ENTERPRISE {"merge vgroup ", 0, 0, NULL}, +#endif + {"pause stream ;", 0, 0, NULL}, + {"resume stream ;", 0, 0, NULL}, {"reset query cache;", 0, 0, NULL}, + {"restore dnode ;", 0, 0, NULL}, + {"restore vnode on dnode ;", 0, 0, NULL}, + {"restore mnode on dnode ;", 0, 0, NULL}, + {"restore qnode on dnode ;", 0, 0, NULL}, {"revoke all on from ;", 0, 0, NULL}, {"revoke read on from ;", 0, 0, NULL}, {"revoke write on from ;", 0, 0, NULL}, @@ -170,7 +182,9 @@ SWords shellCommands[] = { {"show vgroups;", 0, 0, NULL}, {"show consumers;", 0, 0, NULL}, {"show grants;", 0, 0, NULL}, +#ifdef TD_ENTERPRISE {"split vgroup ", 0, 0, NULL}, +#endif {"insert into values(", 0, 0, NULL}, {"insert into using tags(", 0, 0, NULL}, {"insert into using values(", 0, 0, NULL}, @@ -326,7 +340,7 @@ TdThreadMutex tiresMutex; TdThread* threads[WT_FROM_DB_CNT]; // obtain var name with sql from server char varTypes[WT_VAR_CNT][64] = { - "", "", "", "", "", "", "", + "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""}; @@ -345,12 +359,12 @@ int cntDel = 0; // delete byte count after next press tab // show auto tab introduction void printfIntroduction() { - printf(" ****************************** Tab Completion **********************************\n"); + printf(" ******************************** Tab Completion ************************************\n"); char secondLine[160] = "\0"; - sprintf(secondLine, " * The %s CLI supports tab completion for a variety of items, ", shell.info.cusName); + sprintf(secondLine, " * The %s CLI supports tab completion for a variety of items, ", shell.info.cusName); printf("%s", secondLine); int secondLineLen = strlen(secondLine); - while (84 - (secondLineLen++) > 0) { + while (87 - (secondLineLen++) > 0) { printf(" "); } printf("*\n"); @@ -382,7 +396,6 @@ void showHelp() { alter all dnodes \"resetlog\";\n\ alter all dnodes \"debugFlag\" \n\ alter table ;\n\ - alter table modify column\n\ alter local \"resetlog\";\n\ alter local \"DebugFlag\" \"143\";\n\ alter topic\n\ @@ -430,10 +443,15 @@ void showHelp() { kill connection ; \n\ kill query ; \n\ kill transaction ;\n\ - ----- M ----- \n\ - merge vgroup ...\n\ + ----- P ----- \n\ + pause stream ;\n\ ----- R ----- \n\ + resume stream ;\n\ reset query cache;\n\ + restore dnode ;\n\ + restore vnode on dnode ;\n\ + restore mnode on dnode ;\n\ + restore qnode on dnode ;\n\ revoke all on from ;\n\ revoke read on from ;\n\ revoke write on from ;\n\ @@ -483,14 +501,20 @@ void showHelp() { show vgroups;\n\ show consumers;\n\ show grants;\n\ - split vgroup ...\n\ ----- T ----- \n\ trim database ;\n\ ----- U ----- \n\ use ;"); - printf("\n\n"); +#ifdef TD_ENTERPRISE + printf( + "\n\n\ + ----- special commands on enterpise version ----- \n\ + compact database ; \n\ + split vgroup ;"); +#endif + printf("\n\n"); // define in getDuration() function printf( "\ diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 0e305f57e9c781a8149db1fecccd06a703bc1465..8c91ff53e2770fec8118ab9eee7e0121720a2c42 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -501,7 +501,7 @@ int32_t shellReadCommand(char *command) { while (1) { c = taosGetConsoleChar(); - if (c == EOF) { + if (c == (char)EOF) { return c; } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index e2752f7899cbd61fc03bc3338c4125d3f15f6391..e9dd067ac49b1dbc3c90b55fc4e75d12faa33ae6 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -20,6 +20,7 @@ #define _DEFAULT_SOURCE #include "shellInt.h" #include "shellAuto.h" +#include "geosWrapper.h" static bool shellIsEmptyCommand(const char *cmd); static int32_t shellRunSingleCommand(char *command); @@ -29,6 +30,7 @@ static void shellRunSingleCommandImp(char *command); static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision); static int32_t shellDumpResultToFile(const char *fname, TAOS_RES *tres); static void shellPrintNChar(const char *str, int32_t length, int32_t width); +static void shellPrintGeometry(const unsigned char *str, int32_t length, int32_t width); static int32_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql); static int32_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql); static int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql); @@ -306,6 +308,15 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { return buf; } +char *shellDumpHexValue(char *buf, const char *val, int32_t length) { + for (int32_t i = 0; i < length; i++) { + sprintf(buf + (i * 2), "%02X", val[i]); + } + buf[length * 2] = 0; + + return buf; +} + void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int32_t precision) { if (val == NULL) { taosFprintfFile(pFile, "NULL"); @@ -315,6 +326,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i char quotationStr[2]; quotationStr[0] = '\"'; quotationStr[1] = 0; + int32_t width; int n; char buf[TSDB_MAX_BYTES_PER_ROW]; @@ -347,20 +359,27 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i taosFprintfFile(pFile, "%" PRIu64, *((uint64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: + width = SHELL_FLOAT_WIDTH; if (tsEnableScience) { - taosFprintfFile(pFile, "%e", GET_FLOAT_VAL(val)); + taosFprintfFile(pFile, "%*.7e", width, GET_FLOAT_VAL(val)); } else { - taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.7f", width, GET_FLOAT_VAL(val)); + if (n > SHELL_FLOAT_WIDTH) { + taosFprintfFile(pFile, "%*.7e", width, GET_FLOAT_VAL(val)); + } else { + taosFprintfFile(pFile, "%s", buf); + } } break; case TSDB_DATA_TYPE_DOUBLE: + width = SHELL_DOUBLE_WIDTH; if (tsEnableScience) { - snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9e", 23, GET_DOUBLE_VAL(val)); + snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", width, GET_DOUBLE_VAL(val)); taosFprintfFile(pFile, "%s", buf); } else { - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val)); - if (n > TMAX(25, length)) { - taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val)); + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15f", width, GET_DOUBLE_VAL(val)); + if (n > SHELL_DOUBLE_WIDTH) { + taosFprintfFile(pFile, "%*.15e", width, GET_DOUBLE_VAL(val)); } else { taosFprintfFile(pFile, "%s", buf); } @@ -384,6 +403,10 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr); } break; + case TSDB_DATA_TYPE_GEOMETRY: + shellDumpHexValue(buf, val, length); + taosFprintfFile(pFile, "%s", buf); + break; case TSDB_DATA_TYPE_TIMESTAMP: shellFormatTimestamp(buf, *(int64_t *)val, precision); taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr); @@ -507,16 +530,53 @@ void shellPrintNChar(const char *str, int32_t length, int32_t width) { } } -void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t length, int32_t precision) { - if (val == NULL) { - int32_t w = width; - if (field->type < TSDB_DATA_TYPE_TINYINT || field->type > TSDB_DATA_TYPE_DOUBLE) { - w = 0; +void shellPrintString(const char *str, int32_t width) { + int32_t len = strlen(str); + + if (width == 0) { + printf("%s", str); + } + else if (len > width) { + if (width <= 3) { + printf("%.*s.", width - 1, str); } - w = printf("%*s", w, TSDB_DATA_NULL_STR); - for (; w < width; w++) { - putchar(' '); + else { + printf("%.*s...", width - 3, str); } + } else { + printf("%s%*.s", str, width - len, ""); + } +} + +void shellPrintGeometry(const unsigned char *val, int32_t length, int32_t width) { + if (length == 0) { //empty value + shellPrintString("", width); + return; + } + + int32_t code = TSDB_CODE_FAILED; + + code = initCtxAsText(); + if (code != TSDB_CODE_SUCCESS) { + shellPrintString(getThreadLocalGeosCtx()->errMsg, width); + return; + } + + char *outputWKT = NULL; + code = doAsText(val, length, &outputWKT); + if (code != TSDB_CODE_SUCCESS) { + shellPrintString(getThreadLocalGeosCtx()->errMsg, width); //should NOT happen + return; + } + + shellPrintString(outputWKT, width); + + geosFreeBuffer(outputWKT); +} + +void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t length, int32_t precision) { + if (val == NULL) { + shellPrintString(TSDB_DATA_NULL_STR, width); return; } @@ -524,7 +584,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t char buf[TSDB_MAX_BYTES_PER_ROW]; switch (field->type) { case TSDB_DATA_TYPE_BOOL: - printf("%*s", width, ((((int32_t)(*((char *)val))) == 1) ? "true" : "false")); + shellPrintString(((((int32_t)(*((char *)val))) == 1) ? "true" : "false"), width); break; case TSDB_DATA_TYPE_TINYINT: printf("%*d", width, *((int8_t *)val)); @@ -552,11 +612,12 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t break; case TSDB_DATA_TYPE_FLOAT: if (tsEnableScience) { - printf("%*e", width, GET_FLOAT_VAL(val)); + printf("%*.7e",width,GET_FLOAT_VAL(val)); } else { - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.5f", width, GET_FLOAT_VAL(val)); - if (n > TMAX(20, width)) { - printf("%*e", width, GET_FLOAT_VAL(val)); + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.7f", width, GET_FLOAT_VAL(val)); + if (n > SHELL_FLOAT_WIDTH) { + + printf("%*.7e", width,GET_FLOAT_VAL(val)); } else { printf("%s", buf); } @@ -564,14 +625,14 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t break; case TSDB_DATA_TYPE_DOUBLE: if (tsEnableScience) { - snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%.9e", GET_DOUBLE_VAL(val)); - printf("%*s", width, buf); + snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", width,GET_DOUBLE_VAL(val)); + printf("%s", buf); } else { - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", width, GET_DOUBLE_VAL(val)); - if (n > TMAX(25, width)) { + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15f", width, GET_DOUBLE_VAL(val)); + if (n > SHELL_DOUBLE_WIDTH) { printf("%*.15e", width, GET_DOUBLE_VAL(val)); } else { - printf("%s", buf); + printf("%*s", width,buf); } } break; @@ -580,6 +641,9 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t case TSDB_DATA_TYPE_JSON: shellPrintNChar(val, length, width); break; + case TSDB_DATA_TYPE_GEOMETRY: + shellPrintGeometry(val, length, width); + break; case TSDB_DATA_TYPE_TIMESTAMP: shellFormatTimestamp(buf, *(int64_t *)val, precision); printf("%s", buf); @@ -702,12 +766,13 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { return TMAX(21, width); // '-9223372036854775807' case TSDB_DATA_TYPE_FLOAT: - return TMAX(20, width); + return TMAX(SHELL_FLOAT_WIDTH, width); case TSDB_DATA_TYPE_DOUBLE: - return TMAX(25, width); + return TMAX(SHELL_DOUBLE_WIDTH, width); case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_GEOMETRY: if (field->bytes > shell.args.displayWidth) { return TMAX(shell.args.displayWidth, width); } else { @@ -1089,6 +1154,7 @@ void *shellThreadLoop(void *arg) { taosResetTerminalMode(); } while (shellRunCommand(command, true) == 0); + destroyThreadLocalGeosCtx(); taosMemoryFreeClear(command); shellWriteHistory(); shellExit(); diff --git a/tools/shell/src/shellNettest.c b/tools/shell/src/shellNettest.c index 1a6ac3489dad818feac855625d5cc502875973e0..9fe92212cac5084a108c39440d9f1446a1fbf24b 100644 --- a/tools/shell/src/shellNettest.c +++ b/tools/shell/src/shellNettest.c @@ -15,6 +15,7 @@ #define _GNU_SOURCE #include "shellInt.h" +#include "tversion.h" static void shellWorkAsClient() { SShellArgs *pArgs = &shell.args; @@ -33,6 +34,7 @@ static void shellWorkAsClient() { rpcInit.user = "_dnd"; rpcInit.timeToGetConn = tsTimeToGetAvailableConn; + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); clientRpc = rpcOpen(&rpcInit); if (clientRpc == NULL) { printf("failed to init net test client since %s\r\n", terrstr()); @@ -123,6 +125,8 @@ static void shellWorkAsServer() { rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; + taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); + void *serverRpc = rpcOpen(&rpcInit); if (serverRpc == NULL) { printf("failed to init net test server since %s\r\n", terrstr()); diff --git a/tools/shell/src/shellUtil.c b/tools/shell/src/shellUtil.c index e15b49efcc35da2682d003243c0a19eb278acbc7..93451c85a9a34545a6aa86e3777d92d462f8849d 100644 --- a/tools/shell/src/shellUtil.c +++ b/tools/shell/src/shellUtil.c @@ -68,7 +68,7 @@ int32_t shellCheckIntSize() { return 0; } -void shellPrintVersion() { printf("version: %s\r\n", version); } +void shellPrintVersion() { printf("%s\r\n", shell.info.programVersion); } void shellGenerateAuth() { char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index d8920cb4c3d79aacb441bf9a1f512f4a5a4f62b6..af7f13c69c6088017f33cfabe3b81a17c9ab2587 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -17,6 +17,9 @@ #include #include +// save current database name +char curDBName[128] = ""; // TDB_MAX_DBNAME_LEN is 24, put large + int shell_conn_ws_server(bool first) { char cuttedDsn[SHELL_WS_DSN_BUFF] = {0}; int dsnLen = strlen(shell.args.dsn); @@ -59,6 +62,14 @@ int shell_conn_ws_server(bool first) { fprintf(stdout, "successfully connected to cloud service\n"); } fflush(stdout); + + // switch to current database if have + if(curDBName[0] !=0) { + char command[256]; + sprintf(command, "use %s;", curDBName); + shellRunSingleCommandWebsocketImp(command); + } + return 0; } @@ -290,7 +301,46 @@ void shellRunSingleCommandWebsocketImp(char *command) { if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { - fprintf(stdout, "Database changed.\r\n\r\n"); + + // copy dbname to curDBName + char *p = command; + bool firstStart = false; + bool firstEnd = false; + int i = 0; + while (*p != 0) { + if (*p != ' ') { + // not blank + if (!firstStart) { + firstStart = true; + } else if (firstEnd) { + if(*p == ';' && *p != '\\') { + break; + } + // database name + curDBName[i++] = *p; + if(i + 4 > sizeof(curDBName)) { + // DBName is too long, reset zero and break + i = 0; + break; + } + } + } else { + // blank + if(firstStart == true && firstEnd == false){ + firstEnd = true; + } + if(firstStart && firstEnd && i > 0){ + // blank after database name + break; + } + } + // move next + p++; + } + // append end + curDBName[i] = 0; + + fprintf(stdout, "Database changed to %s.\r\n\r\n", curDBName); fflush(stdout); ws_free_result(res); return; diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index 87b0d11d1c00f631c7c09081952af82174e2432f..3f52fc8e5d5557f8f9ba52296ebd1f6d1632e589 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -7,37 +7,45 @@ add_executable(write_raw_block_test write_raw_block_test.c) add_executable(sml_test sml_test.c) add_executable(get_db_name_test get_db_name_test.c) add_executable(tmq_offset tmqOffset.c) +add_executable(tmq_offset_test tmq_offset_test.c) target_link_libraries( tmq_offset - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os ) target_link_libraries( create_table - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os ) target_link_libraries( tmq_demo - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os ) target_link_libraries( tmq_sim - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os ) target_link_libraries( tmq_taosx_ci - PUBLIC taos_static + PUBLIC taos + PUBLIC util + PUBLIC common + PUBLIC os +) +target_link_libraries( + tmq_offset_test + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os @@ -45,7 +53,7 @@ target_link_libraries( target_link_libraries( write_raw_block_test - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os @@ -53,7 +61,7 @@ target_link_libraries( target_link_libraries( sml_test - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os @@ -61,7 +69,7 @@ target_link_libraries( target_link_libraries( get_db_name_test - PUBLIC taos_static + PUBLIC taos PUBLIC util PUBLIC common PUBLIC os diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index ac5aff47273fcda49af591f55987dffaae5a7bf7..e4ed6037a34d207ae32d4f0d62525c343413a82b 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1132,6 +1132,155 @@ int sml_td22900_Test() { return code; } +int sml_td24070_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "CREATE user test_db pass 'test'"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS td24070_read"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "grant read on td24070_read to test_db"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS td24070_write"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "grant write on td24070_write to test_db"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + taos_close(taos); + + + // test db privilege + taos = taos_connect("localhost", "test_db", "test", NULL, 0); + const char* sql[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299372000"}; + + pRes = taos_query(taos, "use td24070_read"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + ASSERT(code != 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "use td24070_write"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code == 0); + taos_free_result(pRes); + taos_close(taos); + // test db privilege end + + + // test stable privilege + taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + pRes = taos_query(taos, "CREATE user test_stb_read pass 'test'"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "CREATE user test_stb_write pass 'test'"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "grant read on td24070_write.stb2 to test_stb_read"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "grant write on td24070_write.stb2 to test_stb_write"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + taos_close(taos); + + taos = taos_connect("localhost", "test_stb_read", "test", "td24070_write", 0); + const char* sql1[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299373000"}; + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code != 0); + taos_free_result(pRes); + taos_close(taos); + + taos = taos_connect("localhost", "test_stb_write", "test", "td24070_write", 0); + const char* sql2[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299373000"}; + + pRes = taos_schemaless_insert(taos, (char **)sql2, sizeof(sql2) / sizeof(sql2[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code == 0); + taos_free_result(pRes); + taos_close(taos); + // test stable privilege + + // test table privilege + taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + pRes = taos_query(taos, "CREATE user test_tb_read pass 'test'"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "CREATE user test_tb_write pass 'test'"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "grant read on td24070_write.stb2 with t1=1 to test_tb_read"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "grant write on td24070_write.stb2 with t1=1 to test_tb_write"); + ASSERT(taos_errno(pRes) == 0); + taos_free_result(pRes); + taos_close(taos); + + taos = taos_connect("localhost", "test_tb_read", "test", "td24070_write", 0); + const char* sql3[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299374000"}; + + + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code != 0); + taos_free_result(pRes); + taos_close(taos); + + taos = taos_connect("localhost", "test_tb_write", "test", "td24070_write", 0); + const char* sql4[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299374000"}; + + pRes = taos_schemaless_insert(taos, (char **)sql4, sizeof(sql4) / sizeof(sql4[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code == 0); + taos_free_result(pRes); + taos_close(taos); + // test table privilege + + return code; +} + int sml_td23881_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1373,21 +1522,53 @@ int sml_ts2385_Test() { return code; } +int sml_ts3724_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "drop database if exists ts3724"); + taos_free_result(pRes); + + pRes = taos_query(taos, "create database if not exists ts3724"); + taos_free_result(pRes); + + const char *sql[] = { + "stb.2,t1=1 f1=283i32 1632299372000", + ".stb2,t1=1 f1=106i32 1632299378000", + "stb2.,t1=1 f1=106i32 1632299378000", + }; + + pRes = taos_query(taos, "use ts3724"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + int code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + taos_free_result(pRes); + + taos_close(taos); + + return code; +} + int main(int argc, char *argv[]) { if (argc == 2) { taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); } int ret = 0; + ret = sml_td24070_Test(); + ASSERT(!ret); ret = sml_td23881_Test(); ASSERT(ret); ret = sml_escape_Test(); ASSERT(!ret); ret = sml_ts3116_Test(); ASSERT(!ret); -// ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file -// ASSERT(!ret); - ret = sml_ts3303_Test(); // this test case need config sml table name using ./sml_test config_file + ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file + ASSERT(!ret); + ret = sml_ts3303_Test(); ASSERT(!ret); // for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ @@ -1428,5 +1609,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_19221_Test(); ASSERT(!ret); + ret = sml_ts3724_Test(); + ASSERT(!ret); + return ret; } diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index d98a45f0d3c3873f16ddc0134c58eeea2e6555dd..dd6875185f60de3fc13c4c1bbe05ea0d9e331372 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -542,7 +542,8 @@ static void shellDumpFieldToFile(TdFilePtr pFile, const char* val, TAOS_FIELD* f break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON: { + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_GEOMETRY: { int32_t bufIndex = 0; for (int32_t i = 0; i < length; i++) { buf[bufIndex] = val[i]; diff --git a/utils/test/c/tmq_offset_test.c b/utils/test/c/tmq_offset_test.c new file mode 100644 index 0000000000000000000000000000000000000000..6be9b38979edb32298ec71efbfa76d7b2dbcc720 --- /dev/null +++ b/utils/test/c/tmq_offset_test.c @@ -0,0 +1,311 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "taos.h" +#include "types.h" + +int buildData(TAOS* pConn){ + TAOS_RES* pRes = taos_query(pConn, "drop topic if exists tp"); + if (taos_errno(pRes) != 0) { + printf("error in drop tp, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists db_ts3756"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists db_ts3756 vgroups 2 wal_retention_period 3600"); + if (taos_errno(pRes) != 0) { + printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use db_ts3756"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn,"CREATE TABLE `t1` (`ts` TIMESTAMP, `voltage` INT)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table meters, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into t1 values(now, 1)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into t1 values(now + 1s, 2)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create topic tp as select * from t1"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic tp, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + return 0; +} + +void test_offset(TAOS* pConn){ + if(buildData(pConn) != 0){ + ASSERT(0); + } + tmq_conf_t* conf = tmq_conf_new(); + + tmq_conf_set(conf, "enable.auto.commit", "false"); + tmq_conf_set(conf, "auto.commit.interval.ms", "2000"); + tmq_conf_set(conf, "group.id", "group_id_2"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "msg.with.table.name", "false"); + + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + tmq_conf_destroy(conf); + + // 创建订阅 topics 列表 + tmq_list_t* topicList = tmq_list_new(); + tmq_list_append(topicList, "tp"); + + // 启动订阅 + tmq_subscribe(tmq, topicList); + tmq_list_destroy(topicList); + + int32_t timeout = 200; + + tmq_topic_assignment* pAssign1 = NULL; + int32_t numOfAssign1 = 0; + + tmq_topic_assignment* pAssign2 = NULL; + int32_t numOfAssign2 = 0; + + tmq_topic_assignment* pAssign3 = NULL; + int32_t numOfAssign3 = 0; + + int32_t code = tmq_get_topic_assignment(tmq, "tp", &pAssign1, &numOfAssign1); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_free_assignment(pAssign1); + tmq_consumer_close(tmq); + ASSERT(0); + } + + code = tmq_get_topic_assignment(tmq, "tp", &pAssign2, &numOfAssign2); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_free_assignment(pAssign2); + tmq_consumer_close(tmq); + ASSERT(0); + } + + code = tmq_get_topic_assignment(tmq, "tp", &pAssign3, &numOfAssign3); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_free_assignment(pAssign3); + tmq_consumer_close(tmq); + ASSERT(0); + return; + } + + ASSERT(numOfAssign1 == 2); + ASSERT(numOfAssign1 == numOfAssign2); + ASSERT(numOfAssign1 == numOfAssign3); + + for(int i = 0; i < numOfAssign1; i++){ + int j = 0; + int k = 0; + for(; j < numOfAssign2; j++){ + if(pAssign1[i].vgId == pAssign2[j].vgId){ + break; + } + } + for(; k < numOfAssign3; k++){ + if(pAssign1[i].vgId == pAssign3[k].vgId){ + break; + } + } + + ASSERT(pAssign1[i].currentOffset == pAssign2[j].currentOffset); + ASSERT(pAssign1[i].currentOffset == pAssign3[k].currentOffset); + + ASSERT(pAssign1[i].begin == pAssign2[j].begin); + ASSERT(pAssign1[i].begin == pAssign3[k].begin); + + ASSERT(pAssign1[i].end == pAssign2[j].end); + ASSERT(pAssign1[i].end == pAssign3[k].end); + } + tmq_free_assignment(pAssign1); + tmq_free_assignment(pAssign2); + tmq_free_assignment(pAssign3); + + int cnt = 0; + int offset1 = -1; + int offset2 = -1; + while (cnt++ < 10) { + printf("start to poll:%d\n", cnt); + TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout); + if (pRes) { + tmq_topic_assignment* pAssign = NULL; + int32_t numOfAssign = 0; + + code = tmq_get_topic_assignment(tmq, "tp", &pAssign, &numOfAssign); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_free_assignment(pAssign); + tmq_consumer_close(tmq); + ASSERT(0); + } + + for(int i = 0; i < numOfAssign; i++){ + int64_t position = tmq_position(tmq, "tp", pAssign[i].vgId); + if(position == 0) continue; + + printf("position = %d\n", (int)position); + tmq_commit_offset_sync(tmq, "tp", pAssign[i].vgId, position); + int64_t committed = tmq_committed(tmq, "tp", pAssign[i].vgId); + ASSERT(position == committed); + } + + tmq_offset_seek(tmq, "tp", pAssign[0].vgId, pAssign[0].currentOffset); + tmq_offset_seek(tmq, "tp", pAssign[1].vgId, pAssign[1].currentOffset); + + if(offset1 != -1){ + ASSERT(offset1 == pAssign[0].currentOffset); + } + if(offset2 != -1){ + ASSERT(offset2 == pAssign[1].currentOffset); + } + + offset1 = pAssign[0].currentOffset; + offset2 = pAssign[1].currentOffset; + + tmq_free_assignment(pAssign); + + taos_free_result(pRes); + } + } + + tmq_consumer_close(tmq); +} + +// run taosBenchmark first +void test_ts3756(TAOS* pConn){ + TAOS_RES*pRes = taos_query(pConn, "use test"); + if (taos_errno(pRes) != 0) { + ASSERT(0); + } + taos_free_result(pRes); + pRes = taos_query(pConn, "drop topic if exists t1"); + if (taos_errno(pRes) != 0) { + ASSERT(0); + } + taos_free_result(pRes); + pRes = taos_query(pConn, "create topic t1 as select * from meters"); + if (taos_errno(pRes) != 0) { + ASSERT(0); + } + taos_free_result(pRes); + tmq_conf_t* conf = tmq_conf_new(); + + tmq_conf_set(conf, "enable.auto.commit", "false"); + tmq_conf_set(conf, "auto.commit.interval.ms", "2000"); + tmq_conf_set(conf, "group.id", "group_id_2"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "auto.offset.reset", "latest"); + tmq_conf_set(conf, "msg.with.table.name", "false"); + + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + tmq_conf_destroy(conf); + + // 创建订阅 topics 列表 + tmq_list_t* topicList = tmq_list_new(); + tmq_list_append(topicList, "t1"); + + // 启动订阅 + tmq_subscribe(tmq, topicList); + tmq_list_destroy(topicList); + + int32_t timeout = 200; + + tmq_topic_assignment* pAssign = NULL; + int32_t numOfAssign = 0; + + while (1) { +// printf("start to poll\n"); + + pRes = tmq_consumer_poll(tmq, timeout); + if (pRes) { + tmq_topic_assignment* pAssignTmp = NULL; + int32_t numOfAssignTmp = 0; + + int32_t code = tmq_get_topic_assignment(tmq, "t1", &pAssignTmp, &numOfAssignTmp); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_free_assignment(pAssign); + tmq_consumer_close(tmq); + ASSERT(0); + } + + if(numOfAssign != 0){ + int i = 0; + for(; i < numOfAssign; i++){ + if(pAssign[i].currentOffset != pAssignTmp[i].currentOffset){ + break; + } + } + if(i == numOfAssign){ + ASSERT(0); + } + tmq_free_assignment(pAssign); + } + numOfAssign = numOfAssignTmp; + pAssign = pAssignTmp; + taos_free_result(pRes); + } + } + + tmq_free_assignment(pAssign); +} + +int main(int argc, char* argv[]) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + test_offset(pConn); + test_ts3756(pConn); + taos_close(pConn); + return 0; +} diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index c4becdd38196727b0ac3417c028fb4f9d698ae80..5d4d73c4484092cab49fdb52e9a3479d471b7d9e 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -27,6 +27,7 @@ typedef struct { bool snapShot; bool dropTable; bool subTable; + int meta; int srcVgroups; int dstVgroups; char dir[64]; @@ -511,14 +512,18 @@ int32_t create_topic() { taos_free_result(pRes); if (g_conf.subTable) { - pRes = taos_query(pConn, "create topic meters_summary_t1 with meta as stable meters_summary"); + char topic[128] = {0}; + sprintf(topic, "create topic meters_summary_t1 %s as stable meters_summary", g_conf.meta == 0 ? "with meta" : "only meta"); + pRes = taos_query(pConn, topic); if (taos_errno(pRes) != 0) { printf("failed to create topic meters_summary_t1, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); } else { - pRes = taos_query(pConn, "create topic topic_db with meta as database abc1"); + char topic[128] = {0}; + sprintf(topic, "create topic topic_db %s as database abc1", g_conf.meta == 0 ? "with meta" : "only meta"); + pRes = taos_query(pConn, topic); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_db, reason:%s\n", taos_errstr(pRes)); return -1; @@ -692,95 +697,185 @@ void initLogFile() { } } } else { - if (g_conf.subTable) { - char* result[] = { - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"meters_summary\",\"columns\":[{\"name\":\"_" - "wstart\",\"type\":9},{\"name\":\"current\",\"type\":6},{\"name\":\"groupid\",\"type\":4},{\"name\":" - "\"location\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"group_id\",\"type\":14}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"t_d2a450ee819dcf7576f0282d9ac22dbc\",\"using\":" - "\"meters_summary\",\"tagNum\":1,\"tags\":[{\"name\":\"group_id\",\"type\":14,\"value\":1.313555008277358e+" - "19}],\"createList\":[]}"}; - - for (int i = 0; i < sizeof(result) / sizeof(result[0]); i++) { - taosFprintfFile(pFile2, result[i]); - taosFprintfFile(pFile2, "\n"); + if (g_conf.meta) { + if (g_conf.subTable){ + + }else{ + char* result[] = { + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"tb1\",\"columns\":[{\"name\":\"ts\",\"type\":" + "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":4}],\"tags\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":3000}],\"createList\":[]}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," + "\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\"," + "\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\"," + "\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"alterType\":4,\"colName\":\"t1\"," + "\"colValue\":\"5000\",\"colValueNull\":false}", + "{\"type\":\"drop\",\"tableNameList\":[\"ct3\",\"ct1\"]}", + "{\"type\":\"drop\",\"tableType\":\"super\",\"tableName\":\"st1\"}", + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\"," + "\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\"," + "\"colType\":10,\"colLength\":8}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":10,\"colName\":\"c3\"," + "\"colNewName\":\"cc3\"}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":9}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":6,\"colName\":\"c1\"}", + "{\"type\":\"drop\",\"tableNameList\":[\"n1\"]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9}," + "{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{" + "\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]" + ",\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9}," + "{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", + "{\"type\":\"drop\",\"tableType\":\"super\",\"tableName\":\"st1\"}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":" + "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" + "[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt1\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb1\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt2\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb2\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" + "[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt3\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb3\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt4\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":433},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb4\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":543},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}"}; + + for (int i = 0; i < sizeof(result) / sizeof(result[0]); i++) { + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } } } else { - char* result[] = { - "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"tb1\",\"columns\":[{\"name\":\"ts\",\"type\":" - "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":4}],\"tags\":[]}", - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" - ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," - "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" - "1}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" - "{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{" - "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" - "{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" - "],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" - "{\"name\":\"t1\",\"type\":4,\"value\":3000}],\"createList\":[]}", - "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," - "\"colType\":5}", - "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\"," - "\"colType\":8,\"colLength\":64}", - "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\"," - "\"colType\":8,\"colLength\":64}", - "{\"type\":\"alter\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"alterType\":4,\"colName\":\"t1\"," - "\"colValue\":\"5000\",\"colValueNull\":false}", - "{\"type\":\"delete\",\"sql\":\"delete from `ct3` where `ts` >= 1626006833600 and `ts` <= 1626006833605\"}", - "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" - ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}", - "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\"," - "\"colType\":5}", - "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\"," - "\"colType\":10,\"colLength\":8}", - "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":10,\"colName\":\"c3\"," - "\"colNewName\":\"cc3\"}", - "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":9}", - "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":6,\"colName\":\"c1\"}", - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9}," - "{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{" - "\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]" - ",\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9}" - ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," - "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" - "1}]}", - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":" - "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," - "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" - "1}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" - "[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{" - "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt1\",\"using\":\"stt\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb1\",\"using\":\"sttb\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt2\",\"using\":\"stt\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb2\",\"using\":\"sttb\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" - "[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{" - "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt3\",\"using\":\"stt\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb3\",\"using\":\"sttb\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"sttb3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt4\",\"using\":\"stt\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":433},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"stt4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb4\",\"using\":\"sttb\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":543},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"sttb4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}"}; - - for (int i = 0; i < sizeof(result) / sizeof(result[0]); i++) { - taosFprintfFile(pFile2, result[i]); - taosFprintfFile(pFile2, "\n"); + if (g_conf.subTable) { + char* result[] = { + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"meters_summary\",\"columns\":[{\"name\":\"_" + "wstart\",\"type\":9},{\"name\":\"current\",\"type\":6},{\"name\":\"groupid\",\"type\":4},{\"name\":" + "\"location\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"group_id\",\"type\":14}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"t_d2a450ee819dcf7576f0282d9ac22dbc\",\"using\":" + "\"meters_summary\",\"tagNum\":1,\"tags\":[{\"name\":\"group_id\",\"type\":14,\"value\":1.313555008277358e+" + "19}],\"createList\":[]}"}; + + for (int i = 0; i < sizeof(result) / sizeof(result[0]); i++) { + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } + } + else { + char* result[] = { + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"tb1\",\"columns\":[{\"name\":\"ts\",\"type\":" + "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":4}],\"tags\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":3000}],\"createList\":[]}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," + "\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\"," + "\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\"," + "\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"alterType\":4,\"colName\":\"t1\"," + "\"colValue\":\"5000\",\"colValueNull\":false}", + "{\"type\":\"delete\",\"sql\":\"delete from `ct3` where `ts` >= 1626006833600 and `ts` <= 1626006833605\"}", + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\"," + "\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\"," + "\"colType\":10,\"colLength\":8}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":10,\"colName\":\"c3\"," + "\"colNewName\":\"cc3\"}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":9}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":6,\"colName\":\"c1\"}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9}," + "{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{" + "\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]" + ",\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":" + "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" + "[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt1\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb1\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt2\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb2\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" + "[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt3\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb3\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt4\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":433},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb4\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":543},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}"}; + + for (int i = 0; i < sizeof(result) / sizeof(result[0]); i++) { + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } } } } @@ -802,6 +897,8 @@ int main(int argc, char* argv[]) { g_conf.dstVgroups = atol(argv[++i]); } else if (strcmp(argv[i], "-t") == 0) { g_conf.subTable = true; + } else if (strcmp(argv[i], "-onlymeta") == 0) { + g_conf.meta = 1; } } diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c index 8e5dd6275217b194fa4154fdd1235fdb27e397f1..ee2594af7a89a5710336177debcaff62d6936a59 100644 --- a/utils/test/c/write_raw_block_test.c +++ b/utils/test/c/write_raw_block_test.c @@ -52,7 +52,7 @@ int buildStable(TAOS* pConn, TAOS_RES* pRes) { pRes = taos_query(pConn, "create table d2 using meters tags(3, 'San Francisco')"); if (taos_errno(pRes) != 0) { - printf("failed to create child table d1, reason:%s\n", taos_errstr(pRes)); + printf("failed to create child table d2, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c index 1be28635ef721167d67a33e7837f782c70e17d49..9a0a156717026c1fcc8c4da265199bd2c5aa4e54 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExe.c @@ -434,7 +434,7 @@ bool simExecuteSystemCmd(SScript *script, char *option) { simLogSql(buf, true); int32_t code = system(buf); int32_t repeatTimes = 0; - while (code < 0) { + while (code != 0) { simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d", script->fileName, buf, code, errno, strerror(errno), repeatTimes); taosMsleep(1000); @@ -752,6 +752,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_GEOMETRY: if (length[i] < 0 || length[i] > 1 << 20) { fprintf(stderr, "Invalid length(%d) of BINARY or NCHAR\n", length[i]); exit(-1);