diff --git a/Jenkinsfile2 b/Jenkinsfile2 index c9c9c3a7caa64c626fc5bee918ff5d513a593593..49e13b7831cd903735a33a06a86080ea2b9fcc87 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -8,135 +8,202 @@ def skipbuild=0 def win_stop=0 def abortPreviousBuilds() { - def currentJobName = env.JOB_NAME - def currentBuildNumber = env.BUILD_NUMBER.toInteger() - def jobs = Jenkins.instance.getItemByFullName(currentJobName) - def builds = jobs.getBuilds() + def currentJobName = env.JOB_NAME + def currentBuildNumber = env.BUILD_NUMBER.toInteger() + def jobs = Jenkins.instance.getItemByFullName(currentJobName) + def builds = jobs.getBuilds() - for (build in builds) { - if (!build.isBuilding()) { - continue; - } + for (build in builds) { + if (!build.isBuilding()) { + continue; + } - if (currentBuildNumber == build.getNumber().toInteger()) { - continue; - } + if (currentBuildNumber == build.getNumber().toInteger()) { + continue; + } - build.doKill() //doTerm(),doKill(),doTerm() - } + build.doKill() //doTerm(),doKill(),doTerm() + } } // abort previous build abortPreviousBuilds() def abort_previous(){ - def buildNumber = env.BUILD_NUMBER as int - if (buildNumber > 1) milestone(buildNumber - 1) - milestone(buildNumber) + def buildNumber = env.BUILD_NUMBER as int + if (buildNumber > 1) milestone(buildNumber - 1) + milestone(buildNumber) } def pre_test(){ - sh'hostname' + sh 'hostname' sh ''' - date - sudo rmtaos || echo "taosd has not installed" + date + sudo rmtaos || echo "taosd has not installed" ''' sh ''' - killall -9 taosd ||echo "no taosd running" - killall -9 gdb || echo "no gdb running" - killall -9 python3.8 || echo "no python program running" - cd ${WKC} + killall -9 taosd ||echo "no taosd running" + killall -9 gdb || echo "no gdb running" + killall -9 python3.8 || echo "no python program running" + cd ${WKC} ''' script { - if (env.CHANGE_TARGET == 'master') { - sh ''' - cd ${WKC} - git checkout master - ''' + if (env.CHANGE_TARGET == 'master') { + sh ''' + cd ${WKC} + git checkout master + ''' + } else if(env.CHANGE_TARGET == '2.0') { + sh ''' + cd ${WKC} + git checkout 2.0 + ''' + } else if(env.CHANGE_TARGET == '3.0') { + sh ''' + cd ${WKC} + git checkout 3.0 + [ -d contrib/bdb ] && cd contrib/bdb && git clean -fxd && cd ../.. + ''' + } else { + sh ''' + cd ${WKC} + git checkout develop + ''' } - else if(env.CHANGE_TARGET == '2.0'){ - sh ''' - cd ${WKC} - git checkout 2.0 - ''' - } - else if(env.CHANGE_TARGET == '3.0'){ - sh ''' + } + sh ''' cd ${WKC} - git checkout 3.0 - [ -d contrib/bdb ] && cd contrib/bdb && git clean -fxd && cd ../.. - ''' - } - else{ - sh ''' + git pull >/dev/null + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + git submodule update --init --recursive + ''' + sh ''' cd ${WKC} - git checkout develop - ''' - } - } - sh''' - cd ${WKC} - git pull >/dev/null - git fetch origin +refs/pull/${CHANGE_ID}/merge - git checkout -qf FETCH_HEAD - git submodule update --init --recursive + export TZ=Asia/Harbin + date + rm -rf debug + mkdir debug + cd debug + cmake .. > /dev/null + make -j4> /dev/null ''' - sh''' - cd ${WKC} - export TZ=Asia/Harbin - date - rm -rf debug - mkdir debug - cd debug - cmake .. > /dev/null - make -j4> /dev/null + sh ''' + cd ${WKPY} + git reset --hard + git pull + pip3 install . ''' - sh''' - cd ${WKPY} - git reset --hard - git pull - pip3 install . + return 1 +} +def pre_test_win(){ + bat ''' + hostname + date /t + time /t + taskkill /f /t /im python.exe + taskkill /f /t /im bash.exe + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine\\debug + exit 0 + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git reset --hard + git fetch || git fetch + git checkout -f + ''' + script { + if (env.CHANGE_TARGET == 'master') { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout master + ''' + } else if(env.CHANGE_TARGET == '2.0') { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout 2.0 + ''' + } else if(env.CHANGE_TARGET == '3.0') { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout 3.0 + ''' + } else { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout develop + ''' + } + } + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git branch + git pull || git pull + git fetch origin +refs/pull/%CHANGE_ID%/merge + git checkout -qf FETCH_HEAD + ''' +} +def pre_test_build_win() { + bat ''' + echo "building ..." + time /t + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + mkdir debug + cd debug + call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64 + set CL=/MP8 + cmake .. -G "NMake Makefiles JOM" + jom -j 4 || exit 8 + time /t ''' return 1 } pipeline { - agent none - options { skipDefaultCheckout() } - environment{ - WK = '/var/lib/jenkins/workspace/TDinternal' - WKC= '/var/lib/jenkins/workspace/TDengine' - WKPY= '/var/lib/jenkins/workspace/taos-connector-python' - } - stages { - stage('pre_build'){ - agent{label " slave3_0 || slave15 || slave16 || slave17 "} - options { skipDefaultCheckout() } - when { - changeRequest() - } - steps { - script{ - abort_previous() - abortPreviousBuilds() - } - timeout(time: 45, unit: 'MINUTES'){ - pre_test() - sh''' - cd ${WKC}/debug - ctest -VV - ''' - sh''' - export LD_LIBRARY_PATH=${WKC}/debug/build/lib - cd ${WKC}/tests/system-test - ./fulltest.sh - ''' - sh''' - cd ${WKC}/tests - ./test-all.sh b1fq - ''' + agent none + options { skipDefaultCheckout() } + environment{ + WK = '/var/lib/jenkins/workspace/TDinternal' + WKC= '/var/lib/jenkins/workspace/TDengine' + WKPY= '/var/lib/jenkins/workspace/taos-connector-python' + } + stages { + stage('run test') { + parallel { + stage('windows test') { + agent {label " windows11 "} + steps { + pre_test_win() + pre_test_build_win() + } + } + stage('linux test') { + agent{label " slave3_0 || slave15 || slave16 || slave17 "} + options { skipDefaultCheckout() } + when { + changeRequest() + } + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + cd ${WKC}/debug + ctest -VV + ''' + sh ''' + export LD_LIBRARY_PATH=${WKC}/debug/build/lib + cd ${WKC}/tests/system-test + ./fulltest.sh + ''' + sh ''' + cd ${WKC}/tests + ./test-all.sh b1fq + ''' + } + } + } } - } - } - } - post { + } + } + post { success { emailext ( subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' SUCCESS", diff --git a/cmake/cmake.define b/cmake/cmake.define index fb6ba8cc2e2703d17e45593a925fffdcbcb5bbb8..d1d9266bca9319bb997bf16f6ce278dd682c2958 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 3.16) set(CMAKE_VERBOSE_MAKEFILE OFF) +SET(BUILD_SHARED_LIBS "OFF") + #set output directory SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin) @@ -44,7 +46,6 @@ ENDIF () IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") - SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) SET(COMMON_FLAGS "/W3 /D_WIN32") # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) @@ -68,7 +69,13 @@ ELSE () IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") ADD_DEFINITIONS("-D_TD_ARM_") ELSE () - ADD_DEFINITIONS("-msse4.2 -mfma") + ADD_DEFINITIONS("-msse4.2") + IF("${FMA_SUPPORT}" MATCHES "true") + MESSAGE(STATUS "turn fma function support on") + ADD_DEFINITIONS("-mfma") + ELSE () + MESSAGE(STATUS "turn fma function support off") + ENDIF() ENDIF () ENDIF () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e797911adde33b1561e6d6094446dd28f0f1bc88..df69eb8aa1092dbb3730bdc64eaaf130c56aa32e 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -228,7 +228,7 @@ endif() # iconv if(${BUILD_WITH_ICONV}) - add_subdirectory(iconv EXCLUDE_FROM_ALL) + add_library(iconv STATIC iconv/win_iconv.c) endif(${BUILD_WITH_ICONV}) # wingetopt diff --git a/example/src/tmq.c b/example/src/tmq.c index 5b8f66f6665d91a04c418c9fb8978be0a81a7d8a..976d658fa6fc6557c11fde9e5c692fba801ab1f4 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -24,7 +24,7 @@ static void msg_process(TAOS_RES* msg) { char buf[1024]; memset(buf, 0, 1024); printf("topic: %s\n", tmq_get_topic_name(msg)); - printf("vg:%d\n", tmq_get_vgroup_id(msg)); + printf("vg: %d\n", tmq_get_vgroup_id(msg)); while (1) { TAOS_ROW row = taos_fetch_row(msg); if (row == NULL) break; @@ -140,8 +140,8 @@ int32_t create_topic() { return 0; } -void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets) { - printf("commit %d\n", resp); +void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { + printf("commit %d tmq %p offsets %p param %p\n", resp, tmq, offsets, param); } tmq_t* build_consumer() { @@ -161,7 +161,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); /*tmq_conf_set(conf, "td.connect.db", "abc1");*/ - tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); + tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); assert(tmq); return tmq; @@ -215,12 +215,24 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { return; } + tmq_list_t* subList = NULL; + tmq_subscription(tmq, &subList); + char** subTopics = tmq_list_to_c_array(subList); + int32_t sz = tmq_list_get_size(subList); + printf("subscribed topics: "); + for (int32_t i = 0; i < sz; i++) { + printf("%s, ", subTopics[i]); + } + printf("\n"); + tmq_list_destroy(subList); + while (running) { TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000); if (tmqmessage) { msg_process(tmqmessage); taos_free_result(tmqmessage); + tmq_commit(tmq, NULL, 1); /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/ } } diff --git a/example/src/tstream.c b/example/src/tstream.c index 8ffa932bd29d6f89441c7f045611f42e058dd872..5bd833213d7706b1151f3b565d4b7e567f62c655 100644 --- a/example/src/tstream.c +++ b/example/src/tstream.c @@ -25,7 +25,7 @@ int32_t init_env() { return -1; } - TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); if (taos_errno(pRes) != 0) { printf("error in create db, reason:%s\n", taos_errstr(pRes)); return -1; @@ -78,11 +78,15 @@ int32_t create_stream() { taos_free_result(pRes); /*const char* sql = "select min(k), max(k), sum(k) from tu1";*/ - const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1"; + /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/ /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ - pRes = tmq_create_stream(pConn, "stream1", "out1", sql); + /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ + pRes = + taos_query(pConn, + "create stream stream1 trigger window_close as select _wstartts, min(k), max(k), sum(k) as sum_of_k " + "from tu1 interval(10m)"); if (taos_errno(pRes) != 0) { - printf("failed to create stream out1, reason:%s\n", taos_errstr(pRes)); + printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); diff --git a/include/client/taos.h b/include/client/taos.h index 1ac92c61a57a9a7bcef10d03930de73466ecd8b4..6e209006681f2122a5a68ef890f9471adae61831 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -86,9 +86,9 @@ typedef struct taosField { } TAOS_FIELD; #ifdef WINDOWS - #define DLL_EXPORT __declspec(dllexport) +#define DLL_EXPORT __declspec(dllexport) #else - #define DLL_EXPORT +#define DLL_EXPORT #endif typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); @@ -123,42 +123,42 @@ DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT setConfRet taos_set_config(const char *config); DLL_EXPORT int taos_init(void); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); -DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, +DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port); -DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); -DLL_EXPORT void taos_close(TAOS *taos); - -const char *taos_data_type(int type); - -DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); -DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); -DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags); -DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); -DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); - -DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); -DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); -DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); -DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); -DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); -DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); -DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); -DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); -DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); - -DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); -DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); - -DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); -DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result -DLL_EXPORT void taos_free_result(TAOS_RES *res); -DLL_EXPORT int taos_field_count(TAOS_RES *res); -DLL_EXPORT int taos_num_fields(TAOS_RES *res); -DLL_EXPORT int taos_affected_rows(TAOS_RES *res); +DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); +DLL_EXPORT void taos_close(TAOS *taos); + +const char *taos_data_type(int type); + +DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); +DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); +DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags); +DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); +DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); + +DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); +DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); +DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); +DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); +DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); +DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); +DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); +DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); +DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); + +DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); +DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); + +DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); +DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result +DLL_EXPORT void taos_free_result(TAOS_RES *res); +DLL_EXPORT int taos_field_count(TAOS_RES *res); +DLL_EXPORT int taos_num_fields(TAOS_RES *res); +DLL_EXPORT int taos_affected_rows(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); @@ -213,7 +213,7 @@ typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t; typedef struct tmq_conf_t tmq_conf_t; typedef struct tmq_list_t tmq_list_t; -typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *)); +typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param)); DLL_EXPORT tmq_list_t *tmq_list_new(); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); @@ -253,12 +253,12 @@ typedef enum tmq_conf_res_t tmq_conf_res_t; DLL_EXPORT tmq_conf_t *tmq_conf_new(); DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); -DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb); +DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param); /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */ -DLL_EXPORT char *tmq_get_topic_name(TAOS_RES *res); -DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); +DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); +DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); // TODO #if 0 DLL_EXPORT char *tmq_get_table_name(TAOS_RES *res); @@ -271,10 +271,8 @@ DLL_EXPORT int64_t tmq_get_response_offset(tmq_message_t *message); /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */ #if 0 DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); -#endif - DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql); - +#endif /* ------------------------------ TMQ END -------------------------------- */ #if 1 // Shuduo: temporary enable for app build typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); diff --git a/include/common/systable.h b/include/common/systable.h new file mode 100644 index 0000000000000000000000000000000000000000..bd8aae998f9ab6a692b7c5793fbac0c58ae84eb7 --- /dev/null +++ b/include/common/systable.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" + +#ifndef TDENGINE_SYSTABLE_H +#define TDENGINE_SYSTABLE_H + +#define TSDB_INFORMATION_SCHEMA_DB "information_schema" +#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" +#define TSDB_INS_TABLE_DNODES "dnodes" +#define TSDB_INS_TABLE_MNODES "mnodes" +#define TSDB_INS_TABLE_MODULES "modules" +#define TSDB_INS_TABLE_QNODES "qnodes" +#define TSDB_INS_TABLE_BNODES "bnodes" +#define TSDB_INS_TABLE_SNODES "snodes" +#define TSDB_INS_TABLE_CLUSTER "cluster" +#define TSDB_INS_TABLE_USER_DATABASES "user_databases" +#define TSDB_INS_TABLE_USER_FUNCTIONS "user_functions" +#define TSDB_INS_TABLE_USER_INDEXES "user_indexes" +#define TSDB_INS_TABLE_USER_STABLES "user_stables" +#define TSDB_INS_TABLE_USER_STREAMS "user_streams" +#define TSDB_INS_TABLE_USER_TABLES "user_tables" +#define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed" +#define TSDB_INS_TABLE_USER_USERS "user_users" +#define TSDB_INS_TABLE_LICENCES "grants" +#define TSDB_INS_TABLE_VGROUPS "vgroups" +#define TSDB_INS_TABLE_VNODES "vnodes" +#define TSDB_INS_TABLE_CONFIGS "configs" + +#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" +#define TSDB_PERFS_TABLE_SMAS "smas" +#define TSDB_PERFS_TABLE_SUBSCRIBES "subscribes" +#define TSDB_PERFS_TABLE_CONNECTIONS "connections" +#define TSDB_PERFS_TABLE_QUERIES "queries" +#define TSDB_PERFS_TABLE_TOPICS "topics" +#define TSDB_PERFS_TABLE_CONSUMERS "consumers" +#define TSDB_PERFS_TABLE_SUBSCRIPTIONS "subscriptions" +#define TSDB_PERFS_TABLE_OFFSETS "offsets" +#define TSDB_PERFS_TABLE_TRANS "trans" +#define TSDB_PERFS_TABLE_STREAMS "streams" + +typedef struct SSysDbTableSchema { + const char *name; + const int32_t type; + const int32_t bytes; +} SSysDbTableSchema; + +typedef struct SSysTableMeta { + const char *name; + const SSysDbTableSchema *schema; + const int32_t colNum; +} SSysTableMeta; + +void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size); +void getPerfDbMeta(const SSysTableMeta** pPerfsTableMeta, size_t* size); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_SYSTABLE_H diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 1fbbb3f1599afdf77957869ff6d3399a318fda2f..e1f8832edf146f71deed320a21c24c5df5a25292 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -28,15 +28,16 @@ typedef int64_t tb_uid_t; #define TSWINDOW_INITIALIZER ((STimeWindow){INT64_MIN, INT64_MAX}) #define TSWINDOW_DESC_INITIALIZER ((STimeWindow){INT64_MAX, INT64_MIN}) #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) +#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) typedef enum { - TSDB_SUPER_TABLE = 1, // super table - TSDB_CHILD_TABLE = 2, // table created from super table - TSDB_NORMAL_TABLE = 3, // ordinary table - TSDB_STREAM_TABLE = 4, // table created from stream computing - TSDB_TEMP_TABLE = 5, // temp table created by nest query + TSDB_SUPER_TABLE = 1, // super table + TSDB_CHILD_TABLE = 2, // table created from super table + TSDB_NORMAL_TABLE = 3, // ordinary table + TSDB_STREAM_TABLE = 4, // table created from stream computing + TSDB_TEMP_TABLE = 5, // temp table created by nest query TSDB_SYSTEM_TABLE = 6, - TSDB_TABLE_MAX = 7 + TSDB_TABLE_MAX = 7 } ETableType; typedef enum { @@ -71,16 +72,22 @@ typedef enum { TSDB_SMA_STAT_DROPPED = 2, // sma dropped } ETsdbSmaStat; // bit operation - typedef enum { TSDB_SMA_TYPE_BLOCK = 0, // Block-wise SMA TSDB_SMA_TYPE_TIME_RANGE = 1, // Time-range-wise SMA TSDB_SMA_TYPE_ROLLUP = 2, // Rollup SMA } ETsdbSmaType; +typedef enum { + TSDB_RETENTION_L0 = 0, + TSDB_RETENTION_L1 = 1, + TSDB_RETENTION_L2 = 2, + TSDB_RETENTION_MAX = 3 +} ERetentionLevel; + extern char *qtypeStr[]; -#define TSDB_PORT_HTTP 11 +#define TSDB_PORT_HTTP 11 #undef TD_DEBUG_PRINT_ROW diff --git a/include/common/tcommon.h b/include/common/tcommon.h index fa428a994a3bb2ebe9a3e90fd6dcfc1622fbd31a..16653ebfeed9001c80b897e8c59dcd6cb2b7436c 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -65,20 +65,18 @@ typedef struct SDataBlockInfo { STimeWindow window; int32_t rows; int32_t rowSize; - union { - int64_t uid; // from which table of uid, comes from this data block - int64_t blockId; - }; - uint64_t groupId; // no need to serialize - int16_t numOfCols; - int16_t hasVarCol; - int16_t capacity; + int64_t uid; // the uid of table, from which current data block comes + int64_t blockId; // block id, generated by physical planner + uint64_t groupId; // no need to serialize + int16_t numOfCols; + int16_t hasVarCol; + int32_t capacity; } SDataBlockInfo; typedef struct SSDataBlock { - SColumnDataAgg* pBlockAgg; - SArray* pDataBlock; // SArray - SDataBlockInfo info; + SColumnDataAgg** pBlockAgg; + SArray* pDataBlock; // SArray + SDataBlockInfo info; } SSDataBlock; typedef struct SVarColAttr { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 71e2e4fba36371afa64e07d35a19adcca7938bfb..19b108dcbf6f6c7f542f8d978a8b63eae17c3af9 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -54,6 +54,11 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet); BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) +#define colDataSetNotNull_f(bm_, r_) \ + do { \ + BMCharPos(bm_, r_) &= ~(1u << (7u - BitPos(r_))); \ + } while (0) + #define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1) #define colDataSetNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] = -1) @@ -63,16 +68,15 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet); #define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes)) // SColumnInfoData, rowNumber -#define colDataGetData(p1_, r_) \ - ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) \ - : colDataGetNumData(p1_, r_)) +#define colDataGetData(p1_, r_) \ + ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_)) static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { - if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON){ - if(colDataIsNull_var(pColumnInfoData, row)){ + if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) { + if (colDataIsNull_var(pColumnInfoData, row)) { return true; } - char *data = colDataGetVarData(pColumnInfoData, row); + char* data = colDataGetVarData(pColumnInfoData, row); return (*data == TSDB_DATA_TYPE_NULL); } @@ -80,7 +84,7 @@ static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, return false; } - if (pColumnInfoData->info.type== TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) { + if (pColumnInfoData->info.type == TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) { return colDataIsNull_var(pColumnInfoData, row); } else { if (pColumnInfoData->nullbitmap == NULL) { @@ -132,7 +136,7 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { for (int32_t i = start; i < start + nRows; ++i) { - colDataSetNull_var(pColumnInfoData,i); // it is a null value of VAR type. + colDataSetNull_var(pColumnInfoData, i); // it is a null value of VAR type. } } else { for (int32_t i = start; i < start + nRows; ++i) { @@ -183,7 +187,7 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u } int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock); @@ -225,6 +229,11 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); void blockDebugShowData(const SArray* dataBlocks); +int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, + tb_uid_t uid, tb_uid_t suid); + +SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema); + static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { return blockDataGetSerialMetaSize(pBlock) + blockDataGetSize(pBlock); } @@ -238,10 +247,10 @@ static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32 static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { - int32_t* actualLen = (int32_t*) data; + int32_t* actualLen = (int32_t*)data; data += sizeof(int32_t); - uint64_t* groupId = (uint64_t*) data; + uint64_t* groupId = (uint64_t*)data; data += sizeof(uint64_t); int32_t* colSizes = (int32_t*)data; diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index a0f4351bbf725c23b8d9e9291fc6a04b5dfb0460..11d0f34c4560b0399acac2a455836e7ed9326c9e 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -61,22 +61,22 @@ extern "C" { // ----------------- TSDB COLUMN DEFINITION #pragma pack(push, 1) typedef struct { - col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) - int32_t type : 8; // column type - int32_t bytes : 24; // column bytes (0~16M) - int32_t sma : 8; // block SMA: 0, no SMA, 1, sum/min/max, 2, ... - int32_t offset : 24; // point offset in STpRow after the header part. + col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) + int8_t type; // column type + int8_t flags; // flags: 0 no index, 1 SCHEMA_SMA_ON, 2 SCHEMA_IDX_ON + int32_t bytes; // column bytes (0~16M) + int32_t offset; // point offset in STpRow after the header part. } STColumn; #pragma pack(pop) #define colType(col) ((col)->type) -#define colSma(col) ((col)->sma) +#define colFlags(col) ((col)->flags) #define colColId(col) ((col)->colId) #define colBytes(col) ((col)->bytes) #define colOffset(col) ((col)->offset) #define colSetType(col, t) (colType(col) = (t)) -#define colSetSma(col, s) (colSma(col) = (s)) +#define colSetFlags(col, f) (colFlags(col) = (f)) #define colSetColId(col, id) (colColId(col) = (id)) #define colSetBytes(col, b) (colBytes(col) = (b)) #define colSetOffset(col, o) (colOffset(col) = (o)) @@ -146,7 +146,7 @@ typedef struct { int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes); +int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); // ----------------- Semantic timestamp key definition diff --git a/include/common/tglobal.h b/include/common/tglobal.h index fc3d575317a5e8e68657d967b6d2e2afa3458d13..f253d3196396b1a880ee5a6f73d08f739c3ac710 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -121,6 +121,10 @@ extern char tsCompressor[]; extern int32_t tsDiskCfgNum; extern SDiskCfg tsDiskCfg[]; +// internal +extern int32_t tsTransPullupMs; +extern int32_t tsMaRebalanceMs; + #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 89f7cd6c39d86c15da0eebb590bb6c963d4ad215..3889784f2f0c1876427124b98c73a795d65e7b69 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -73,7 +73,8 @@ typedef uint16_t tmsg_t; enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__MAX }; enum { - HEARTBEAT_KEY_DBINFO = 1, + HEARTBEAT_KEY_USER_AUTHINFO = 1, + HEARTBEAT_KEY_DBINFO, HEARTBEAT_KEY_STBINFO, HEARTBEAT_KEY_MQ_TMP, }; @@ -181,8 +182,8 @@ typedef struct SField { } SField; typedef struct SRetention { - int32_t freq; - int32_t keep; + int64_t freq; + int64_t keep; int8_t freqUnit; int8_t keepUnit; } SRetention; @@ -243,17 +244,10 @@ typedef struct { int32_t tInitSubmitMsgIter(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); -int32_t tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter); +int32_t tInitSubmitBlkIter(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter); STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); - -// TODO: KEEP one suite of iterator API finally. -// 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts -// 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later -// 3) finally, rename tInitSubmitMsgIterEx to tInitSubmitMsgIter -int32_t tInitSubmitMsgIterEx(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); -int32_t tGetSubmitMsgNextEx(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); -int32_t tInitSubmitBlkIterEx(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter); -STSRow* tGetSubmitBlkNextEx(SSubmitBlkIter* pIter); +// for debug +int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq* pReq, STSchema* pSchema); typedef struct { int32_t index; // index of failed block in submit blocks @@ -281,8 +275,10 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; +#define IS_BSMA_ON(s) (((s)->flags & 0x01) == SCHEMA_SMA_ON) + #define SSCHMEA_TYPE(s) ((s)->type) -#define SSCHMEA_SMA(s) ((s)->sma) +#define SSCHMEA_FLAGS(s) ((s)->flags) #define SSCHMEA_COLID(s) ((s)->colId) #define SSCHMEA_BYTES(s) ((s)->bytes) #define SSCHMEA_NAME(s) ((s)->name) @@ -431,7 +427,9 @@ int32_t tDeserializeSGetUserAuthReq(void* buf, int32_t bufLen, SGetUserAuthReq* typedef struct { char user[TSDB_USER_LEN]; + int32_t version; int8_t superAuth; + SHashObj* createdDbs; SHashObj* readDbs; SHashObj* writeDbs; } SGetUserAuthRsp; @@ -531,27 +529,24 @@ int32_t tDeserializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pR typedef struct { char db[TSDB_DB_FNAME_LEN]; int32_t numOfVgroups; - int32_t cacheBlockSize; // MB - int32_t totalBlocks; + int32_t numOfStables; // single_stable + int32_t buffer; // MB + int32_t pageSize; + int32_t pages; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; int32_t daysToKeep2; int32_t minRows; int32_t maxRows; - int32_t commitTime; int32_t fsyncPeriod; - int32_t ttl; int8_t walLevel; int8_t precision; // time resolution int8_t compression; int8_t replications; int8_t strict; - int8_t update; int8_t cacheLastRow; int8_t ignoreExist; - int8_t streamMode; - int8_t singleSTable; int32_t numOfRetensions; SArray* pRetensions; // SRetention } SCreateDbReq; @@ -562,7 +557,10 @@ void tFreeSCreateDbReq(SCreateDbReq* pReq); typedef struct { char db[TSDB_DB_FNAME_LEN]; - int32_t totalBlocks; + int32_t buffer; + int32_t pageSize; + int32_t pages; + int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; int32_t daysToKeep2; @@ -626,26 +624,23 @@ int32_t tDeserializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq); typedef struct { int32_t numOfVgroups; - int32_t cacheBlockSize; - int32_t totalBlocks; + int32_t numOfStables; + int32_t buffer; + int32_t pageSize; + int32_t pages; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; int32_t daysToKeep2; int32_t minRows; int32_t maxRows; - int32_t commitTime; int32_t fsyncPeriod; - int32_t ttl; int8_t walLevel; int8_t precision; int8_t compression; int8_t replications; int8_t strict; - int8_t update; int8_t cacheLastRow; - int8_t streamMode; - int8_t singleSTable; int32_t numOfRetensions; SArray* pRetensions; } SDbCfgRsp; @@ -677,10 +672,20 @@ typedef struct { SArray* pArray; // Array of SUseDbRsp } SUseDbBatchRsp; + int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); +typedef struct { + SArray* pArray; // Array of SGetUserAuthRsp +} SUserAuthBatchRsp; + +int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); +int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); +void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp); + + typedef struct { char db[TSDB_DB_FNAME_LEN]; } SCompactDbReq; @@ -845,15 +850,16 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; int64_t dbUid; int32_t vgVersion; - int32_t cacheBlockSize; - int32_t totalBlocks; + int32_t numOfStables; + int32_t buffer; + int32_t pageSize; + int32_t pages; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; int32_t daysToKeep2; int32_t minRows; int32_t maxRows; - int32_t commitTime; int32_t fsyncPeriod; uint32_t hashBegin; uint32_t hashEnd; @@ -862,11 +868,9 @@ typedef struct { int8_t precision; int8_t compression; int8_t strict; - int8_t update; int8_t cacheLastRow; int8_t replica; int8_t selfIndex; - int8_t streamMode; SReplica replicas[TSDB_MAX_REPLICA]; int32_t numOfRetensions; SArray* pRetensions; // SRetention @@ -896,10 +900,14 @@ int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq typedef struct { int32_t vgVersion; - int32_t totalBlocks; + int32_t buffer; + int32_t pageSize; + int32_t pages; + int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; int32_t daysToKeep2; + int32_t fsyncPeriod; int8_t walLevel; int8_t strict; int8_t cacheLastRow; @@ -954,7 +962,6 @@ typedef struct { int32_t numOfColumns; int8_t precision; int8_t tableType; - int8_t update; int32_t sversion; int32_t tversion; uint64_t suid; @@ -1039,6 +1046,7 @@ typedef struct { int8_t compressed; int32_t compLen; int32_t numOfRows; + int32_t numOfCols; char data[]; } SRetrieveTableRsp; @@ -1442,39 +1450,38 @@ typedef struct { SArray* lostConsumers; // SArray SArray* removedConsumers; // SArray SArray* newConsumers; // SArray -} SMqRebSubscribe; +} SMqRebInfo; -static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) { - SMqRebSubscribe* pRebSub = (SMqRebSubscribe*)taosMemoryCalloc(1, sizeof(SMqRebSubscribe)); - if (pRebSub == NULL) { +static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) { + SMqRebInfo* pRebInfo = (SMqRebInfo*)taosMemoryCalloc(1, sizeof(SMqRebInfo)); + if (pRebInfo == NULL) { goto _err; } - strcpy(pRebSub->key, key); - pRebSub->lostConsumers = taosArrayInit(0, sizeof(int64_t)); - if (pRebSub->lostConsumers == NULL) { + strcpy(pRebInfo->key, key); + pRebInfo->lostConsumers = taosArrayInit(0, sizeof(int64_t)); + if (pRebInfo->lostConsumers == NULL) { goto _err; } - pRebSub->removedConsumers = taosArrayInit(0, sizeof(int64_t)); - if (pRebSub->removedConsumers == NULL) { + pRebInfo->removedConsumers = taosArrayInit(0, sizeof(int64_t)); + if (pRebInfo->removedConsumers == NULL) { goto _err; } - pRebSub->newConsumers = taosArrayInit(0, sizeof(int64_t)); - if (pRebSub->newConsumers == NULL) { + pRebInfo->newConsumers = taosArrayInit(0, sizeof(int64_t)); + if (pRebInfo->newConsumers == NULL) { goto _err; } - return pRebSub; + return pRebInfo; _err: - taosArrayDestroy(pRebSub->lostConsumers); - taosArrayDestroy(pRebSub->removedConsumers); - taosArrayDestroy(pRebSub->newConsumers); - taosMemoryFreeClear(pRebSub); + taosArrayDestroy(pRebInfo->lostConsumers); + taosArrayDestroy(pRebInfo->removedConsumers); + taosArrayDestroy(pRebInfo->newConsumers); + taosMemoryFreeClear(pRebInfo); return NULL; } // this message is sent from mnode to mnode(read thread to write thread), so there is no need for serialization or // deserialization typedef struct { - int8_t* mqInReb; SHashObj* rebSubHash; // SHashObj } SMqDoRebalanceMsg; @@ -1525,6 +1532,7 @@ typedef struct { int32_t tEncodeSRSmaParam(SCoder* pCoder, const SRSmaParam* pRSmaParam); int32_t tDecodeSRSmaParam(SCoder* pCoder, SRSmaParam* pRSmaParam); +// TDMT_VND_CREATE_STB ============== typedef struct SVCreateStbReq { const char* name; tb_uid_t suid; @@ -1537,17 +1545,14 @@ typedef struct SVCreateStbReq { int tEncodeSVCreateStbReq(SCoder* pCoder, const SVCreateStbReq* pReq); int tDecodeSVCreateStbReq(SCoder* pCoder, SVCreateStbReq* pReq); +// TDMT_VND_DROP_STB ============== typedef struct SVDropStbReq { - // data -#ifdef WINDOWS - size_t avoidCompilationErrors; -#endif - + const char* name; + tb_uid_t suid; } SVDropStbReq; -typedef struct SVCreateStbRsp { - int code; -} SVCreateStbRsp; +int32_t tEncodeSVDropStbReq(SCoder* pCoder, const SVDropStbReq* pReq); +int32_t tDecodeSVDropStbReq(SCoder* pCoder, SVDropStbReq* pReq); typedef struct SVCreateTbReq { tb_uid_t uid; @@ -1604,19 +1609,37 @@ int tDecodeSVCreateTbBatchRsp(SCoder* pCoder, SVCreateTbBatchRsp* pRsp); int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); +// TDMT_VND_DROP_TABLE ================= typedef struct { - int64_t ver; - char* name; - uint8_t type; - tb_uid_t suid; + const char* name; + int8_t igNotExists; } SVDropTbReq; typedef struct { - int tmp; // TODO: to avoid compile error + int32_t code; } SVDropTbRsp; -int32_t tSerializeSVDropTbReq(void** buf, SVDropTbReq* pReq); -void* tDeserializeSVDropTbReq(void* buf, SVDropTbReq* pReq); +typedef struct { + int32_t nReqs; + union { + SVDropTbReq* pReqs; + SArray* pArray; + }; +} SVDropTbBatchReq; + +int32_t tEncodeSVDropTbBatchReq(SCoder* pCoder, const SVDropTbBatchReq* pReq); +int32_t tDecodeSVDropTbBatchReq(SCoder* pCoder, SVDropTbBatchReq* pReq); + +typedef struct { + int32_t nRsps; + union { + SVDropTbRsp* pRsps; + SArray* pArray; + }; +} SVDropTbBatchRsp; + +int32_t tEncodeSVDropTbBatchRsp(SCoder* pCoder, const SVDropTbBatchRsp* pRsp); +int32_t tDecodeSVDropTbBatchRsp(SCoder* pCoder, SVDropTbBatchRsp* pRsp); typedef struct { SMsgHead head; @@ -2122,7 +2145,7 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SCoder* pDecoder, SSchemaWrapp if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1; if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1; - pSW->pSchema = (SSchema*)tCoderMalloc(pDecoder, sizeof(SSchema) * pSW->nCols); + pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) return -1; for (int32_t i = 0; i < pSW->nCols; i++) { if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1; @@ -2568,6 +2591,28 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp); } +#define TD_AUTO_CREATE_TABLE 0x1 +typedef struct { + int64_t suid; + int64_t uid; + int32_t sver; + uint64_t nData; + const void* pData; + SVCreateTbReq cTbReq; +} SVSubmitBlk; + +typedef struct { + int32_t flags; + int32_t nBlocks; + union { + SArray* pArray; + SVSubmitBlk* pBlocks; + }; +} SVSubmitReq; + +int32_t tEncodeSVSubmitReq(SCoder* pCoder, const SVSubmitReq* pReq); +int32_t tDecodeSVSubmitReq(SCoder* pCoder, SVSubmitReq* pReq); + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 02d8b76b9b50528cb782ae05036f115f15374600..4002db06ea9d72e084ef28045ed233e779330b75 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -42,6 +42,7 @@ typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueTy typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, const SEpSet* epSet, SRpcMsg* pReq); typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq); typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp); +typedef void (*SendRedirectRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp, const SEpSet* pNewEpSet); typedef void (*RegisterBrokenLinkArgFp)(SMgmtWrapper* pWrapper, SRpcMsg* pMsg); typedef void (*ReleaseHandleFp)(SMgmtWrapper* pWrapper, void* handle, int8_t type); typedef void (*ReportStartup)(SMgmtWrapper* pWrapper, const char* name, const char* desc); @@ -52,6 +53,7 @@ typedef struct { GetQueueSizeFp qsizeFp; SendReqFp sendReqFp; SendRspFp sendRspFp; + SendRedirectRspFp sendRedirectRspFp; RegisterBrokenLinkArgFp registerBrokenLinkArgFp; ReleaseHandleFp releaseHandleFp; ReportStartup reportStartupFp; @@ -62,6 +64,7 @@ int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq); int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype); int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq); void tmsgSendRsp(const SRpcMsg* pRsp); +void tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet); void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg); void tmsgReleaseHandle(void* handle, int8_t type); void tmsgReportStartup(const char* name, const char* desc); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index e1ee946ce4e0678d35a809304429a5d5d7d57b90..36bef5e85a905bae1f4c84f71e352257791a600e 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -170,9 +170,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_UPDATE_TAG_VAL, "vnode-update-tag-val", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TABLE_META, "vnode-table-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TABLES_META, "vnode-tables-meta", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_CREATE_STB, "vnode-create-stb", SVCreateTbReq, SVCreateTbRsp) + TD_DEF_MSG_TYPE(TDMT_VND_CREATE_STB, "vnode-create-stb", SVCreateStbReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_STB, "vnode-alter-stb", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", SVDropStbReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONSUME, "vnode-mq-consume", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL) @@ -202,6 +202,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp) // sync integration TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL) diff --git a/include/common/trow.h b/include/common/trow.h index 24baf25e1f2cdb8af5189fabfd2594d918ec0760..f87ea1f00926cf480390f82b515708b4cab18dc3 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -141,7 +141,7 @@ typedef struct { /// row total length uint32_t len; /// row version - uint64_t ver; + // uint64_t ver; /// the inline data, maybe a tuple or a k-v tuple char data[]; } STSRow; @@ -176,7 +176,7 @@ typedef struct { #define TD_ROW_DATA(r) ((r)->data) #define TD_ROW_LEN(r) ((r)->len) #define TD_ROW_KEY(r) ((r)->ts) -#define TD_ROW_VER(r) ((r)->ver) +// #define TD_ROW_VER(r) ((r)->ver) #define TD_ROW_KEY_ADDR(r) (r) // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and @@ -595,6 +595,33 @@ static FORCE_INLINE int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, return TSDB_CODE_SUCCESS; } +/** + * @brief + * + * @param pBuilder + * @param nCols + * @param nBoundCols use -1 if not available + * @param flen + * @return FORCE_INLINE + */ +static FORCE_INLINE int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) { + pBuilder->flen = flen; + pBuilder->nCols = nCols; + if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } +#ifdef TD_SUPPORT_BITMAP + // the primary TS key is stored separatedly + pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); +#else + pBuilder->nBitmaps = 0; + pBuilder->nBoundBitmaps = 0; +#endif + return TSDB_CODE_SUCCESS; +} + /** * @brief To judge row type: STpRow/SKvRow * @@ -730,7 +757,6 @@ static int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { return TSDB_CODE_SUCCESS; } - /** * @brief 由调用方管理存储空间的分配及释放，一次输入多个参数 * @@ -1222,16 +1248,16 @@ static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, in } /** - * @brief - * - * @param pRow - * @param colId - * @param colType - * @param flen - * @param offset + * @brief + * + * @param pRow + * @param colId + * @param colType + * @param flen + * @param offset * @param colIdx start from 0 - * @param pVal - * @return FORCE_INLINE + * @param pVal + * @return FORCE_INLINE */ static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { @@ -1245,14 +1271,14 @@ static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t } /** - * @brief - * - * @param pRow - * @param colId - * @param offset + * @brief + * + * @param pRow + * @param colId + * @param offset * @param colIdx start from 0 - * @param pVal - * @return FORCE_INLINE + * @param pVal + * @return FORCE_INLINE */ static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { @@ -1369,14 +1395,14 @@ static void tdSCellValPrint(SCellVal *pVal, int8_t colType) { } } -static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char* tag) { +static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { STSRowIter iter = {0}; tdSTSRowIterInit(&iter, pSchema); tdSTSRowIterReset(&iter, row); printf("%s >>>", tag); for (int i = 0; i < pSchema->numOfCols; ++i) { STColumn *stCol = pSchema->columns + i; - SCellVal sVal = {.valType = 255, .val = NULL}; + SCellVal sVal = {255, NULL}; if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { break; } @@ -1385,7 +1411,6 @@ static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char* tag) { } printf("\n"); } - #ifdef TROW_ORIGIN_HZ typedef struct { uint32_t nRows; diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 4fab11e2379627ec828de77d263002a2182fc1bc..a56c6ca16dc785a14845d64f1ee897023f72d366 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -107,10 +107,9 @@ typedef enum { typedef enum { SDB_STATUS_INIT = 0, SDB_STATUS_CREATING = 1, - SDB_STATUS_UPDATING = 2, - SDB_STATUS_DROPPING = 3, + SDB_STATUS_DROPPING = 2, + SDB_STATUS_DROPPED = 3, SDB_STATUS_READY = 4, - SDB_STATUS_DROPPED = 5 } ESdbStatus; typedef enum { diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 30d1bd0a518415d98b7358337c94834aa70716a2..04a24c4f3204cff586d8754fae5679f421707dce 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -40,6 +40,11 @@ enum { CTG_DBG_STB_RENT_NUM, }; +typedef enum { + AUTH_TYPE_READ = 1, + AUTH_TYPE_WRITE, + AUTH_TYPE_OTHER, +} AUTH_TYPE; typedef struct SCatalogReq { SArray *pTableName; // element is SNAME @@ -57,6 +62,7 @@ typedef struct SMetaData { typedef struct SCatalogCfg { uint32_t maxTblCacheNum; uint32_t maxDBCacheNum; + uint32_t maxUserCacheNum; uint32_t dbRentSec; uint32_t stbRentSec; } SCatalogCfg; @@ -77,6 +83,11 @@ typedef struct SDbVgVersion { int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT } SDbVgVersion; +typedef struct SUserAuthVersion { + char user[TSDB_USER_LEN]; + int32_t version; +} SUserAuthVersion; + typedef SDbCfgRsp SDbCfgInfo; typedef SUserIndexRsp SIndexInfo; @@ -219,12 +230,18 @@ int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stable int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); +int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num); + int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg); int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo); +int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass); + +int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); + /** * Destroy catalog and relase all resources diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 293e7ae52f55d72ff4b6af8166c8f84d6eb37f97..339743f153968a2ae6910ac68735bbf295925041 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -45,6 +45,7 @@ typedef struct SInputData { typedef struct SOutputData { int32_t numOfRows; + int32_t numOfCols; int8_t compressed; char* pData; bool queryEnd; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 0cd1a9265da36d1f65e0c312fdf06342e34b8cd5..ad57cbf4e4a9b5afb67a3954ed29c19546ea1064 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -33,6 +33,7 @@ typedef struct SReadHandle { void* meta; void* config; void* vnode; + void* mnd; } SReadHandle; #define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1 diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 094ce801062764cbe8e7e83a7cb1d52e21295c75..347303a051b29b48aedf154bcfd1cc2817fedb31 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -165,7 +165,7 @@ typedef struct SInputColumnInfoData { SColumnInfoData *pPTS; // primary timestamp column SColumnInfoData **pData; SColumnDataAgg **pColumnDataAgg; - uint64_t uid; // table uid + uint64_t uid; // table uid, used to set the tag value when building the final query result for selectivity functions. } SInputColumnInfoData; // sql function runtime context diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 0572262bfc99a2b64f381c6b148ef79fda6d4c0f..1fa9db992729f26aa855d045c3d574a9e222d500 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -40,6 +40,7 @@ typedef enum EFunctionType { FUNCTION_TYPE_STDDEV, FUNCTION_TYPE_SUM, FUNCTION_TYPE_TWA, + FUNCTION_TYPE_HISTOGRAM, // nonstandard SQL function FUNCTION_TYPE_BOTTOM = 500, @@ -112,6 +113,9 @@ typedef enum EFunctionType { FUNCTION_TYPE_WENDTS, FUNCTION_TYPE_WDURATION, + // internal function + FUNCTION_TYPE_SELECT_VALUE, + // user defined funcion FUNCTION_TYPE_UDF = 10000 } EFunctionType; @@ -140,6 +144,7 @@ bool fmIsScalarFunc(int32_t funcId); bool fmIsNonstandardSQLFunc(int32_t funcId); bool fmIsStringFunc(int32_t funcId); bool fmIsDatetimeFunc(int32_t funcId); +bool fmIsSelectFunc(int32_t funcId); bool fmIsTimelineFunc(int32_t funcId); bool fmIsTimeorderFunc(int32_t funcId); bool fmIsPseudoColumnFunc(int32_t funcId); diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index e49f5cac45d1563b620004f106fad51481483039..d59a7c23f7a077160278d5f19dcea28b48514ff0 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -22,13 +22,18 @@ #include "tmsg.h" #include "tcommon.h" #include "function.h" +#include "tdatablock.h" #ifdef __cplusplus extern "C" { #endif #define UDF_LISTEN_PIPE_NAME_LEN 32 -#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock." +#ifdef _WIN32 +#define UDF_LISTEN_PIPE_NAME_PREFIX "\\\\?\\pipe\\udfd.sock" +#else +#define UDF_LISTEN_PIPE_NAME_PREFIX ".udfd.sock." +#endif #define UDF_DNODE_ID_ENV_NAME "DNODE_ID" //====================================================================================== @@ -54,14 +59,14 @@ int32_t setupUdf(char udfName[], UdfcFuncHandle *handle); typedef struct SUdfColumnMeta { int16_t type; - int32_t bytes; // <0 var length, others fixed length bytes + int32_t bytes; uint8_t precision; uint8_t scale; } SUdfColumnMeta; typedef struct SUdfColumnData { int32_t numOfRows; - bool varLengthColumn; + int32_t rowsAlloc; union { struct { int32_t nullBitmapLen; @@ -72,9 +77,10 @@ typedef struct SUdfColumnData { struct { int32_t varOffsetsLen; - char *varOffsets; + int32_t *varOffsets; int32_t payloadLen; char *payload; + int32_t payloadAllocLen; } varLenCol; }; } SUdfColumnData; @@ -127,16 +133,119 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); // begin API to UDF writer. // dynamic lib init and destroy -typedef int32_t (*TUdfSetupFunc)(); -typedef int32_t (*TUdfTeardownFunc)(); +typedef int32_t (*TUdfInitFunc)(); +typedef int32_t (*TUdfDestroyFunc)(); //TODO: add API to check function arguments type, number etc. -//TODO: another way to manage memory is provide api for UDF to add data to SUdfColumnData and UDF framework will allocate memory. -// then UDF framework will free the memory -//typedef int32_t addFixedLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t colBytes, char* data); -//typedef int32_t addVariableLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t dataLen, char * data); -typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column); +#define UDF_MEMORY_EXP_GROWTH 1.5 + +static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn* pColumn, int32_t newCapacity) { + SUdfColumnMeta *meta = &pColumn->colMeta; + SUdfColumnData *data = &pColumn->colData; + + if (newCapacity== 0 || newCapacity <= data->rowsAlloc) { + return TSDB_CODE_SUCCESS; + } + + int allocCapacity = TMAX(data->rowsAlloc, 8); + while (allocCapacity < newCapacity) { + allocCapacity *= UDF_MEMORY_EXP_GROWTH; + } + + if (IS_VAR_DATA_TYPE(meta->type)) { + char* tmp = taosMemoryRealloc(data->varLenCol.varOffsets, sizeof(int32_t) * allocCapacity); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + data->varLenCol.varOffsets = (int32_t*)tmp; + data->varLenCol.varOffsetsLen = sizeof(int32_t) * allocCapacity; + // for payload, add data in udfColDataAppend + } else { + char* tmp = taosMemoryRealloc(data->fixLenCol.nullBitmap, BitmapLen(allocCapacity)); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + data->fixLenCol.nullBitmap = tmp; + data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity); + if (meta->type == TSDB_DATA_TYPE_NULL) { + return TSDB_CODE_SUCCESS; + } + + tmp = taosMemoryRealloc(data->fixLenCol.data, allocCapacity* meta->bytes); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + data->fixLenCol.data = tmp; + data->fixLenCol.dataLen = allocCapacity* meta->bytes; + } + + data->rowsAlloc = allocCapacity; + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE int32_t udfColSetRow(SUdfColumn* pColumn, uint32_t currentRow, const char* pData, bool isNull) { + SUdfColumnMeta *meta = &pColumn->colMeta; + SUdfColumnData *data = &pColumn->colData; + udfColEnsureCapacity(pColumn, currentRow+1); + bool isVarCol = IS_VAR_DATA_TYPE(meta->type); + if (isNull) { + if (isVarCol) { + data->varLenCol.varOffsets[currentRow] = -1; + } else { + colDataSetNull_f(data->fixLenCol.nullBitmap, currentRow); + } + } else { + if (!isVarCol) { + colDataSetNotNull_f(data->fixLenCol.nullBitmap, currentRow); + memcpy(data->fixLenCol.data + meta->bytes * currentRow, pData, meta->bytes); + } else { + int32_t dataLen = varDataTLen(pData); + if (meta->type == TSDB_DATA_TYPE_JSON) { + if (*pData == TSDB_DATA_TYPE_NULL) { + dataLen = 0; + } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData + CHAR_BYTES); + } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + dataLen = LONG_BYTES; + } else if (*pData == TSDB_DATA_TYPE_BOOL) { + dataLen = CHAR_BYTES; + } + dataLen += CHAR_BYTES; + } + + if (data->varLenCol.payloadAllocLen < data->varLenCol.payloadLen + dataLen) { + uint32_t newSize = data->varLenCol.payloadAllocLen; + if (newSize <= 1) { + newSize = 8; + } + + while (newSize < data->varLenCol.payloadLen + dataLen) { + newSize = newSize * UDF_MEMORY_EXP_GROWTH; + } + + char *buf = taosMemoryRealloc(data->varLenCol.payload, newSize); + if (buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + data->varLenCol.payload = buf; + data->varLenCol.payloadAllocLen = newSize; + } + + uint32_t len = data->varLenCol.payloadLen; + data->varLenCol.varOffsets[currentRow] = len; + + memcpy(data->varLenCol.payload + len, pData, dataLen); + data->varLenCol.payloadLen += dataLen; + } + } + data->numOfRows = TMAX(currentRow + 1, data->numOfRows); + return 0; +} + typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol); typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf); diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 6f77e8edb69641aec4f2e2799e35da1bad8c49a2..115bb4349d14c4505cc6000dae5b3237036640f5 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -22,19 +22,20 @@ extern "C" { #include "tdef.h" -#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) +#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) #define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type)) #define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0) -#define FOREACH(node, list) \ - for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) +#define FOREACH(node, list) \ + for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \ + (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) #define REPLACE_NODE(newNode) cell->pNode = (SNode*)(newNode) #define INSERT_LIST(target, src) nodesListInsertList((target), cell, src) -#define WHERE_EACH(node, list) \ +#define WHERE_EACH(node, list) \ SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \ while (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)) @@ -43,16 +44,26 @@ extern "C" { // only be use in WHERE_EACH #define ERASE_NODE(list) cell = nodesListErase((list), cell) -#define FORBOTH(node1, list1, node2, list2) \ - for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \ - (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ - cell1 = cell1->pNext, cell2 = cell2->pNext) +#define FORBOTH(node1, list1, node2, list2) \ + for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), \ + *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \ + (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), \ + (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), \ + (node1 != NULL && node2 != NULL); \ + cell1 = cell1->pNext, cell2 = cell2->pNext) #define REPLACE_LIST1_NODE(newNode) cell1->pNode = (SNode*)(newNode) #define REPLACE_LIST2_NODE(newNode) cell2->pNode = (SNode*)(newNode) -#define FOREACH_FOR_REWRITE(node, list) \ - for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext) +#define FOREACH_FOR_REWRITE(node, list) \ + for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \ + (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext) + +#define DESTORY_LIST(list) \ + do { \ + nodesDestroyList(list); \ + list = NULL; \ + } while (0) typedef enum ENodeType { // Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN, @@ -174,6 +185,7 @@ typedef enum ENodeType { QUERY_NODE_LOGIC_PLAN_VNODE_MODIF, QUERY_NODE_LOGIC_PLAN_EXCHANGE, QUERY_NODE_LOGIC_PLAN_WINDOW, + QUERY_NODE_LOGIC_PLAN_FILL, QUERY_NODE_LOGIC_PLAN_SORT, QUERY_NODE_LOGIC_PLAN_PARTITION, QUERY_NODE_LOGIC_SUBPLAN, @@ -191,6 +203,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_FILL, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW, QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW, QUERY_NODE_PHYSICAL_PLAN_PARTITION, @@ -202,7 +215,7 @@ typedef enum ENodeType { /** * The first field of a node of any type is guaranteed to be the ENodeType. - * Hence the type of any node can be gotten by casting it to SNode. + * Hence the type of any node can be gotten by casting it to SNode. */ typedef struct SNode { ENodeType type; @@ -211,41 +224,36 @@ typedef struct SNode { typedef struct SListCell { struct SListCell* pPrev; struct SListCell* pNext; - SNode* pNode; + SNode* pNode; } SListCell; typedef struct SNodeList { - int32_t length; + int32_t length; SListCell* pHead; SListCell* pTail; } SNodeList; -#define SNodeptr void* +#define SNodeptr void* SNodeptr nodesMakeNode(ENodeType type); -void nodesDestroyNode(SNodeptr pNode); +void nodesDestroyNode(SNodeptr pNode); SNodeList* nodesMakeList(); -int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode); -int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode); -int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode); -int32_t nodesListMakeStrictAppend(SNodeList** pList, SNodeptr pNode); -int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc); -int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc); -int32_t nodesListPushFront(SNodeList* pList, SNodeptr pNode); +int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode); +int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode); +int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode); +int32_t nodesListMakeStrictAppend(SNodeList** pList, SNodeptr pNode); +int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc); +int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc); +int32_t nodesListPushFront(SNodeList* pList, SNodeptr pNode); SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); -void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc); -SNodeptr nodesListGetNode(SNodeList* pList, int32_t index); -void nodesDestroyList(SNodeList* pList); +void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc); +SNodeptr nodesListGetNode(SNodeList* pList, int32_t index); +void nodesDestroyList(SNodeList* pList); // Only clear the linked list structure, without releasing the elements inside void nodesClearList(SNodeList* pList); -typedef enum EDealRes { - DEAL_RES_CONTINUE = 1, - DEAL_RES_IGNORE_CHILD, - DEAL_RES_ERROR, - DEAL_RES_END -} EDealRes; +typedef enum EDealRes { DEAL_RES_CONTINUE = 1, DEAL_RES_IGNORE_CHILD, DEAL_RES_ERROR, DEAL_RES_END } EDealRes; typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext); void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext); @@ -261,18 +269,18 @@ void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* bool nodesEqualNode(const SNodeptr a, const SNodeptr b); -SNodeptr nodesCloneNode(const SNodeptr pNode); +SNodeptr nodesCloneNode(const SNodeptr pNode); SNodeList* nodesCloneList(const SNodeList* pList); const char* nodesNodeName(ENodeType type); -int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen); -int32_t nodesStringToNode(const char* pStr, SNode** pNode); +int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen); +int32_t nodesStringToNode(const char* pStr, SNode** pNode); int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen); int32_t nodesStringToList(const char* pStr, SNodeList** pList); -int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len); -char *nodesGetNameFromColumnNode(SNode *pNode); +int32_t nodesNodeToSQL(SNode* pNode, char* buf, int32_t bufSize, int32_t* len); +char* nodesGetNameFromColumnNode(SNode* pNode); int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots); #ifdef __cplusplus diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index a34d4e3c80a75e29c5fb6e81125c98bcd5fec56f..a756217c7a95a94a9bdb1af762c1fb4c8ee67e33 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -102,7 +102,6 @@ typedef struct SWindowLogicNode { int64_t sliding; int8_t intervalUnit; int8_t slidingUnit; - SFillNode* pFill; int64_t sessionGap; SNode* pTspk; SNode* pStateExpr; @@ -110,6 +109,14 @@ typedef struct SWindowLogicNode { int64_t watermark; } SWindowLogicNode; +typedef struct SFillLogicNode { + SLogicNode node; + EFillMode mode; + SNode* pWStartTs; + SNode* pValues; // SNodeListNode + STimeWindow timeRange; +} SFillLogicNode; + typedef struct SSortLogicNode { SLogicNode node; SNodeList* pSortKeys; @@ -223,10 +230,12 @@ typedef struct SProjectPhysiNode { typedef struct SJoinPhysiNode { SPhysiNode node; EJoinType joinType; - SNode* pOnConditions; // in or out tuple ? + SNode* pOnConditions; SNodeList* pTargets; } SJoinPhysiNode; +typedef SJoinPhysiNode SSortMergeJoinPhysiNode; + typedef struct SAggPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function @@ -263,9 +272,17 @@ typedef struct SIntervalPhysiNode { int64_t sliding; int8_t intervalUnit; int8_t slidingUnit; - SFillNode* pFill; } SIntervalPhysiNode; +typedef struct SFillPhysiNode { + SPhysiNode node; + EFillMode mode; + SNode* pWStartTs; // SColumnNode + SNode* pValues; // SNodeListNode + SNodeList* pTargets; + STimeWindow timeRange; +} SFillPhysiNode; + typedef struct SMultiTableIntervalPhysiNode { SIntervalPhysiNode interval; SNodeList* pPartitionKeys; @@ -340,7 +357,7 @@ typedef struct SQueryPlan { int32_t numOfSubplans; SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. SExplainInfo explainInfo; - SArray* pPlaceholderValues; + SArray* pPlaceholderValues; } SQueryPlan; void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 238c4c85384393225390e8a31eeb0b4ab80e554e..91c80f6cf52db5c99fad8e2b413a39face2f3515 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -88,7 +88,8 @@ typedef struct SValueNode { double d; char* p; } datum; - char unit; + int64_t typeData; + char unit; } SValueNode; typedef struct SOperatorNode { @@ -208,9 +209,11 @@ typedef enum EFillMode { } EFillMode; typedef struct SFillNode { - ENodeType type; // QUERY_NODE_FILL - EFillMode mode; - SNode* pValues; // SNodeListNode + ENodeType type; // QUERY_NODE_FILL + EFillMode mode; + SNode* pValues; // SNodeListNode + SNode* pWStartTs; // _wstartts pseudo column + STimeWindow timeRange; } SFillNode; typedef struct SSelectStmt { @@ -230,6 +233,7 @@ typedef struct SSelectStmt { uint8_t precision; bool isEmptyResult; bool hasAggFuncs; + bool isTimeOrderQuery; } SSelectStmt; typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType; @@ -294,12 +298,13 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext); typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType; - int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type, SNodeList** pCols); typedef bool (*FFuncClassifier)(int32_t funcId); -int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs); +int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifier classifier, SNodeList** pFuncs); + +int32_t nodesCollectSpecialNodes(SSelectStmt* pSelect, ESqlClause clause, ENodeType type, SNodeList** pNodes); bool nodesIsExprNode(const SNode* pNode); @@ -311,10 +316,11 @@ bool nodesIsJsonOp(const SOperatorNode* pOp); bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); -void* nodesGetValueFromNode(SValueNode* pNode); -char* nodesGetStrValueFromNode(SValueNode* pNode); -char* getFillModeString(EFillMode mode); -void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal); +void* nodesGetValueFromNode(SValueNode* pNode); +int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value); +char* nodesGetStrValueFromNode(SValueNode* pNode); +char* getFillModeString(EFillMode mode); +void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal); #ifdef __cplusplus } diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index de71192a9bdb928c3c9dfed061d8fca972518731..84fa1668817dff49d428e0cf39909b14e9ed3d75 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -20,8 +20,8 @@ extern "C" { #endif -#include "querynodes.h" #include "query.h" +#include "querynodes.h" typedef struct SStmtCallback { TAOS_STMT* pStmt; @@ -34,24 +34,26 @@ typedef struct SStmtCallback { typedef struct SParseContext { uint64_t requestId; int32_t acctId; - const char *db; + const char* db; bool topicQuery; - void *pTransporter; + void* pTransporter; SEpSet mgmtEpSet; - const char *pSql; // sql string - size_t sqlLen; // length of the sql string - char *pMsg; // extended error message if exists to help identifying the problem in sql statement. - int32_t msgLen; // max length of the msg - struct SCatalog *pCatalog; - SStmtCallback *pStmtCb; + const char* pSql; // sql string + size_t sqlLen; // length of the sql string + char* pMsg; // extended error message if exists to help identifying the problem in sql statement. + int32_t msgLen; // max length of the msg + struct SCatalog* pCatalog; + SStmtCallback* pStmtCb; + const char* pUser; + bool isSuperUser; } SParseContext; typedef struct SCmdMsgInfo { int16_t msgType; - SEpSet epSet; - void* pMsg; + SEpSet epSet; + void* pMsg; int32_t msgLen; - void* pExtension; // todo remove it soon + void* pExtension; // todo remove it soon } SCmdMsgInfo; typedef enum EQueryExecMode { @@ -63,21 +65,21 @@ typedef enum EQueryExecMode { typedef struct SQuery { EQueryExecMode execMode; - bool haveResultSet; - SNode* pRoot; - int32_t numOfResCols; - SSchema* pResSchema; - int8_t precision; - SCmdMsgInfo* pCmdMsg; - int32_t msgType; - SArray* pDbList; - SArray* pTableList; - bool showRewrite; - int32_t placeholderNum; + bool haveResultSet; + SNode* pRoot; + int32_t numOfResCols; + SSchema* pResSchema; + int8_t precision; + SCmdMsgInfo* pCmdMsg; + int32_t msgType; + SArray* pDbList; + SArray* pTableList; + bool showRewrite; + int32_t placeholderNum; } SQuery; int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); -bool isInsertSql(const char* pStr, size_t length); +bool isInsertSql(const char* pStr, size_t length); void qDestroyQuery(SQuery* pQueryNode); @@ -89,14 +91,16 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); void qFreeStmtDataBlock(void* pDataBlock); int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc); void qDestroyStmtDataBlock(void* pBlock); -int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen); -int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum); -int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields); -int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields); -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen); -void destroyBoundColumnInfo(void* pBoundInfo); -int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen); - +int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); +int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, + int32_t rowNum); +int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD** fields); +int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD** fields); +int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_MULTI_BIND* bind, + char* msgBuf, int32_t msgBufLen); +void destroyBoundColumnInfo(void* pBoundInfo); +int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, + int32_t msgBufLen); #ifdef __cplusplus } diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 0c7db45c4b43479db85ab43876697f9ab3baae9f..555274599a1028c0a9dd875f9bd16cd438cc5f54 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -91,6 +91,8 @@ int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + #ifdef __cplusplus } #endif diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index eadc90138919137a80c8110563d59bbe21a11b50..42180833df63fde955339d42bad3f21b4464490c 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -35,19 +35,6 @@ enum { STREAM_CREATED_BY__SMA, }; -#if 0 -// pipe -> fetch/pipe queue -// merge -> merge queue -// write -> write queue -enum { - TASK_DISPATCH_MSG__SND_PIPE = 1, - TASK_DISPATCH_MSG__SND_MERGE, - TASK_DISPATCH_MSG__VND_PIPE, - TASK_DISPATCH_MSG__VND_MERGE, - TASK_DISPATCH_MSG__VND_WRITE, -}; -#endif - typedef struct { int32_t nodeId; // 0 for snode SEpSet epSet; @@ -83,8 +70,10 @@ typedef struct { } STaskDispatcherShuffle; typedef struct { - int8_t reserved; + int8_t reserved; + SSchemaWrapper* pSchemaWrapper; // not applicable to encoder and decoder + STSchema* pTSchema; SHashObj* pHash; // groupId to tbuid } STaskSinkTb; @@ -100,10 +89,6 @@ typedef struct { int8_t reserved; } STaskSinkFetch; -typedef struct { - int8_t reserved; -} STaskSinkShow; - enum { TASK_SOURCE__SCAN = 1, TASK_SOURCE__PIPE, @@ -128,7 +113,6 @@ enum { TASK_SINK__TABLE, TASK_SINK__SMA, TASK_SINK__FETCH, - TASK_SINK__SHOW, }; typedef struct { @@ -155,7 +139,6 @@ typedef struct { STaskSinkTb tbSink; STaskSinkSma smaSink; STaskSinkFetch fetchSink; - STaskSinkShow showSink; }; // dispatch diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index ffe435a8b28c3b09007eb12b58b74b00be934db0..ca10465ed40962c5138839004f3301f4f9f5ce18 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -89,7 +89,7 @@ typedef struct SSyncFSM { struct SSyncRaftEntry; typedef struct SSyncRaftEntry SSyncRaftEntry; -#define SYNC_INDEX_BEGIN 0 +#define SYNC_INDEX_BEGIN 0 #define SYNC_INDEX_INVALID -1 // abstract definition of log store in raft @@ -149,6 +149,8 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); ESyncState syncGetMyRole(int64_t rid); const char* syncGetMyRoleStr(int64_t rid); SyncTerm syncGetMyTerm(int64_t rid); +void syncGetEpSet(int64_t rid, SEpSet* pEpSet); +int32_t syncGetVgId(int64_t rid); typedef enum { TAOS_SYNC_PROPOSE_SUCCESS = 0, diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 0e7d486eab698db56f3d5ca3191ac7cba3bf37a9..a7d1522d12d1e6b773b197b99adf0d1a9beb5a24 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -68,6 +68,7 @@ typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, cha typedef bool (*RpcRfp)(int32_t code); typedef struct SRpcInit { + char localFqdn[TSDB_FQDN_LEN]; uint16_t localPort; // local port char * label; // for debug purpose int numOfThreads; // number of threads to handle connections diff --git a/include/os/osDef.h b/include/os/osDef.h index 8da0c2c00cc7713d2d5dcdeda1b60634ea89b236..6f6199de7af6a09d7be1b4b60009f2d7507f2d46 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -62,7 +62,7 @@ extern "C" { #define strncasecmp _strnicmp #define wcsncasecmp _wcsnicmp #define strtok_r strtok_s - #define snprintf _snprintf + // #define snprintf _snprintf #define in_addr_t unsigned long // #define socklen_t int diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 62c3771669c64b0b08c7287bb4c657608c5c4558..213a6930eebb4fe7a97bf45f1f5db80a5d026a22 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -161,6 +161,7 @@ int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec); TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +bool taosValidIpAndPort(uint32_t ip, uint16_t port); TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port); int32_t taosKeepTcpAlive(TdSocketPtr pSocket); TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen); diff --git a/include/os/osString.h b/include/os/osString.h index a4100652c340e5a48446c9d97fb5d9bd44acb83e..026cb33ad9ddf2e8f18ac84bd8f2779c81701c57 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -59,6 +59,8 @@ bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); bool taosValidateEncodec(const char *encodec); +int32_t taosHexEncode(const char *src, char *dst, int32_t len); +int32_t taosHexDecode(const char *src, char *dst, int32_t len); int32_t taosWcharWidth(TdWchar wchar); int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index cb949468ebe7f1cc998ff9f24e9472d87bb0d3db..441e87eff7bb5cac0222c498228acce660bb2fea 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -87,6 +87,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0113) #define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0114) #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x0115) +#define TSDB_CODE_DUP_KEY TAOS_DEF_ERROR_CODE(0, 0x0116) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0140) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0141) @@ -244,9 +245,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x03AC) #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AD) #define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AE) +#define TSDB_CODE_MND_SINGLE_STB_MODE_DB TAOS_DEF_ERROR_CODE(0, 0x03B0) // mnode-infoSchema -#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x03B0) +#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x03BA) // mnode-func #define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C0) @@ -262,7 +264,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) #define TSDB_CODE_MND_TRANS_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D1) #define TSDB_CODE_MND_TRANS_INVALID_STAGE TAOS_DEF_ERROR_CODE(0, 0x03D2) -#define TSDB_CODE_MND_TRANS_CANT_PARALLEL TAOS_DEF_ERROR_CODE(0, 0x03D4) +#define TSDB_CODE_MND_TRANS_CONFLICT TAOS_DEF_ERROR_CODE(0, 0x03D3) +#define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03D4) // mnode-mq #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0) @@ -317,6 +320,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_TB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0515) #define TSDB_CODE_VND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0516) #define TSDB_CODE_VND_HASH_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x0517) +#define TSDB_CODE_VND_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0518) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -615,9 +619,21 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_OFFSET_LESS_ZERO TAOS_DEF_ERROR_CODE(0, 0x2637) #define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY TAOS_DEF_ERROR_CODE(0, 0x2638) #define TSDB_CODE_PAR_INVALID_TOPIC_QUERY TAOS_DEF_ERROR_CODE(0, 0x2639) +#define TSDB_CODE_PAR_INVALID_DROP_STABLE TAOS_DEF_ERROR_CODE(0, 0x263A) +#define TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE TAOS_DEF_ERROR_CODE(0, 0x263B) +#define TSDB_CODE_PAR_DUPLICATED_COLUMN TAOS_DEF_ERROR_CODE(0, 0x263C) +#define TSDB_CODE_PAR_INVALID_TAGS_LENGTH TAOS_DEF_ERROR_CODE(0, 0x263D) +#define TSDB_CODE_PAR_INVALID_ROW_LENGTH TAOS_DEF_ERROR_CODE(0, 0x263E) +#define TSDB_CODE_PAR_INVALID_COLUMNS_NUM TAOS_DEF_ERROR_CODE(0, 0x263F) +#define TSDB_CODE_PAR_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x2640) +#define TSDB_CODE_PAR_INVALID_FIRST_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2641) +#define TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN TAOS_DEF_ERROR_CODE(0, 0x2642) +#define TSDB_CODE_PAR_INVALID_TAGS_NUM TAOS_DEF_ERROR_CODE(0, 0x2643) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) +#define TSDB_CODE_PLAN_EXPECTED_TS_EQUAL TAOS_DEF_ERROR_CODE(0, 0x2701) +#define TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN TAOS_DEF_ERROR_CODE(0, 0x2702) //function #define TSDB_CODE_FUNC_FUNTION_ERROR TAOS_DEF_ERROR_CODE(0, 0x2800) diff --git a/include/util/tdef.h b/include/util/tdef.h index bc8b4b6e85d83566f0b464017c63090e880a27f9..316640797be355e053117a3e8db7b3de5009fce6 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -97,43 +97,6 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_TIME_PRECISION_MICRO_DIGITS 16 #define TSDB_TIME_PRECISION_NANO_DIGITS 19 -#define TSDB_INFORMATION_SCHEMA_DB "information_schema" -#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" -#define TSDB_INS_TABLE_DNODES "dnodes" -#define TSDB_INS_TABLE_MNODES "mnodes" -#define TSDB_INS_TABLE_MODULES "modules" -#define TSDB_INS_TABLE_QNODES "qnodes" -#define TSDB_INS_TABLE_BNODES "bnodes" -#define TSDB_INS_TABLE_SNODES "snodes" -#define TSDB_INS_TABLE_CLUSTER "cluster" -#define TSDB_INS_TABLE_USER_DATABASES "user_databases" -#define TSDB_INS_TABLE_USER_FUNCTIONS "user_functions" -#define TSDB_INS_TABLE_USER_INDEXES "user_indexes" -#define TSDB_INS_TABLE_USER_STABLES "user_stables" -#define TSDB_INS_TABLE_USER_STREAMS "user_streams" -#define TSDB_INS_TABLE_USER_TABLES "user_tables" -#define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed" -#define TSDB_INS_TABLE_USER_USERS "user_users" -#define TSDB_INS_TABLE_LICENCES "grants" -#define TSDB_INS_TABLE_VGROUPS "vgroups" -#define TSDB_INS_TABLE_CONSUMERS "consumers" -#define TSDB_INS_TABLE_SUBSCRIBES "subscribes" -#define TSDB_INS_TABLE_TRANS "trans" -#define TSDB_INS_TABLE_SMAS "smas" -#define TSDB_INS_TABLE_CONFIGS "configs" -#define TSDB_INS_TABLE_CONNS "connections" -#define TSDB_INS_TABLE_QUERIES "queries" -#define TSDB_INS_TABLE_VNODES "vnodes" - -#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" -#define TSDB_PERFS_TABLE_CONNECTIONS "connections" -#define TSDB_PERFS_TABLE_QUERIES "queries" -#define TSDB_PERFS_TABLE_TOPICS "topics" -#define TSDB_PERFS_TABLE_CONSUMERS "consumers" -#define TSDB_PERFS_TABLE_SUBSCRIPTIONS "subscriptions" -#define TSDB_PERFS_TABLE_OFFSETS "offsets" -#define TSDB_PERFS_TABLE_STREAMS "streams" - #define TSDB_INDEX_TYPE_SMA "SMA" #define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT" @@ -205,16 +168,6 @@ typedef enum ELogicConditionType { LOGIC_COND_TYPE_NOT, } ELogicConditionType; -#define FUNCTION_CEIL 4500 -#define FUNCTION_FLOOR 4501 -#define FUNCTION_ABS 4502 -#define FUNCTION_ROUND 4503 - -#define FUNCTION_LENGTH 4800 -#define FUNCTION_CONCAT 4801 -#define FUNCTION_LTRIM 4802 -#define FUNCTION_RTRIM 4803 - #define TSDB_NAME_DELIMITER_LEN 1 #define TSDB_UNI_LEN 24 @@ -327,15 +280,21 @@ typedef enum ELogicConditionType { #define TSDB_MIN_VNODES_PER_DB 1 #define TSDB_MAX_VNODES_PER_DB 4096 #define TSDB_DEFAULT_VN_PER_DB 2 -#define TSDB_MIN_CACHE_BLOCK_SIZE 1 -#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode -#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 -#define TSDB_MIN_TOTAL_BLOCKS 3 -#define TSDB_MAX_TOTAL_BLOCKS 10000 -#define TSDB_DEFAULT_TOTAL_BLOCKS 6 +#define TSDB_MIN_BUFFER_PER_VNODE 3 // unit MB +#define TSDB_MAX_BUFFER_PER_VNODE 16384 // unit MB +#define TSDB_DEFAULT_BUFFER_PER_VNODE 96 +#define TSDB_MIN_PAGES_PER_VNODE 64 +#define TSDB_MAX_PAGES_PER_VNODE 16384 +#define TSDB_DEFAULT_PAGES_PER_VNODE 256 +#define TSDB_MIN_PAGESIZE_PER_VNODE 1 // unit KB +#define TSDB_MAX_PAGESIZE_PER_VNODE 16384 +#define TSDB_DEFAULT_PAGESIZE_PER_VNODE 4 #define TSDB_MIN_DAYS_PER_FILE 60 // unit minute #define TSDB_MAX_DAYS_PER_FILE (3650 * 1440) #define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440) +#define TSDB_MIN_DURATION_PER_FILE 60 // unit minute +#define TSDB_MAX_DURATION_PER_FILE (3650 * 1440) +#define TSDB_DEFAULT_DURATION_PER_FILE (10 * 1440) #define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute #define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. #define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years @@ -345,9 +304,6 @@ typedef enum ELogicConditionType { #define TSDB_MIN_MAXROWS_FBLOCK 200 #define TSDB_MAX_MAXROWS_FBLOCK 10000 #define TSDB_DEFAULT_MAXROWS_FBLOCK 4096 -#define TSDB_MIN_COMMIT_TIME 30 -#define TSDB_MAX_COMMIT_TIME 40960 -#define TSDB_DEFAULT_COMMIT_TIME 3600 #define TSDB_MIN_FSYNC_PERIOD 0 #define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond #define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second @@ -366,9 +322,6 @@ typedef enum ELogicConditionType { #define TSDB_DB_STRICT_OFF 0 #define TSDB_DB_STRICT_ON 1 #define TSDB_DEFAULT_DB_STRICT 0 -#define TSDB_MIN_DB_UPDATE 0 -#define TSDB_MAX_DB_UPDATE 2 -#define TSDB_DEFAULT_DB_UPDATE 0 #define TSDB_MIN_DB_CACHE_LAST_ROW 0 #define TSDB_MAX_DB_CACHE_LAST_ROW 3 #define TSDB_DEFAULT_CACHE_LAST_ROW 0 @@ -378,13 +331,6 @@ typedef enum ELogicConditionType { #define TSDB_DB_SINGLE_STABLE_ON 0 #define TSDB_DB_SINGLE_STABLE_OFF 1 #define TSDB_DEFAULT_DB_SINGLE_STABLE 0 -#define TSDB_MIN_BUFFER_PER_VNODE 3 // unit MB -#define TSDB_DEFAULT_BUFFER_PER_VNODE 96 -#define TSDB_MIN_PAGES_PER_VNODE 64 -#define TSDB_DEFAULT_PAGES_PER_VNODE 256 -#define TSDB_MIN_PAGESIZE_PER_VNODE 1 // unit KB -#define TSDB_MAX_PAGESIZE_PER_VNODE 16384 -#define TSDB_DEFAULT_PAGESIZE_PER_VNODE 4 #define TSDB_MIN_ROLLUP_FILE_FACTOR 0 #define TSDB_MAX_ROLLUP_FILE_FACTOR 1 @@ -424,21 +370,9 @@ typedef enum ELogicConditionType { * 1. ordinary sub query for select * from super_table * 2. all sqlobj generated by createSubqueryObj with this flag */ -#define TSDB_QUERY_TYPE_SUBQUERY 0x02u -#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table - -#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side -#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table -#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query -#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query -#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage - -#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u #define TSDB_QUERY_TYPE_INSERT 0x100u // insert type -#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u #define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file #define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type -#define TSDB_QUERY_TYPE_NEST_SUBQUERY 0x1000u // nested sub query #define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) #define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) diff --git a/include/util/tencode.h b/include/util/tencode.h index c5066996f3ffb7091f3bf396b625c6ba50ce0ec0..b081d1a157369ad244fed870b126901f1009374a 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -79,17 +79,6 @@ typedef struct { #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) -// #define TCODER_MALLOC(PCODER, SIZE) \ -// ({ \ -// void* ptr = NULL; \ -// SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + (SIZE)); \ -// if (pMem) { \ -// pMem->next = (PCODER)->mList; \ -// (PCODER)->mList = pMem; \ -// ptr = (void*)&pMem[1]; \ -// } \ -// ptr; \ -// }) static FORCE_INLINE void* tCoderMalloc(SCoder* pCoder, int32_t size) { void* ptr = NULL; SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(SCoderMem*) + size); @@ -102,8 +91,9 @@ static FORCE_INLINE void* tCoderMalloc(SCoder* pCoder, int32_t size) { } #define tEncodeSize(E, S, SIZE, RET) \ - do{ \ + do { \ SCoder coder = {0}; \ + RET = 0; \ tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \ if ((E)(&coder, S) == 0) { \ SIZE = coder.pos; \ @@ -111,7 +101,7 @@ static FORCE_INLINE void* tCoderMalloc(SCoder* pCoder, int32_t size) { RET = -1; \ } \ tCoderClear(&coder); \ - }while(0) + } while (0) // #define tEncodeSize(E, S, SIZE) \ // ({ \ // SCoder coder = {0}; \ diff --git a/include/util/tskiplist.h b/include/util/tskiplist.h index a2382ad541e104430ad7334b458a8d50a5fb5284..eeae1b47daf2ff62511dfe1329ef55e62ed916be 100644 --- a/include/util/tskiplist.h +++ b/include/util/tskiplist.h @@ -52,72 +52,21 @@ typedef struct SSkipListNode { #define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] #define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)] -/* - * @version 0.3 - * @date 2017/11/12 - * the simple version of skip list. - * - * for multi-thread safe purpose, we employ TdThreadRwlock to guarantee to generate - * deterministic result. Later, we will remove the lock in SkipList to further enhance the performance. - * In this case, one should use the concurrent skip list (by using michael-scott algorithm) instead of - * this simple version in a multi-thread environment, to achieve higher performance of read/write operations. - * - * Note: Duplicated primary key situation. - * In case of duplicated primary key, two ways can be employed to handle this situation: - * 1. add as normal insertion without special process. - * 2. add an overflow pointer at each list node, all nodes with the same key will be added in the overflow pointer. - * In this case, the total steps of each search will be reduced significantly. - * Currently, we implement the skip list in a line with the first means, maybe refactor it soon. - * - * Memory consumption: the memory alignment causes many memory wasted. So, employ a memory - * pool will significantly reduce the total memory consumption, as well as the calloc/malloc operation costs. - * - */ - -// state struct, record following information: -// number of links in each level. -// avg search steps, for latest 1000 queries -// avg search rsp time, for latest 1000 queries -// total memory size -typedef struct tSkipListState { - // in bytes, sizeof(SSkipList)+sizeof(SSkipListNode)*SSkipList->nSize - uint64_t nTotalMemSize; - uint64_t nLevelNodeCnt[MAX_SKIP_LIST_LEVEL]; - uint64_t queryCount; // total query count - - /* - * only record latest 1000 queries - * when the value==1000, = 0, - * nTotalStepsForQueries = 0, - * nTotalElapsedTimeForQueries = 0 - */ - uint64_t nRecQueries; - uint16_t nTotalStepsForQueries; - uint64_t nTotalElapsedTimeForQueries; - - uint16_t nInsertObjs; - uint16_t nTotalStepsForInsert; - uint64_t nTotalElapsedTimeForInsert; -} tSkipListState; - typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutSkipOne = 2 } SSkipListPutStatus; typedef struct SSkipList { - uint32_t seed; - __compar_fn_t comparFn; - __sl_key_fn_t keyFn; - TdThreadRwlock *lock; - uint16_t len; - uint8_t maxLevel; - uint8_t flags; - uint8_t type; // static info above - uint8_t level; - uint32_t size; - SSkipListNode *pHead; // point to the first element - SSkipListNode *pTail; // point to the last element -#if SKIP_LIST_RECORD_PERFORMANCE - tSkipListState state; // skiplist state -#endif + uint32_t seed; + __compar_fn_t comparFn; + __sl_key_fn_t keyFn; + TdThreadRwlock *lock; + uint16_t len; + uint8_t maxLevel; + uint8_t flags; + uint8_t type; // static info above + uint8_t level; + uint32_t size; + SSkipListNode *pHead; // point to the first element + SSkipListNode *pTail; // point to the last element tGenericSavedFunc *insertHandleFn; } SSkipList; diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h new file mode 100644 index 0000000000000000000000000000000000000000..83e6ad08681e0b6177c1e060d9b02e71517b30da --- /dev/null +++ b/include/util/tskiplist2.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef _TD_UTIL_SKIPLIST2_H_ +#define _TD_UTIL_SKIPLIST2_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define SL_MAX_LEVEL 15 + +typedef struct SSkipList2 SSkipList2; +typedef struct SSLCursor SSLCursor; +typedef struct SSLCfg SSLCfg; +typedef struct SSLNode SSLNode; + +typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2); + +// SSkipList2 +int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl); +int32_t slClose(SSkipList2 *pSl); +int32_t slClear(SSkipList2 *pSl); + +// SSLCursor +int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc); +int32_t slcClose(SSLCursor *pSlc); +int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey); +int32_t slcMoveToNext(SSLCursor *pSlc); +int32_t slcMoveToPrev(SSLCursor *pSlc); +int32_t slcMoveToFirst(SSLCursor *pSlc); +int32_t slcMoveToLast(SSLCursor *pSlc); +int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData); +int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData); +int32_t slcDrop(SSLCursor *pSlc); + +// struct +struct SSLCfg { + int8_t maxLevel; + int32_t nKey; + int32_t nData; + tslCmprFn cmprFn; + void *pPool; + void *(*xMalloc)(void *, int32_t size); + void (*xFree)(void *, void *); +}; + +struct SSLCursor { + SSkipList2 *pSl; + SSLNode **forwards[SL_MAX_LEVEL]; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_SKIPLIST2_H_*/ \ No newline at end of file diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index ab27d7f00c4a1c1d55b67240d7d7e641f69e3e6e..43e886faf54c3f3b92fd604369d0e0286dab6133 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -46,11 +46,12 @@ typedef struct SStmtTableCache { void* boundTags; } SStmtTableCache; -typedef struct SQueryFields { +typedef struct SStmtQueryResInfo { TAOS_FIELD* fields; TAOS_FIELD* userFields; uint32_t numOfCols; -} SQueryFields; + int32_t precision; +} SStmtQueryResInfo; typedef struct SStmtBindInfo { bool needParse; @@ -72,17 +73,17 @@ typedef struct SStmtExecInfo { } SStmtExecInfo; typedef struct SStmtSQLInfo { - STMT_TYPE type; - STMT_STATUS status; - bool autoCreate; - uint64_t runTimes; - SHashObj* pTableCache; //SHash - SQuery* pQuery; - char* sqlStr; - int32_t sqlLen; - SArray* nodeList; - SQueryPlan* pQueryPlan; - SQueryFields fields; + STMT_TYPE type; + STMT_STATUS status; + bool autoCreate; + uint64_t runTimes; + SHashObj* pTableCache; //SHash + SQuery* pQuery; + char* sqlStr; + int32_t sqlLen; + SArray* nodeList; + SQueryPlan* pQueryPlan; + SStmtQueryResInfo queryRes; } SStmtSQLInfo; typedef struct STscStmt { diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index b11a49fa1aadbd1fd0be04f3e11ad230c150f62e..fc39e80c1e783914518d0b931f20310aad73df18 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -28,6 +28,27 @@ static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } +static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { + int32_t code = 0; + + SUserAuthBatchRsp batchRsp = {0}; + if (tDeserializeSUserAuthBatchRsp(value, valueLen, &batchRsp) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int32_t numOfBatchs = taosArrayGetSize(batchRsp.pArray); + for (int32_t i = 0; i < numOfBatchs; ++i) { + SGetUserAuthRsp *rsp = taosArrayGet(batchRsp.pArray, i); + tscDebug("hb user auth rsp, user:%s, version:%d", rsp->user, rsp->version); + + catalogUpdateUserAuthInfo(pCatalog, rsp); + } + + tFreeSUserAuthBatchRsp(&batchRsp); + return TSDB_CODE_SUCCESS; +} + static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; @@ -148,6 +169,24 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { for (int32_t i = 0; i < kvNum; ++i) { SKv *kv = taosArrayGet(pRsp->info, i); switch (kv->key) { + case HEARTBEAT_KEY_USER_AUTHINFO: { + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid hb user auth info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + + int64_t *clusterId = (int64_t *)info->param; + struct SCatalog *pCatalog = NULL; + + int32_t code = catalogGetHandle(*clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); + break; + } + + hbProcessUserAuthInfoRsp(kv->value, kv->valueLen, pCatalog); + break; + } case HEARTBEAT_KEY_DBINFO: { if (kv->valueLen <= 0 || NULL == kv->value) { tscError("invalid hb db info, len:%d, value:%p", kv->valueLen, kv->value); @@ -327,6 +366,39 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_SUCCESS; } +int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { + SUserAuthVersion *users = NULL; + uint32_t userNum = 0; + int32_t code = 0; + + code = catalogGetExpiredUsers(pCatalog, &users, &userNum); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + if (userNum <= 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < userNum; ++i) { + SUserAuthVersion *user = &users[i]; + user->version = htonl(user->version); + } + + SKv kv = { + .key = HEARTBEAT_KEY_USER_AUTHINFO, + .valueLen = sizeof(SUserAuthVersion) * userNum, + .value = users, + }; + + tscDebug("hb got %d expired users, valueLen:%d", userNum, kv.valueLen); + + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); + + return TSDB_CODE_SUCCESS; +} + + int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SDbVgVersion *dbs = NULL; uint32_t dbNum = 0; @@ -407,6 +479,11 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req hbGetQueryBasicInfo(connKey, req); + code = hbGetExpiredUserInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + code = hbGetExpiredDBInfo(connKey, pCatalog, req); if (TSDB_CODE_SUCCESS != code) { return code; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index ca6a11a668aedcf797b31fe6d0f1d615ba2290f9..f7ef1f7e813d629dbc0cae21afd629f4d55ce666 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -74,17 +74,44 @@ int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) { } int32_t stmtBackupQueryFields(STscStmt* pStmt) { - SQueryFields *pFields = &pStmt->sql.fields; - int32_t size = pFields->numOfCols * sizeof(TAOS_FIELD); + SStmtQueryResInfo *pRes = &pStmt->sql.queryRes; + pRes->numOfCols = pStmt->exec.pRequest->body.resInfo.numOfCols; + pRes->precision = pStmt->exec.pRequest->body.resInfo.precision; - pFields->numOfCols = pStmt->exec.pRequest->body.resInfo.numOfCols; - pFields->fields = taosMemoryMalloc(size); - pFields->userFields = taosMemoryMalloc(size); - if (NULL == pFields->fields || NULL == pFields->userFields) { + int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD); + pRes->fields = taosMemoryMalloc(size); + pRes->userFields = taosMemoryMalloc(size); + if (NULL == pRes->fields || NULL == pRes->userFields) { STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); } - memcpy(pFields->fields, pStmt->exec.pRequest->body.resInfo.fields, size); - memcpy(pFields->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size); + memcpy(pRes->fields, pStmt->exec.pRequest->body.resInfo.fields, size); + memcpy(pRes->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size); + + return TSDB_CODE_SUCCESS; +} + +int32_t stmtRestoreQueryFields(STscStmt* pStmt) { + SStmtQueryResInfo *pRes = &pStmt->sql.queryRes; + int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD); + + pStmt->exec.pRequest->body.resInfo.numOfCols = pRes->numOfCols; + pStmt->exec.pRequest->body.resInfo.precision = pRes->precision; + + if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { + pStmt->exec.pRequest->body.resInfo.fields = taosMemoryMalloc(size); + if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { + STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); + } + memcpy(pStmt->exec.pRequest->body.resInfo.fields, pRes->fields, size); + } + + if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { + pStmt->exec.pRequest->body.resInfo.userFields = taosMemoryMalloc(size); + if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { + STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); + } + memcpy(pStmt->exec.pRequest->body.resInfo.userFields, pRes->userFields, size); + } return TSDB_CODE_SUCCESS; } @@ -235,6 +262,8 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { } int32_t stmtCleanSQLInfo(STscStmt* pStmt) { + taosMemoryFree(pStmt->sql.queryRes.fields); + taosMemoryFree(pStmt->sql.queryRes.userFields); taosMemoryFree(pStmt->sql.sqlStr); qDestroyQuery(pStmt->sql.pQuery); qDestroyQueryPlan(pStmt->sql.pQueryPlan); @@ -497,6 +526,8 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) { pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag; pStmt->exec.pRequest->body.pDag = NULL; STMT_ERR_RET(stmtBackupQueryFields(pStmt)); + } else { + STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); } STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId)); @@ -509,7 +540,11 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) { } if (colIdx < 0) { - qBindStmtColsValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); + int32_t code = qBindStmtColsValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); + if (code) { + tscError("qBindStmtColsValue failed, error:%s", tstrerror(code)); + STMT_ERR_RET(code); + } } else { if (colIdx != (pStmt->bInfo.sBindLastIdx + 1) && colIdx != 0) { tscError("bind column index not in sequence"); @@ -614,6 +649,19 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + pStmt->bInfo.needParse = false; + } + + if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) { + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; + } + + if (NULL == pStmt->exec.pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); + } + if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); } @@ -623,8 +671,11 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList)); pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag; pStmt->exec.pRequest->body.pDag = NULL; + STMT_ERR_RET(stmtBackupQueryFields(pStmt)); + } else { + STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); } - + *nums = taosArrayGetSize(pStmt->sql.pQueryPlan->pPlaceholderValues); } else { STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL)); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index a093b7449b4114477f96917981669ce9782378e7..698c0cc4e7b712225cb97c01e608586abba027a7 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -68,6 +68,7 @@ struct tmq_conf_t { char* pass; char* db; tmq_commit_cb* commitCb; + void* commitCbUserParam; }; struct tmq_t { @@ -78,7 +79,8 @@ struct tmq_t { int32_t autoCommitInterval; int32_t resetOffsetCfg; int64_t consumerId; - tmq_commit_cb* commit_cb; + tmq_commit_cb* commitCb; + void* commitCbUserParam; // status int8_t status; @@ -372,10 +374,18 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; - if (pParam->tmq->commit_cb) { - pParam->tmq->commit_cb(pParam->tmq, pParam->rspErr, NULL); + if (pParam->tmq->commitCb) { + pParam->tmq->commitCb(pParam->tmq, pParam->rspErr, NULL, pParam->tmq->commitCbUserParam); + } + if (!pParam->async) + tsem_post(&pParam->rspSem); + else { + tsem_destroy(&pParam->rspSem); + /*if (pParam->pArray) {*/ + /*taosArrayDestroy(pParam->pArray);*/ + /*}*/ + taosMemoryFree(pParam); } - if (!pParam->async) tsem_post(&pParam->rspSem); return 0; } @@ -384,7 +394,7 @@ tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { *topics = tmq_list_new(); } for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { - SMqClientTopic* topic = taosArrayGetP(tmq->clientTopics, i); + SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i); tmq_list_append(*topics, topic->topicName); } return TMQ_RESP_ERR__SUCCESS; @@ -477,7 +487,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { strcpy(pTmq->groupId, conf->groupId); pTmq->autoCommit = conf->autoCommit; pTmq->autoCommitInterval = conf->autoCommitInterval; - pTmq->commit_cb = conf->commitCb; + pTmq->commitCb = conf->commitCb; + pTmq->commitCbUserParam = conf->commitCbUserParam; pTmq->resetOffsetCfg = conf->resetOffset; // assign consumerId @@ -557,7 +568,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in tscError("failed to malloc request"); } - SMqCommitCbParam* pParam = taosMemoryMalloc(sizeof(SMqCommitCbParam)); + SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam)); if (pParam == NULL) { return -1; } @@ -572,6 +583,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in }; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; sendInfo->param = pParam; sendInfo->fp = tmqCommitCb; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); @@ -582,13 +594,12 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in if (!async) { tsem_wait(&pParam->rspSem); resp = pParam->rspErr; - } - - tsem_destroy(&pParam->rspSem); - taosMemoryFree(pParam); + tsem_destroy(&pParam->rspSem); + taosMemoryFree(pParam); - if (pArray) { - taosArrayDestroy(pArray); + if (pArray) { + taosArrayDestroy(pArray); + } } return resp; @@ -667,7 +678,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { if (code != 0) goto FAIL; while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { - tscDebug("not ready, retry"); + tscDebug("consumer not ready, retry"); taosMsleep(500); } @@ -688,11 +699,13 @@ FAIL: return code; } -void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { +void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* param) { // conf->commitCb = cb; + conf->commitCbUserParam = param; } +#if 0 TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) { STscObj* pTscObj = (STscObj*)taos; SRequestObj* pRequest = NULL; @@ -777,6 +790,7 @@ _return: return pRequest; } +#endif #if 0 int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { @@ -1304,10 +1318,10 @@ const char* tmq_err2str(tmq_resp_err_t err) { return "fail"; } -char* tmq_get_topic_name(TAOS_RES* res) { +const char* tmq_get_topic_name(TAOS_RES* res) { if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; - return pRspObj->topic; + return strchr(pRspObj->topic, '.') + 1; } else { return NULL; } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 8e8c62cd946562d26ef5d3062683fc2f9c5bdc10..fc5781cb4d1e13f54c61415f060db46c50613ed5 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -587,15 +587,34 @@ TEST(testCase, projection_query_tables) { } taos_free_result(pRes); + pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + pRes = taos_query(pConn, "create table tu using st1 tags(1)"); if (taos_errno(pRes) != 0) { printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); } taos_free_result(pRes); - for(int32_t i = 0; i < 10000; ++i) { - char sql[512] = {0}; - sprintf(sql, "insert into tu values(now+%da, %d)", i, i); + pRes = taos_query(pConn, "create table tu2 using st2 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + for(int32_t i = 0; i < 10000000; i += 20) { + char sql[1024] = {0}; + sprintf(sql, + "insert into tu values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", + i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7, + i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14, + i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); TAOS_RES* p = taos_query(pConn, sql); if (taos_errno(p) != 0) { printf("failed to insert data, reason:%s\n", taos_errstr(p)); @@ -604,24 +623,44 @@ TEST(testCase, projection_query_tables) { taos_free_result(p); } - pRes = taos_query(pConn, "select * from tu"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } + printf("start to insert next table\n"); - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); + for(int32_t i = 0; i < 10000000; i += 20) { + char sql[1024] = {0}; + sprintf(sql, + "insert into tu2 values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", + i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7, + i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14, + i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); + TAOS_RES* p = taos_query(pConn, sql); + if (taos_errno(p) != 0) { + printf("failed to insert data, reason:%s\n", taos_errstr(p)); + } - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); + taos_free_result(p); } - taos_free_result(pRes); +// pRes = taos_query(pConn, "select * from tu"); +// if (taos_errno(pRes) != 0) { +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } + +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } + +// taos_free_result(pRes); taos_close(pConn); } @@ -660,9 +699,14 @@ TEST(testCase, agg_query_tables) { ASSERT_NE(pConn, nullptr); TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } taos_free_result(pRes); - pRes = taos_query(pConn, "select now() from m1"); + pRes = taos_query(pConn, "select tbname from st1"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); diff --git a/source/common/src/systable.c b/source/common/src/systable.c new file mode 100644 index 0000000000000000000000000000000000000000..51f924280ae2447aa2de50ecda78bd661857e67d --- /dev/null +++ b/source/common/src/systable.c @@ -0,0 +1,340 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "systable.h" +#include "tdef.h" +#include "types.h" +#include "taos.h" + +#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) +#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) +#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) + +static const SSysDbTableSchema dnodesSchema[] = { + {.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "max_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, +}; + +static const SSysDbTableSchema mnodesSchema[] = { + {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, +}; + +static const SSysDbTableSchema modulesSchema[] = { + {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "endpoint", .bytes = 134 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "module", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, +}; + +static const SSysDbTableSchema qnodesSchema[] = { + {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, +}; + +static const SSysDbTableSchema snodesSchema[] = { + {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, +}; + +static const SSysDbTableSchema bnodesSchema[] = { + {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, +}; + +static const SSysDbTableSchema clusterSchema[] = { + {.name = "id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, +}; + +static const SSysDbTableSchema userDBSchema[] = { + {.name = "name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "replica", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "strict", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "duration", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "buffer", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "pages", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "wal", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + // {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // disable update +}; + +static const SSysDbTableSchema userFuncSchema[] = { + {.name = "name", .bytes = TSDB_FUNC_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "comment", .bytes = PATH_MAX - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "aggregate", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "output_type", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "code_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "bufsize", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, +}; + +static const SSysDbTableSchema userIdxSchema[] = { + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "index_database", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "column_name", .bytes = SYSTABLE_SCH_COL_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_VARCHAR}, +}; + +static const SSysDbTableSchema userStbsSchema[] = { + {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, +}; + +static const SSysDbTableSchema userStreamsSchema[] = { + {.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "dest_table", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_VARCHAR}, +}; + +static const SSysDbTableSchema userTblsSchema[] = { + {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "table_comment", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "type", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, +}; + +static const SSysDbTableSchema userTblDistSchema[] = { + {.name = "db_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "distributed_histogram", .bytes = 500 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "stddev_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "storage_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "compression_ratio", .bytes = 8, .type = TSDB_DATA_TYPE_DOUBLE}, + {.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, +}; + +static const SSysDbTableSchema userUsersSchema[] = { + {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, +}; + +static const SSysDbTableSchema grantsSchema[] = { + {.name = "version", .bytes = 8 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "expire time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "storage(GB)", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "databases", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "users", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "accounts", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "connections", .bytes = 11 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "streams", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "cpu cores", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "speed(PPS)", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "querytime", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, +}; + +static const SSysDbTableSchema vgroupsSchema[] = { + {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v1_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v2_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v3_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, +}; + +static const SSysDbTableSchema smaSchema[] = { + {.name = "sma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, +}; + +static const SSysDbTableSchema transSchema[] = { + {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "stage", .bytes = TSDB_TRANS_STAGE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "type", .bytes = TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "last_exec_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "last_error", .bytes = (TSDB_TRANS_ERROR_LEN - 1) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, +}; + +static const SSysDbTableSchema configSchema[] = { + {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "value", .bytes = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, +}; + +static const SSysTableMeta infosMeta[] = { + {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)}, + {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)}, + {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)}, + {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)}, + {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, + {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, + {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema)}, + {TSDB_INS_TABLE_USER_DATABASES, userDBSchema, tListLen(userDBSchema)}, + {TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)}, + {TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)}, + {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)}, + {TSDB_INS_TABLE_USER_STREAMS, userStreamsSchema, tListLen(userStreamsSchema)}, + {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)}, + {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)}, + {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)}, + {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)}, + {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)}, + {TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema)}, +}; + +static const SSysDbTableSchema connectionsSchema[] = { + {.name = "conn_id", .bytes = 4, .type = TSDB_DATA_TYPE_UINT}, + {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "program", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "login_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, +}; + +static const SSysDbTableSchema topicSchema[] = { + {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + // TODO config +}; + +static const SSysDbTableSchema consumerSchema[] = { + {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "topics", .bytes = TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "subscribe_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "rebalance_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, +}; + +static const SSysDbTableSchema subscriptionSchema[] = { + {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, +}; + +static const SSysDbTableSchema offsetSchema[] = { + {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "committed_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "current_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "skip_log_cnt", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, +}; + +static const SSysDbTableSchema querySchema[] = { + {.name = "query_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "end_point", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "qid", .bytes = 22 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "time", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "sql_obj_id", .bytes = QUERY_OBJ_ID_SIZE + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "ep", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "stable_query", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, + {.name = "sub_queries", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "sub_query_info", .bytes = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, +}; + +static const SSysDbTableSchema streamSchema[] = { + {.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "source_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "target_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "target_table", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "watermark", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "trigger", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, +}; + +static const SSysTableMeta perfsMeta[] = { + {TSDB_PERFS_TABLE_CONNECTIONS, connectionsSchema, tListLen(connectionsSchema)}, + {TSDB_PERFS_TABLE_QUERIES, querySchema, tListLen(querySchema)}, + {TSDB_PERFS_TABLE_TOPICS, topicSchema, tListLen(topicSchema)}, + {TSDB_PERFS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)}, + {TSDB_PERFS_TABLE_SUBSCRIPTIONS, subscriptionSchema, tListLen(subscriptionSchema)}, + {TSDB_PERFS_TABLE_OFFSETS, offsetSchema, tListLen(offsetSchema)}, + {TSDB_PERFS_TABLE_TRANS, transSchema, tListLen(transSchema)}, + {TSDB_PERFS_TABLE_SMAS, smaSchema, tListLen(smaSchema)}, + {TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)}, +}; + +void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size) { + *pInfosTableMeta = infosMeta; + *size = tListLen(infosMeta); +} + +void getPerfDbMeta(const SSysTableMeta** pPerfsTableMeta, size_t* size) { + *pPerfsTableMeta = perfsMeta; + *size = tListLen(perfsMeta); +} diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f30a74bf11a56048294366a3b6ee1e96d11c0666..a4b36dcb67d02fb802062f3c1c16f155ee0b1324 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -168,13 +168,6 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c uint32_t total = numOfRow1 + numOfRow2; - if (BitmapLen(numOfRow1) < BitmapLen(total)) { - char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(total)); - uint32_t extend = BitmapLen(total) - BitmapLen(numOfRow1); - memset(tmp + BitmapLen(numOfRow1), 0, extend); - pColumnInfoData->nullbitmap = tmp; - } - uint32_t remindBits = BitPos(numOfRow1); uint32_t shiftBits = 8 - remindBits; @@ -194,25 +187,24 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c int32_t i = 0; uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)]; - int32_t overCount = BitmapLen(total) - BitmapLen(numOfRow1); - while (i < len) { // size limit of pSource->nullbitmap + int32_t overCount = BitmapLen(total) - BitmapLen(numOfRow1); + while (i < len) { // size limit of pSource->nullbitmap if (i >= 1) { - start[i - 1] |= (p[i] >> remindBits); //copy remind bits + start[i - 1] |= (p[i] >> remindBits); // copy remind bits } - if (i >= overCount) { // size limit of pColumnInfoData->nullbitmap + if (i >= overCount) { // size limit of pColumnInfoData->nullbitmap return; } - start[i] |= (p[i] << shiftBits); //copy shift bits + start[i] |= (p[i] << shiftBits); // copy shift bits i += 1; } } -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, - uint32_t numOfRow2) { +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, + const SColumnInfoData* pSource, uint32_t numOfRow2) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); - if (numOfRow2 == 0) { return numOfRow1; } @@ -221,14 +213,19 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co pColumnInfoData->hasNull = pSource->hasNull; } + uint32_t finalNumOfRows = numOfRow1 + numOfRow2; if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { // Handle the bitmap - char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); - if (p == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + if (finalNumOfRows > *capacity) { + char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + *capacity = finalNumOfRows; + pColumnInfoData->varmeta.offset = (int32_t*)p; } - pColumnInfoData->varmeta.offset = (int32_t*)p; for (int32_t i = 0; i < numOfRow2; ++i) { if (pSource->varmeta.offset[i] == -1) { pColumnInfoData->varmeta.offset[i + numOfRow1] = -1; @@ -253,15 +250,27 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); pColumnInfoData->varmeta.length = len + oldLen; } else { - doBitmapMerge(pColumnInfoData, numOfRow1, pSource, numOfRow2); + if (finalNumOfRows > *capacity) { + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); + if (tmp == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } - int32_t newSize = (numOfRow1 + numOfRow2) * pColumnInfoData->info.bytes; - char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); - if (tmp == NULL) { - return TSDB_CODE_VND_OUT_OF_MEMORY; + pColumnInfoData->pData = tmp; + + if (BitmapLen(numOfRow1) < BitmapLen(finalNumOfRows)) { + char* btmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(finalNumOfRows)); + uint32_t extend = BitmapLen(finalNumOfRows) - BitmapLen(numOfRow1); + memset(btmp + BitmapLen(numOfRow1), 0, extend); + + pColumnInfoData->nullbitmap = btmp; + } + + *capacity = finalNumOfRows; } - pColumnInfoData->pData = tmp; + doBitmapMerge(pColumnInfoData, numOfRow1, pSource, numOfRow2); + int32_t offset = pColumnInfoData->info.bytes * numOfRow1; memcpy(pColumnInfoData->pData + offset, pSource->pData, pSource->info.bytes * numOfRow2); } @@ -350,29 +359,22 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) { // if pIndexMap = NULL, merger one column by on column int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pIndexMap) { assert(pSrc != NULL && pDest != NULL); + int32_t capacity = pDest->info.capacity; - int32_t numOfCols = pDest->info.numOfCols; - for (int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < pDest->info.numOfCols; ++i) { int32_t mapIndex = i; - if(pIndexMap) { - mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i); - } +// if (pIndexMap) { +// mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i); +// } + SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, mapIndex); - uint32_t oldLen = colDataGetLength(pCol2, pDest->info.rows); - uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows); - - int32_t newSize = oldLen + newLen; - char* tmp = taosMemoryRealloc(pCol2->pData, newSize); - if (tmp != NULL) { - pCol2->pData = tmp; - colDataMergeCol(pCol2, pDest->info.rows, pCol1, pSrc->info.rows); - } else { - return TSDB_CODE_VND_OUT_OF_MEMORY; - } + capacity = pDest->info.capacity; + colDataMergeCol(pCol2, pDest->info.rows, &capacity, pCol1, pSrc->info.rows); } + pDest->info.capacity = capacity; pDest->info.rows += pSrc->info.rows; return TSDB_CODE_SUCCESS; } @@ -451,7 +453,6 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd // all fit in *stopIndex = numOfRows - 1; return TSDB_CODE_SUCCESS; - } SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount) { @@ -490,9 +491,14 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i); for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) { - bool isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg); - char* p = colDataGetData(pColData, j); + bool isNull = false; + if (pBlock->pBlockAgg == NULL) { + isNull = colDataIsNull_s(pColData, pBlock->info.rows); + } else { + isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]); + } + char* p = colDataGetData(pColData, j); colDataAppend(pDstCol, j - startIndex, p, isNull); } } @@ -556,7 +562,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { if (IS_VAR_DATA_TYPE(pCol->info.type)) { size_t metaSize = pBlock->info.rows * sizeof(int32_t); - char* tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize); // preview calloc is too small + char* tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize); // preview calloc is too small if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -702,8 +708,8 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { SColumnInfoData* pColInfoData = pOrder->pColData; // TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); if (pColInfoData->hasNull) { - bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); - bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg); + bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, NULL); + bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, NULL); if (leftNull && rightNull) { continue; // continue to next slot } @@ -742,7 +748,7 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co SColumnInfoData* pDst = &pDstCols[i]; SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, i); - if (pSrc->hasNull && colDataIsNull(pSrc, pSrcBlock->info.rows, tupleIndex, pSrcBlock->pBlockAgg)) { + if (pSrc->hasNull && colDataIsNull(pSrc, pSrcBlock->info.rows, tupleIndex, pSrcBlock->pBlockAgg[i])) { code = colDataAppend(pDst, numOfRows, NULL, true); if (code != TSDB_CODE_SUCCESS) { return code; @@ -938,8 +944,9 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { copyBackToBlock(pDataBlock, pCols); int64_t p4 = taosGetTimestampUs(); - uDebug("blockDataSort complex sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, - p3 - p2, p4 - p3, rows); + uDebug("blockDataSort complex sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 + ", rows:%d\n", + p1 - p0, p2 - p1, p3 - p2, p4 - p3, rows); destroyTupleIndex(index); return TSDB_CODE_SUCCESS; @@ -1176,7 +1183,7 @@ void* blockDataDestroy(SSDataBlock* pBlock) { } SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { - if(pDataBlock == NULL){ + if (pDataBlock == NULL) { return NULL; } @@ -1187,7 +1194,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { pBlock->info.numOfCols = numOfCols; pBlock->info.hasVarCol = pDataBlock->info.hasVarCol; - pBlock->info.rowSize = pDataBlock->info.rows; + pBlock->info.rowSize = pDataBlock->info.rows; for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {0}; @@ -1217,7 +1224,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { } size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { - return (int32_t) ((pageSize - blockDataGetSerialMetaSize(pBlock))/ blockDataGetSerialRowSize(pBlock)); + return (int32_t)((pageSize - blockDataGetSerialMetaSize(pBlock)) / blockDataGetSerialRowSize(pBlock)); } void colDataDestroy(SColumnInfoData* pColData) { @@ -1234,14 +1241,14 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { int32_t len = BitmapLen(total); int32_t newLen = BitmapLen(total - n); - if (n%8 == 0) { - memmove(nullBitmap, nullBitmap + n/8, newLen); + if (n % 8 == 0) { + memmove(nullBitmap, nullBitmap + n / 8, newLen); } else { int32_t tail = n % 8; int32_t i = 0; - uint8_t* p = (uint8_t*) nullBitmap; - while(i < len) { + uint8_t* p = (uint8_t*)nullBitmap; + while (i < len) { uint8_t v = p[i]; p[i] = 0; @@ -1268,7 +1275,7 @@ static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_ } } -int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n) { +int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { if (n == 0) { return TSDB_CODE_SUCCESS; } @@ -1276,7 +1283,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n) { if (pBlock->info.rows <= n) { blockDataCleanup(pBlock); } else { - for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows); } @@ -1462,3 +1469,193 @@ void blockDebugShowData(const SArray* dataBlocks) { } } +/** + * @brief TODO: Assume that the final generated result it less than 3M + * + * @param pReq + * @param pDataBlocks + * @param vgId + * @param uid set as parameter temporarily // TODO: remove this parameter, and the executor should set uid in + * SDataBlock->info.uid + * @param suid // TODO: check with Liao whether suid response is reasonable + * + * TODO: colId should be set + */ +int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid, + tb_uid_t suid) { + int32_t sz = taosArrayGetSize(pDataBlocks); + int32_t bufSize = sizeof(SSubmitReq); + for (int32_t i = 0; i < sz; ++i) { + SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info; + bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols)); + bufSize += sizeof(SSubmitBlk); + } + + ASSERT(bufSize < 3 * 1024 * 1024); + + *pReq = taosMemoryCalloc(1, bufSize); + if(!(*pReq)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + void* pDataBuf = *pReq; + + int32_t msgLen = sizeof(SSubmitReq); + int32_t numOfBlks = 0; + SRowBuilder rb = {0}; + tdSRowInit(&rb, 0); // TODO: use the latest version + + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); + int32_t colNum = pDataBlock->info.numOfCols; + int32_t rows = pDataBlock->info.rows; + int32_t rowSize = pDataBlock->info.rowSize; + int64_t groupId = pDataBlock->info.groupId; + + if(rb.nCols != colNum) { + tdSRowSetTpInfo(&rb, colNum, pTSchema->flen); + } + + SSubmitBlk* pSubmitBlk = POINTER_SHIFT(pDataBuf, msgLen); + pSubmitBlk->suid = suid; + pSubmitBlk->uid = uid; + pSubmitBlk->numOfRows = rows; + + ++numOfBlks; + + msgLen += sizeof(SSubmitBlk); + int32_t dataLen = 0; + for (int32_t j = 0; j < rows; ++j) { // iterate by row + tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf + printf("|"); + bool isStartKey = false; + for (int32_t k = 0; k < colNum; ++k) { // iterate by column + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + if (!isStartKey) { + isStartKey = true; + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 0, 0); + } else { + tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 8, k); + break; + } + break; + case TSDB_DATA_TYPE_NCHAR: { + tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, 8, k); + break; + } + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, 8, k); + break; + } + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + printf("the column type %" PRIi16 " is defined but not implemented yet\n", pColInfoData->info.type); + TASSERT(0); + break; + default: + if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { + tdAppendColValToRow(&rb, 2, pColInfoData->info.type, TD_VTYPE_NORM, var, true, 8, k); + } else { + printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + TASSERT(0); + } + break; + } + } + dataLen += TD_ROW_LEN(rb.pBuf); + } + pSubmitBlk->dataLen = dataLen; + msgLen += pSubmitBlk->dataLen; + } + + (*pReq)->length = msgLen; + + (*pReq)->header.vgId = htonl(vgId); + (*pReq)->header.contLen = htonl(msgLen); + (*pReq)->length = (*pReq)->header.contLen; + (*pReq)->numOfBlocks = htonl(numOfBlks); + SSubmitBlk* blk = (SSubmitBlk*)((*pReq) + 1); + while (numOfBlks--) { + int32_t dataLen = blk->dataLen; + blk->uid = htobe64(blk->uid); + blk->suid = htobe64(blk->suid); + blk->padding = htonl(blk->padding); + blk->sversion = htonl(blk->sversion); + blk->dataLen = htonl(blk->dataLen); + blk->schemaLen = htonl(blk->schemaLen); + blk->numOfRows = htons(blk->numOfRows); + blk = (SSubmitBlk*)(blk->data + dataLen); + } + + return TSDB_CODE_SUCCESS; +} + +SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema) { + SSubmitReq* ret = NULL; + + // cal size + int32_t cap = sizeof(SSubmitReq); + int32_t sz = taosArrayGetSize(pBlocks); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + int32_t rows = pDataBlock->info.rows; + // TODO min + int32_t rowSize = pDataBlock->info.rowSize; + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + cap += sizeof(SSubmitBlk) + rows * maxLen; + } + + // assign data + ret = taosMemoryCalloc(1, cap); + ret->version = htonl(1); + ret->length = htonl(cap - sizeof(SSubmitReq)); + ret->numOfBlocks = htonl(sz); + + void* submitBlk = POINTER_SHIFT(ret, sizeof(SSubmitReq)); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + + SSubmitBlk* blkHead = submitBlk; + blkHead->numOfRows = htons(pDataBlock->info.rows); + blkHead->schemaLen = 0; + blkHead->sversion = htonl(pTSchema->version); + // TODO + blkHead->suid = 0; + blkHead->uid = htobe64(pDataBlock->info.uid); + + int32_t rows = pDataBlock->info.rows; + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + /*blkHead->dataLen = htonl(rows * maxLen);*/ + blkHead->dataLen = 0; + + void* blockData = POINTER_SHIFT(submitBlk, sizeof(SSubmitBlk)); + STSRow* rowData = blockData; + + for (int32_t j = 0; j < pDataBlock->info.rows; j++) { + SRowBuilder rb = {0}; + tdSRowInit(&rb, pTSchema->version); + tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); + tdSRowResetBuf(&rb, rowData); + + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pColumn = &pTSchema->columns[k]; + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); + void* data = colDataGetData(pColData, j); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); + } + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + blkHead->dataLen += rowLen; + } + int32_t len = blkHead->dataLen; + blkHead->dataLen = htonl(len); + blkHead = POINTER_SHIFT(blkHead, len); + } + + return ret; +} diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 2ed08ac81fda23e1ef12a95d840e1e4eb7043c3e..e659b9ec04fffc472cae0c07354744ca8c7eb67a 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "tdataformat.h" #include "tcoding.h" +#include "tdatablock.h" #include "tlog.h" static void dataColSetNEleNull(SDataCol *pCol, int nEle); @@ -87,7 +88,7 @@ int tdEncodeSchema(void **buf, STSchema *pSchema) { for (int i = 0; i < schemaNCols(pSchema); i++) { STColumn *pCol = schemaColAt(pSchema, i); tlen += taosEncodeFixedI8(buf, colType(pCol)); - tlen += taosEncodeFixedI8(buf, colSma(pCol)); + tlen += taosEncodeFixedI8(buf, colFlags(pCol)); tlen += taosEncodeFixedI16(buf, colColId(pCol)); tlen += taosEncodeFixedI16(buf, colBytes(pCol)); } @@ -110,14 +111,14 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { for (int i = 0; i < numOfCols; i++) { col_type_t type = 0; - int8_t sma = 0; + int8_t flags = 0; col_id_t colId = 0; col_bytes_t bytes = 0; buf = taosDecodeFixedI8(buf, &type); - buf = taosDecodeFixedI8(buf, &sma); + buf = taosDecodeFixedI8(buf, &flags); buf = taosDecodeFixedI16(buf, &colId); buf = taosDecodeFixedI32(buf, &bytes); - if (tdAddColToSchema(&schemaBuilder, type, sma, colId, bytes) < 0) { + if (tdAddColToSchema(&schemaBuilder, type, flags, colId, bytes) < 0) { tdDestroyTSchemaBuilder(&schemaBuilder); return NULL; } @@ -128,6 +129,50 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { return buf; } +#if 0 +int32_t tEncodeSTColumn(SCoder *pEncoder, const STColumn *pCol) { + if (tEncodeI16(pEncoder, pCol->colId) < 0) return -1; + if (tEncodeI8(pEncoder, pCol->type) < 0) return -1; + if (tEncodeI8(pEncoder, pCol->sma) < 0) return -1; + if (tEncodeI32(pEncoder, pCol->bytes) < 0) return -1; + if (tEncodeI32(pEncoder, pCol->offset) < 0) return -1; + return pEncoder->pos; +} + +int32_t tDecodeSTColumn(SCoder *pDecoder, STColumn *pCol) { + if (tDecodeI16(pDecoder, &pCol->colId) < 0) return -1; + if (tDecodeI8(pDecoder, &pCol->type) < 0) return -1; + if (tDecodeI8(pDecoder, &pCol->sma) < 0) return -1; + if (tDecodeI32(pDecoder, &pCol->bytes) < 0) return -1; + if (tDecodeI32(pDecoder, &pCol->offset) < 0) return -1; + return 0; +} + +int32_t tEncodeSchema(SCoder *pEncoder, const STSchema *pSchema) { + if (tEncodeI32(pEncoder, pSchema->numOfCols) < 0) return -1; + if (tEncodeI16(pEncoder, pSchema->version) < 0) return -1; + if (tEncodeU16(pEncoder, pSchema->flen) < 0) return -1; + if (tEncodeI32(pEncoder, pSchema->vlen) < 0) return -1; + if (tEncodeI32(pEncoder, pSchema->tlen) < 0) return -1; + + for (int32_t i = 0; i < schemaNCols(pSchema); i++) { + const STColumn *pCol = schemaColAt(pSchema, i); + if (tEncodeSTColumn(pEncoder, pCol) < 0) return -1; + } + return 0; +} + +int32_t tDecodeSchema(SCoder *pDecoder, STSchema *pSchema) { + if (tDecodeI32(pDecoder, &pSchema->numOfCols) < 0) return -1; + if (tDecodeI16(pDecoder, &pSchema->version) < 0) return -1; + if (tDecodeU16(pDecoder, &pSchema->flen) < 0) return -1; + if (tDecodeI32(pDecoder, &pSchema->vlen) < 0) return -1; + if (tDecodeI32(pDecoder, &pSchema->tlen) < 0) return -1; + + return 0; +} +#endif + int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { if (pBuilder == NULL) return -1; @@ -153,7 +198,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { pBuilder->version = version; } -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes) { +int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes) { if (!isValidDataType(type)) return -1; if (pBuilder->nCols >= pBuilder->tCols) { @@ -166,7 +211,7 @@ int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]); colSetType(pCol, type); colSetColId(pCol, colId); - colSetSma(pCol, sma); + colSetFlags(pCol, flags); if (pBuilder->nCols == 0) { colSetOffset(pCol, 0); } else { @@ -908,4 +953,4 @@ SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSch taosArrayDestroy(stashRow); return buffer; } -#endif \ No newline at end of file +#endif diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index bbf14b2fdc42ac889a9183eb90981ab217590564..87a9c521afed5c803f454c66388e579c4596c113 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -169,6 +169,10 @@ uint32_t tsMaxRange = 500; // max range uint32_t tsCurRange = 100; // range char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR +// internal +int32_t tsTransPullupMs = 6000; +int32_t tsMaRebalanceMs = 2000; + void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) { tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); tsDiskCfg[index].level = level; @@ -220,7 +224,8 @@ struct SConfig *taosGetCfg() { return tsCfg; } -static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { +static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, + char *apolloUrl) { char cfgDir[PATH_MAX] = {0}; char cfgFile[PATH_MAX + 100] = {0}; @@ -296,15 +301,10 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { static int32_t taosAddClientCfg(SConfig *pCfg) { char defaultFqdn[TSDB_FQDN_LEN] = {0}; int32_t defaultServerPort = 6030; - char defaultFirstEp[TSDB_EP_LEN] = {0}; - char defaultSecondEp[TSDB_EP_LEN] = {0}; - if (taosGetFqdn(defaultFqdn) != 0) return -1; - snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); - snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); - if (cfgAddString(pCfg, "firstEp", defaultFirstEp, 1) != 0) return -1; - if (cfgAddString(pCfg, "secondEp", defaultSecondEp, 1) != 0) return -1; + if (cfgAddString(pCfg, "firstEp", "", 1) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", "", 1) != 0) return -1; if (cfgAddString(pCfg, "fqdn", defaultFqdn, 1) != 0) return -1; if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1; if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1; @@ -474,15 +474,18 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + char defaultFirstEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); SEp firstEp = {0}; - taosGetFqdnPortFromEp(pFirstEpItem->str, &firstEp); + taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); SConfigItem *pSecondpItem = cfgGetItem(pCfg, "secondEp"); SEp secondEp = {0}; - taosGetFqdnPortFromEp(pSecondpItem->str, &secondEp); + taosGetFqdnPortFromEp(strlen(pSecondpItem->str) == 0 ? defaultFirstEp : pSecondpItem->str, &secondEp); snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); @@ -579,8 +582,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { return 0; } -int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, - char *apolloUrl, SArray *pArgs, bool tsc) { +int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, + const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { osDefaultInit(); SConfig *pCfg = cfgInit(); @@ -632,7 +635,24 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi return 0; } -int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { +static int32_t taosCheckGlobalCfg() { + uint32_t ipv4 = taosGetIpv4FromFqdn(tsLocalFqdn); + if (ipv4 == 0xffffffff) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, terrstr()); + return -1; + } + + if (tsServerPort <= 0) { + uError("invalid server port:%u, dnode can not be initialized", tsServerPort); + return -1; + } + + return 0; +} + +int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, + bool tsc) { if (tsCfg != NULL) return 0; tsCfg = cfgInit(); @@ -670,6 +690,11 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile taosSetSystemCfg(tsCfg); cfgDumpCfg(tsCfg, tsc, false); + + if (taosCheckGlobalCfg() != 0) { + return -1; + } + return 0; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 5400e6e08a46080fc3c161dd555b510f966de71b..6278b52a04c259b9ad11652626fe2dfd31ed2247 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -34,77 +34,6 @@ int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { return -1; } - pIter->totalLen = pMsg->length; - ASSERT(pIter->totalLen > 0); - pIter->len = 0; - pIter->pMsg = pMsg; - if (pMsg->length <= sizeof(SSubmitReq)) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - return -1; - } - - return 0; -} - -int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { - ASSERT(pIter->len >= 0); - - if (pIter->len == 0) { - pIter->len += sizeof(SSubmitReq); - } else { - if (pIter->len >= pIter->totalLen) { - ASSERT(0); - } - - SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); - pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen); - ASSERT(pIter->len > 0); - } - - if (pIter->len > pIter->totalLen) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - *pPBlock = NULL; - return -1; - } - - *pPBlock = (pIter->len == pIter->totalLen) ? NULL : (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); - - return 0; -} - -int32_t tInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { - if (pBlock->dataLen <= 0) return -1; - pIter->totalLen = pBlock->dataLen; - pIter->len = 0; - pIter->row = (STSRow *)(pBlock->data + pBlock->schemaLen); - return 0; -} - -STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { - STSRow *row = pIter->row; - - if (pIter->len >= pIter->totalLen) { - return NULL; - } else { - pIter->len += TD_ROW_LEN(row); - if (pIter->len < pIter->totalLen) { - pIter->row = POINTER_SHIFT(row, TD_ROW_LEN(row)); - } - return row; - } -} - -// TODO: KEEP one suite of iterator API finally. -// 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts -// 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later -// 3) finally, rename tInitSubmitMsgIterEx to tInitSubmitMsgIter - -int32_t tInitSubmitMsgIterEx(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { - if (pMsg == NULL) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - return -1; - } - pIter->totalLen = htonl(pMsg->length); ASSERT(pIter->totalLen > 0); pIter->len = 0; @@ -117,7 +46,7 @@ int32_t tInitSubmitMsgIterEx(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { return 0; } -int32_t tGetSubmitMsgNextEx(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { +int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { ASSERT(pIter->len >= 0); if (pIter->len == 0) { @@ -152,7 +81,7 @@ int32_t tGetSubmitMsgNextEx(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { return 0; } -int32_t tInitSubmitBlkIterEx(SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { +int32_t tInitSubmitBlkIter(SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { if (pMsgIter->dataLen <= 0) return -1; pIter->totalLen = pMsgIter->dataLen; pIter->len = 0; @@ -160,7 +89,7 @@ int32_t tInitSubmitBlkIterEx(SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubm return 0; } -STSRow *tGetSubmitBlkNextEx(SSubmitBlkIter *pIter) { +STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { STSRow *row = pIter->row; if (pIter->len >= pIter->totalLen) { @@ -174,6 +103,25 @@ STSRow *tGetSubmitBlkNextEx(SSubmitBlkIter *pIter) { } } +int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq *pReq, STSchema *pTschema) { + SSubmitMsgIter msgIter = {0}; + if (tInitSubmitMsgIter(pReq, &msgIter) < 0) return -1; + while (true) { + SSubmitBlk *pBlock = NULL; + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; + if (pBlock == NULL) break; + SSubmitBlkIter blkIter = {0}; + tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); + STSRowIter rowIter = {0}; + tdSTSRowIterInit(&rowIter, pTschema); + STSRow *row; + while ((row = tGetSubmitBlkNext(&blkIter)) != NULL) { + tdSRowPrint(row, pTschema, "stream"); + } + } + return 0; +} + int32_t tEncodeSEpSet(SCoder *pEncoder, const SEpSet *pEp) { if (tEncodeI8(pEncoder, pEp->inUse) < 0) return -1; if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1; @@ -490,21 +438,6 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR return 0; } -int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pReq->ver); - tlen += taosEncodeString(buf, pReq->name); - tlen += taosEncodeFixedU8(buf, pReq->type); - return tlen; -} - -void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) { - buf = taosDecodeFixedI64(buf, &pReq->ver); - buf = taosDecodeString(buf, &pReq->name); - buf = taosDecodeFixedU8(buf, &pReq->type); - return buf; -} - int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -1231,31 +1164,47 @@ int32_t tDeserializeSGetUserAuthReq(void *buf, int32_t bufLen, SGetUserAuthReq * return 0; } -int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { - SCoder encoder = {0}; - tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pRsp->user) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->superAuth) < 0) return -1; +int32_t tSerializeSGetUserAuthRspImpl(SCoder *pEncoder, SGetUserAuthRsp *pRsp) { + if (tEncodeCStr(pEncoder, pRsp->user) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->superAuth) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->version) < 0) return -1; + int32_t numOfCreatedDbs = taosHashGetSize(pRsp->createdDbs); int32_t numOfReadDbs = taosHashGetSize(pRsp->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pRsp->writeDbs); - if (tEncodeI32(&encoder, numOfReadDbs) < 0) return -1; - if (tEncodeI32(&encoder, numOfWriteDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfCreatedDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfReadDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfWriteDbs) < 0) return -1; + + char *db = taosHashIterate(pRsp->createdDbs, NULL); + while (db != NULL) { + if (tEncodeCStr(pEncoder, db) < 0) return -1; + db = taosHashIterate(pRsp->createdDbs, db); + } - char *db = taosHashIterate(pRsp->readDbs, NULL); + db = taosHashIterate(pRsp->readDbs, NULL); while (db != NULL) { - if (tEncodeCStr(&encoder, db) < 0) return -1; + if (tEncodeCStr(pEncoder, db) < 0) return -1; db = taosHashIterate(pRsp->readDbs, db); } db = taosHashIterate(pRsp->writeDbs, NULL); while (db != NULL) { - if (tEncodeCStr(&encoder, db) < 0) return -1; + if (tEncodeCStr(pEncoder, db) < 0) return -1; db = taosHashIterate(pRsp->writeDbs, db); } + return 0; +} + +int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + + if (tSerializeSGetUserAuthRspImpl(&encoder, pRsp) < 0) return -1; + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1263,39 +1212,58 @@ int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pR return tlen; } -int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { - pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); +int32_t tDeserializeSGetUserAuthRspImpl(SCoder *pDecoder, SGetUserAuthRsp *pRsp) { + pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pRsp->readDbs == NULL || pRsp->writeDbs == NULL) { return -1; } - SCoder decoder = {0}; - tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pRsp->user) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->superAuth) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pRsp->user) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->superAuth) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->version) < 0) return -1; + int32_t numOfCreatedDbs = 0; int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; - if (tDecodeI32(&decoder, &numOfReadDbs) < 0) return -1; - if (tDecodeI32(&decoder, &numOfWriteDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfCreatedDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfReadDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfWriteDbs) < 0) return -1; + + for (int32_t i = 0; i < numOfCreatedDbs; ++i) { + char db[TSDB_DB_FNAME_LEN] = {0}; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; + int32_t len = strlen(db) + 1; + taosHashPut(pRsp->createdDbs, db, len, db, len); + } for (int32_t i = 0; i < numOfReadDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(&decoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; int32_t len = strlen(db) + 1; taosHashPut(pRsp->readDbs, db, len, db, len); } for (int32_t i = 0; i < numOfWriteDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(&decoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; int32_t len = strlen(db) + 1; taosHashPut(pRsp->writeDbs, db, len, db, len); } + return 0; +} + + +int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDeserializeSGetUserAuthRspImpl(&decoder, pRsp) < 0) return -1; + tEndDecode(&decoder); tCoderClear(&decoder); @@ -1303,6 +1271,7 @@ int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp * } void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) { + taosHashCleanup(pRsp->createdDbs); taosHashCleanup(pRsp->readDbs); taosHashCleanup(pRsp->writeDbs); } @@ -1676,32 +1645,29 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfVgroups) < 0) return -1; - if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1; - if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfStables) < 0) return -1; + if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1; + if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1; + if (tEncodeI32(&encoder, pReq->pages) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; if (tEncodeI32(&encoder, pReq->minRows) < 0) return -1; if (tEncodeI32(&encoder, pReq->maxRows) < 0) return -1; - if (tEncodeI32(&encoder, pReq->commitTime) < 0) return -1; if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; - if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1; if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pReq->precision) < 0) return -1; if (tEncodeI8(&encoder, pReq->compression) < 0) return -1; if (tEncodeI8(&encoder, pReq->replications) < 0) return -1; if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; - if (tEncodeI8(&encoder, pReq->update) < 0) return -1; if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1; - if (tEncodeI8(&encoder, pReq->streamMode) < 0) return -1; - if (tEncodeI8(&encoder, pReq->singleSTable) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1; for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { SRetention *pRetension = taosArrayGet(pReq->pRetensions, i); - if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1; - if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->freq) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->keep) < 0) return -1; if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; } @@ -1719,27 +1685,24 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfVgroups) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfStables) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; if (tDecodeI32(&decoder, &pReq->minRows) < 0) return -1; if (tDecodeI32(&decoder, &pReq->maxRows) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->commitTime) < 0) return -1; if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1; if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1; if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1; if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1; if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->update) < 0) return -1; if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->streamMode) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->singleSTable) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfRetensions) < 0) return -1; pReq->pRetensions = taosArrayInit(pReq->numOfRetensions, sizeof(SRetention)); if (pReq->pRetensions == NULL) { @@ -1749,8 +1712,8 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { SRetention rentension = {0}; - if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1; - if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.freq) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.keep) < 0) return -1; if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; if (taosArrayPush(pReq->pRetensions, &rentension) == NULL) { @@ -1776,7 +1739,10 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; - if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; + if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1; + if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1; + if (tEncodeI32(&encoder, pReq->pages) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; @@ -1798,7 +1764,10 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; @@ -2122,6 +2091,62 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { taosArrayDestroy(pRsp->pArray); } +int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp *pUserAuthRsp = taosArrayGet(pRsp->pArray, i); + if (tSerializeSGetUserAuthRspImpl(&encoder, pUserAuthRsp) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + + pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SGetUserAuthRsp)); + if (pRsp->pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp rsp = {0}; + if (tDeserializeSGetUserAuthRspImpl(&decoder, &rsp) < 0) return -1; + taosArrayPush(pRsp->pArray, &rsp); + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp){ + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp *pUserAuthRsp = taosArrayGet(pRsp->pArray, i); + tFreeSGetUserAuthRsp(pUserAuthRsp); + } + + taosArrayDestroy(pRsp->pArray); +} + + int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -2153,29 +2178,28 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) { if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI32(&encoder, pRsp->numOfVgroups) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->cacheBlockSize) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->totalBlocks) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->numOfStables) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->buffer) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->pageSize) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->pages) < 0) return -1; if (tEncodeI32(&encoder, pRsp->daysPerFile) < 0) return -1; if (tEncodeI32(&encoder, pRsp->daysToKeep0) < 0) return -1; if (tEncodeI32(&encoder, pRsp->daysToKeep1) < 0) return -1; if (tEncodeI32(&encoder, pRsp->daysToKeep2) < 0) return -1; if (tEncodeI32(&encoder, pRsp->minRows) < 0) return -1; if (tEncodeI32(&encoder, pRsp->maxRows) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->commitTime) < 0) return -1; if (tEncodeI32(&encoder, pRsp->fsyncPeriod) < 0) return -1; if (tEncodeI8(&encoder, pRsp->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pRsp->precision) < 0) return -1; if (tEncodeI8(&encoder, pRsp->compression) < 0) return -1; if (tEncodeI8(&encoder, pRsp->replications) < 0) return -1; if (tEncodeI8(&encoder, pRsp->strict) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->update) < 0) return -1; if (tEncodeI8(&encoder, pRsp->cacheLastRow) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->streamMode) < 0) return -1; if (tEncodeI32(&encoder, pRsp->numOfRetensions) < 0) return -1; for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i); - if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1; - if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->freq) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->keep) < 0) return -1; if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; } @@ -2192,24 +2216,23 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->numOfVgroups) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->cacheBlockSize) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->totalBlocks) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->numOfStables) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->buffer) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->pageSize) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->pages) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->daysPerFile) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->daysToKeep0) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->daysToKeep1) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->daysToKeep2) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->minRows) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->maxRows) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->commitTime) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->fsyncPeriod) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->precision) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->compression) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->replications) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->strict) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->update) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->cacheLastRow) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->streamMode) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->numOfRetensions) < 0) return -1; pRsp->pRetensions = taosArrayInit(pRsp->numOfRetensions, sizeof(SRetention)); if (pRsp->pRetensions == NULL) { @@ -2219,8 +2242,8 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { SRetention rentension = {0}; - if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1; - if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.freq) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.keep) < 0) return -1; if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; if (taosArrayPush(pRsp->pRetensions, &rentension) == NULL) { @@ -2368,7 +2391,6 @@ static int32_t tEncodeSTableMetaRsp(SCoder *pEncoder, STableMetaRsp *pRsp) { if (tEncodeI32(pEncoder, pRsp->numOfColumns) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->precision) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->tableType) < 0) return -1; - if (tEncodeI8(pEncoder, pRsp->update) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->sversion) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->tversion) < 0) return -1; if (tEncodeU64(pEncoder, pRsp->suid) < 0) return -1; @@ -2391,7 +2413,6 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) { if (tDecodeI32(pDecoder, &pRsp->numOfColumns) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->precision) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->tableType) < 0) return -1; - if (tDecodeI8(pDecoder, &pRsp->update) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->sversion) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->tversion) < 0) return -1; if (tDecodeU64(pDecoder, &pRsp->suid) < 0) return -1; @@ -2811,15 +2832,16 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; - if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1; - if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfStables) < 0) return -1; + if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1; + if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1; + if (tEncodeI32(&encoder, pReq->pages) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; if (tEncodeI32(&encoder, pReq->minRows) < 0) return -1; if (tEncodeI32(&encoder, pReq->maxRows) < 0) return -1; - if (tEncodeI32(&encoder, pReq->commitTime) < 0) return -1; if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; if (tEncodeU32(&encoder, pReq->hashBegin) < 0) return -1; if (tEncodeU32(&encoder, pReq->hashEnd) < 0) return -1; @@ -2828,11 +2850,9 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI8(&encoder, pReq->precision) < 0) return -1; if (tEncodeI8(&encoder, pReq->compression) < 0) return -1; if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; - if (tEncodeI8(&encoder, pReq->update) < 0) return -1; if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; if (tEncodeI8(&encoder, pReq->replica) < 0) return -1; if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1; - if (tEncodeI8(&encoder, pReq->streamMode) < 0) return -1; for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { SReplica *pReplica = &pReq->replicas[i]; if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; @@ -2840,8 +2860,8 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1; for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { SRetention *pRetension = taosArrayGet(pReq->pRetensions, i); - if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1; - if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->freq) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->keep) < 0) return -1; if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; } @@ -2862,15 +2882,16 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfStables) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; if (tDecodeI32(&decoder, &pReq->minRows) < 0) return -1; if (tDecodeI32(&decoder, &pReq->maxRows) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->commitTime) < 0) return -1; if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; if (tDecodeU32(&decoder, &pReq->hashBegin) < 0) return -1; if (tDecodeU32(&decoder, &pReq->hashEnd) < 0) return -1; @@ -2879,11 +2900,9 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1; if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1; if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->update) < 0) return -1; if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1; if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->streamMode) < 0) return -1; for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { SReplica *pReplica = &pReq->replicas[i]; if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; @@ -2898,8 +2917,8 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { SRetention rentension = {0}; - if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1; - if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.freq) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.keep) < 0) return -1; if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; if (taosArrayPush(pReq->pRetensions, &rentension) == NULL) { @@ -2983,10 +3002,14 @@ int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; - if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; + if (tEncodeI32(&encoder, pReq->buffer) < 0) return -1; + if (tEncodeI32(&encoder, pReq->pageSize) < 0) return -1; + if (tEncodeI32(&encoder, pReq->pages) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; + if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; @@ -3010,10 +3033,14 @@ int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pR if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->buffer) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->pageSize) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->pages) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; @@ -3809,3 +3836,181 @@ int tDecodeSVCreateTbRsp(SCoder *pCoder, SVCreateTbRsp *pRsp) { tEndDecode(pCoder); return 0; } + +// TDMT_VND_DROP_TABLE ================= +static int32_t tEncodeSVDropTbReq(SCoder *pCoder, const SVDropTbReq *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeCStr(pCoder, pReq->name) < 0) return -1; + if (tEncodeI8(pCoder, pReq->igNotExists) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +static int32_t tDecodeSVDropTbReq(SCoder *pCoder, SVDropTbReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1; + if (tDecodeI8(pCoder, &pReq->igNotExists) < 0) return -1; + + tEndDecode(pCoder); + return 0; +} + +static int32_t tEncodeSVDropTbRsp(SCoder *pCoder, const SVDropTbRsp *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32(pCoder, pReq->code) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +static int32_t tDecodeSVDropTbRsp(SCoder *pCoder, SVDropTbRsp *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32(pCoder, &pReq->code) < 0) return -1; + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVDropTbBatchReq(SCoder *pCoder, const SVDropTbBatchReq *pReq) { + int32_t nReqs = taosArrayGetSize(pReq->pArray); + SVDropTbReq *pDropTbReq; + + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, nReqs) < 0) return -1; + for (int iReq = 0; iReq < nReqs; iReq++) { + pDropTbReq = (SVDropTbReq *)taosArrayGet(pReq->pArray, iReq); + if (tEncodeSVDropTbReq(pCoder, pDropTbReq) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVDropTbBatchReq(SCoder *pCoder, SVDropTbBatchReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32v(pCoder, &pReq->nReqs) < 0) return -1; + pReq->pReqs = (SVDropTbReq *)tCoderMalloc(pCoder, sizeof(SVDropTbReq) * pReq->nReqs); + if (pReq->pReqs == NULL) return -1; + for (int iReq = 0; iReq < pReq->nReqs; iReq++) { + if (tDecodeSVDropTbReq(pCoder, pReq->pReqs + iReq) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVDropTbBatchRsp(SCoder *pCoder, const SVDropTbBatchRsp *pRsp) { + int32_t nRsps = taosArrayGetSize(pRsp->pArray); + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, nRsps) < 0) return -1; + for (int iRsp = 0; iRsp < nRsps; iRsp++) { + if (tEncodeSVDropTbRsp(pCoder, (SVDropTbRsp *)taosArrayGet(pRsp->pArray, iRsp)) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVDropTbBatchRsp(SCoder *pCoder, SVDropTbBatchRsp *pRsp) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32v(pCoder, &pRsp->nRsps) < 0) return -1; + pRsp->pRsps = (SVDropTbRsp *)tCoderMalloc(pCoder, sizeof(SVDropTbRsp) * pRsp->nRsps); + if (pRsp->pRsps == NULL) return -1; + for (int iRsp = 0; iRsp < pRsp->nRsps; iRsp++) { + if (tDecodeSVDropTbRsp(pCoder, pRsp->pRsps + iRsp) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVDropStbReq(SCoder *pCoder, const SVDropStbReq *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeCStr(pCoder, pReq->name) < 0) return -1; + if (tEncodeI64(pCoder, pReq->suid) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVDropStbReq(SCoder *pCoder, SVDropStbReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1; + + tEndDecode(pCoder); + return 0; +} + +static int32_t tEncodeSVSubmitBlk(SCoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1; + if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1; + if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1; + if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1; + + if (flags & TD_AUTO_CREATE_TABLE) { + if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +static int32_t tDecodeSVSubmitBlk(SCoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1; + if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1; + if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1; + if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1; + + if (flags & TD_AUTO_CREATE_TABLE) { + if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVSubmitReq(SCoder *pCoder, const SVSubmitReq *pReq) { + int32_t nBlocks = taosArrayGetSize(pReq->pArray); + + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, pReq->flags) < 0) return -1; + if (tEncodeI32v(pCoder, nBlocks) < 0) return -1; + for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) { + if (tEncodeSVSubmitBlk(pCoder, (SVSubmitBlk *)taosArrayGet(pReq->pArray, iBlock), pReq->flags) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVSubmitReq(SCoder *pCoder, SVSubmitReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1; + if (tDecodeI32v(pCoder, &pReq->nBlocks) < 0) return -1; + pReq->pBlocks = tCoderMalloc(pCoder, sizeof(SVSubmitBlk) * pReq->nBlocks); + if (pReq->pBlocks == NULL) return -1; + for (int32_t iBlock = 0; iBlock < pReq->nBlocks; iBlock++) { + if (tDecodeSVSubmitBlk(pCoder, pReq->pBlocks + iBlock, pReq->flags) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} \ No newline at end of file diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c index 5b4bb539e3ba0790d42a23a7018833981e270712..42612cecb9531a7d29e92963f7fa7bf582982986 100644 --- a/source/common/src/tmsgcb.c +++ b/source/common/src/tmsgcb.c @@ -15,33 +15,83 @@ #define _DEFAULT_SOURCE #include "tmsgcb.h" +#include "taoserror.h" static SMsgCb tsDefaultMsgCb; void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { tsDefaultMsgCb = *pMsgCb; } int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq) { - return (*pMsgCb->queueFps[qtype])(pMsgCb->pWrapper, pReq); + PutToQueueFp fp = pMsgCb->queueFps[qtype]; + if (fp != NULL) { + return (*fp)(pMsgCb->pWrapper, pReq); + } else { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } } int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) { - return (*pMsgCb->qsizeFp)(pMsgCb->pWrapper, vgId, qtype); + GetQueueSizeFp fp = pMsgCb->qsizeFp; + if (fp != NULL) { + return (*fp)(pMsgCb->pWrapper, vgId, qtype); + } else { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } } int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq) { - return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq); + SendReqFp fp = pMsgCb->sendReqFp; + if (fp != NULL) { + return (*fp)(pMsgCb->pWrapper, epSet, pReq); + } else { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } } -void tmsgSendRsp(const SRpcMsg* pRsp) { return (*tsDefaultMsgCb.sendRspFp)(tsDefaultMsgCb.pWrapper, pRsp); } +void tmsgSendRsp(const SRpcMsg* pRsp) { + SendRspFp fp = tsDefaultMsgCb.sendRspFp; + if (fp != NULL) { + return (*fp)(tsDefaultMsgCb.pWrapper, pRsp); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } +} + +void tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet) { + SendRedirectRspFp fp = tsDefaultMsgCb.sendRedirectRspFp; + if (fp != NULL) { + (*fp)(tsDefaultMsgCb.pWrapper, pRsp, pNewEpSet); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } +} void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) { - (*pMsgCb->registerBrokenLinkArgFp)(pMsgCb->pWrapper, pMsg); + RegisterBrokenLinkArgFp fp = pMsgCb->registerBrokenLinkArgFp; + if (fp != NULL) { + (*fp)(pMsgCb->pWrapper, pMsg); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } } void tmsgReleaseHandle(void* handle, int8_t type) { - (*tsDefaultMsgCb.releaseHandleFp)(tsDefaultMsgCb.pWrapper, handle, type); + ReleaseHandleFp fp = tsDefaultMsgCb.releaseHandleFp; + if (fp != NULL) { + (*fp)(tsDefaultMsgCb.pWrapper, handle, type); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } } void tmsgReportStartup(const char* name, const char* desc) { - (*tsDefaultMsgCb.reportStartupFp)(tsDefaultMsgCb.pWrapper, name, desc); + ReportStartup fp = tsDefaultMsgCb.reportStartupFp; + if (fp != NULL && tsDefaultMsgCb.pWrapper != NULL) { + (*fp)(tsDefaultMsgCb.pWrapper, name, desc); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } } \ No newline at end of file diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index fb8e5e7fb23d39353eb317da9dec72416da8b01e..114d7b6dfcc29be8b677f84d34dce50b1b1f4a99 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -72,6 +72,7 @@ static void dmProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSe NodeMsgFp msgFp = NULL; uint16_t msgType = pRpc->msgType; bool needRelease = false; + bool isReq = msgType & 1U; if (pEpSet && pEpSet->numOfEps > 0 && msgType == TDMT_MND_STATUS_RSP) { dmSetMnodeEpSet(pWrapper->pDnode, pEpSet); @@ -85,13 +86,13 @@ static void dmProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSe if (dmBuildMsg(pMsg, pRpc) != 0) goto _OVER; if (pWrapper->procType == DND_PROC_SINGLE) { - dTrace("msg:%p, is created, type:%s handle:%p user:%s", pMsg, TMSG_INFO(msgType), pRpc->handle, pMsg->user); + dTrace("msg:%p, created, type:%s handle:%p user:%s", pMsg, TMSG_INFO(msgType), pRpc->handle, pMsg->user); code = (*msgFp)(pWrapper, pMsg); } else if (pWrapper->procType == DND_PROC_PARENT) { - dTrace("msg:%p, is created and put into child queue, type:%s handle:%p user:%s", pMsg, TMSG_INFO(msgType), - pRpc->handle, pMsg->user); - code = taosProcPutToChildQ(pWrapper->procObj, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, pRpc->handle, - pRpc->refId, PROC_FUNC_REQ); + dTrace("msg:%p, created and put into child queue, type:%s handle:%p code:0x%04x user:%s contLen:%d", pMsg, + TMSG_INFO(msgType), pRpc->handle, pMsg->rpcMsg.code & 0XFFFF, pMsg->user, pRpc->contLen); + code = taosProcPutToChildQ(pWrapper->procObj, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, + (isReq && (pMsg->rpcMsg.code == 0)) ? pRpc->handle : NULL, pRpc->refId, PROC_FUNC_REQ); } else { dTrace("msg:%p, should not processed in child process, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); ASSERT(1); @@ -100,12 +101,13 @@ static void dmProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSe _OVER: if (code == 0) { if (pWrapper->procType == DND_PROC_PARENT) { - dTrace("msg:%p, is freed in parent process", pMsg); + dTrace("msg:%p, freed in parent process", pMsg); taosFreeQitem(pMsg); rpcFreeCont(pRpc->pCont); } } else { - dError("msg:%p, type:%s failed to process since 0x%04x:%s", pMsg, TMSG_INFO(msgType), code & 0XFFFF, terrstr()); + dError("msg:%p, type:%s handle:%p failed to process since 0x%04x:%s", pMsg, TMSG_INFO(msgType), pRpc->handle, + code & 0XFFFF, terrstr()); if (msgType & 1U) { if (terrno != 0) code = terrno; if (code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_NODE_OFFLINE) { @@ -254,8 +256,17 @@ static void dmSendRpcRedirectRsp(SDnode *pDnode, const SRpcMsg *pReq) { epSet.eps[i].port = htons(epSet.eps[i].port); } - - rpcSendRedirectRsp(pReq->handle, &epSet); + SRpcMsg resp; + SMEpSet msg = {.epSet = epSet}; + int32_t len = tSerializeSMEpSet(NULL, 0, &msg); + resp.pCont = rpcMallocCont(len); + resp.contLen = len; + tSerializeSMEpSet(resp.pCont, len, &msg); + + resp.code = TSDB_CODE_RPC_REDIRECT; + resp.handle = pReq->handle; + resp.refId = pReq->refId; + rpcSendResponse(&resp); } static inline void dmSendRpcRsp(SDnode *pDnode, const SRpcMsg *pRsp) { @@ -309,6 +320,37 @@ static inline void dmSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { } } +static inline void dmSendRedirectRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp, const SEpSet *pNewEpSet) { + ASSERT(pRsp->code == TSDB_CODE_RPC_REDIRECT); + ASSERT(pRsp->pCont == NULL); + if (pWrapper->procType != DND_PROC_CHILD) { + SRpcMsg resp = {0}; + SMEpSet msg = {.epSet = *pNewEpSet}; + int32_t len = tSerializeSMEpSet(NULL, 0, &msg); + resp.pCont = rpcMallocCont(len); + resp.contLen = len; + tSerializeSMEpSet(resp.pCont, len, &msg); + + resp.code = TSDB_CODE_RPC_REDIRECT; + resp.handle = pRsp->handle; + resp.refId = pRsp->refId; + rpcSendResponse(&resp); + } else { + taosProcPutToParentQ(pWrapper->procObj, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_FUNC_RSP); + } +} + +#if 0 +static inline void dmSendRedirectRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp, const SEpSet *pNewEpSet) { + ASSERT(pRsp->code == TSDB_CODE_RPC_REDIRECT); + if (pWrapper->procType != DND_PROC_CHILD) { + rpcSendRedirectRsp(pRsp->handle, pNewEpSet); + } else { + taosProcPutToParentQ(pWrapper->procObj, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_FUNC_RSP); + } +} +#endif + static inline void dmRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { if (pWrapper->procType != DND_PROC_CHILD) { rpcRegisterBrokenLinkArg(pMsg); @@ -350,29 +392,31 @@ static void dmConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t static void dmConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, EProcFuncType ftype) { + int32_t code = pMsg->code & 0xFFFF; pMsg->pCont = pCont; - dTrace("msg:%p, get from parent queue, ftype:%d handle:%p code:0x%04x mtype:%d, app:%p", pMsg, ftype, pMsg->handle, - pMsg->code & 0xFFFF, pMsg->msgType, pMsg->ahandle); - - switch (ftype) { - case PROC_FUNC_REGIST: - rpcRegisterBrokenLinkArg(pMsg); - break; - case PROC_FUNC_RELEASE: - taosProcRemoveHandle(pWrapper->procObj, pMsg->handle); - rpcReleaseHandle(pMsg->handle, (int8_t)pMsg->code); - rpcFreeCont(pCont); - break; - case PROC_FUNC_REQ: - dmSendRpcReq(pWrapper->pDnode, (SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg); - break; - case PROC_FUNC_RSP: - pMsg->refId = taosProcRemoveHandle(pWrapper->procObj, pMsg->handle); - dmSendRpcRsp(pWrapper->pDnode, pMsg); - break; - default: - break; + + if (ftype == PROC_FUNC_REQ) { + dTrace("msg:%p, get from parent queue, send req:%s handle:%p code:0x%04x, app:%p", pMsg, TMSG_INFO(pMsg->msgType), + pMsg->handle, code, pMsg->ahandle); + dmSendRpcReq(pWrapper->pDnode, (SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg); + } else if (ftype == PROC_FUNC_RSP) { + dTrace("msg:%p, get from parent queue, rsp handle:%p code:0x%04x, app:%p", pMsg, pMsg->handle, code, pMsg->ahandle); + pMsg->refId = taosProcRemoveHandle(pWrapper->procObj, pMsg->handle); + dmSendRpcRsp(pWrapper->pDnode, pMsg); + } else if (ftype == PROC_FUNC_REGIST) { + dTrace("msg:%p, get from parent queue, regist handle:%p code:0x%04x, app:%p", pMsg, pMsg->handle, code, + pMsg->ahandle); + rpcRegisterBrokenLinkArg(pMsg); + } else if (ftype == PROC_FUNC_RELEASE) { + dTrace("msg:%p, get from parent queue, release handle:%p code:0x%04x, app:%p", pMsg, pMsg->handle, code, + pMsg->ahandle); + taosProcRemoveHandle(pWrapper->procObj, pMsg->handle); + rpcReleaseHandle(pMsg->handle, (int8_t)pMsg->code); + rpcFreeCont(pCont); + } else { + dError("msg:%p, invalid ftype:%d while get from parent queue, handle:%p", pMsg, ftype, pMsg->handle); } + taosMemoryFree(pMsg); } @@ -393,6 +437,14 @@ SProcCfg dmGenProcCfg(SMgmtWrapper *pWrapper) { return cfg; } +bool rpcRfp(int32_t code) { + if (code == TSDB_CODE_RPC_REDIRECT) { + return true; + } else { + return false; + } +} + static int32_t dmInitClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; @@ -407,6 +459,7 @@ static int32_t dmInitClient(SDnode *pDnode) { rpcInit.ckey = INTERNAL_CKEY; rpcInit.spi = 1; rpcInit.parent = pDnode; + rpcInit.rfp = rpcRfp; char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); @@ -494,6 +547,8 @@ static int32_t dmInitServer(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; + + strncpy(rpcInit.localFqdn, pDnode->data.localFqdn, strlen(pDnode->data.localFqdn)); rpcInit.localPort = pDnode->data.serverPort; rpcInit.label = "DND"; rpcInit.numOfThreads = tsNumOfRpcThreads; @@ -538,6 +593,7 @@ SMsgCb dmGetMsgcb(SMgmtWrapper *pWrapper) { SMsgCb msgCb = { .sendReqFp = dmSendReq, .sendRspFp = dmSendRsp, + .sendRedirectRspFp = dmSendRedirectRsp, .registerBrokenLinkArgFp = dmRegisterBrokenLinkArg, .releaseHandleFp = dmReleaseHandle, .reportStartupFp = dmReportStartupByWrapper, diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 99977f29278f2dd1219ba9ae873264ae8bd0f388..e3816396c9bd766923a7346b934d8ee394e2004c 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -107,14 +107,14 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->vgId = pCreate->vgId; strcpy(pCfg->dbname, pCreate->db); - // pCfg->szBuf = pCreate->cacheBlockSize * 1024 * 1024; - pCfg->streamMode = pCreate->streamMode; pCfg->isWeak = true; pCfg->tsdbCfg.days = 10; - pCfg->tsdbCfg.keep2 = 3650; // pCreate->daysToKeep0; - pCfg->tsdbCfg.keep0 = 3650; // pCreate->daysToKeep2; - pCfg->tsdbCfg.keep1 = 3650; // pCreate->daysToKeep0; - pCfg->tsdbCfg.retentions = pCreate->pRetensions; + pCfg->tsdbCfg.keep2 = 3650; + pCfg->tsdbCfg.keep0 = 3650; + pCfg->tsdbCfg.keep1 = 3650; + for (size_t i = 0; i < taosArrayGetSize(pCreate->pRetensions); ++i) { + memcpy(&pCfg->tsdbCfg.retentions[i], taosArrayGet(pCreate->pRetensions, i), sizeof(SRetention)); + } pCfg->walCfg.vgId = pCreate->vgId; pCfg->hashBegin = pCreate->hashBegin; pCfg->hashEnd = pCreate->hashEnd; @@ -284,6 +284,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_SUBMIT_RSMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 7285503a7300a2687b24c67bf0c37336a30cc876..f3d7253e716de5037194298fb2970f007474e48b 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -136,7 +136,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO // sync integration response for (int i = 0; i < taosArrayGetSize(pArray); i++) { SNodeMsg *pMsg; - SRpcMsg * pRpc; + SRpcMsg *pRpc; pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); pRpc = &pMsg->rpcMsg; @@ -149,8 +149,15 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pRpc, false); if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { - rsp.code = TSDB_CODE_SYN_NOT_LEADER; - tmsgSendRsp(&rsp); + // rsp.code = TSDB_CODE_SYN_NOT_LEADER; + // tmsgSendRsp(&rsp); + dTrace("syncPropose not leader redirect, vgId:%d ", syncGetVgId(vnodeGetSyncHandle(pVnode->pImpl))); + rsp.code = TSDB_CODE_RPC_REDIRECT; + SEpSet newEpSet; + syncGetEpSet(vnodeGetSyncHandle(pVnode->pImpl), &newEpSet); + newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps; + tmsgSendRedirectRsp(&rsp, &newEpSet); + } else if (ret == TAOS_SYNC_PROPOSE_OTHER_ERROR) { rsp.code = TSDB_CODE_SYN_INTERNAL_ERROR; tmsgSendRsp(&rsp); @@ -175,7 +182,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SNodeMsg * pMsg = NULL; + SNodeMsg *pMsg = NULL; SRpcMsg rsp; for (int32_t i = 0; i < numOfMsgs; ++i) { @@ -218,7 +225,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SNodeMsg * pMsg = NULL; + SNodeMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); @@ -231,7 +238,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SNodeMsg * pMsg = NULL; + SNodeMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); @@ -248,7 +255,7 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO } static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueType qtype) { - SRpcMsg * pRpc = &pMsg->rpcMsg; + SRpcMsg *pRpc = &pMsg->rpcMsg; SMsgHead *pHead = pRpc->pCont; pHead->contLen = ntohl(pHead->contLen); pHead->vgId = ntohl(pHead->vgId); @@ -262,23 +269,23 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueTyp int32_t code = 0; switch (qtype) { case QUERY_QUEUE: - dTrace("msg:%p, will be written into vnode-query queue", pMsg); + dTrace("msg:%p, type:%s will be written into vnode-query queue", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pQueryQ, pMsg); break; case FETCH_QUEUE: - dTrace("msg:%p, will be written into vnode-fetch queue", pMsg); + dTrace("msg:%p, type:%s will be written into vnode-fetch queue", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - dTrace("msg:%p, will be written into vnode-write queue", pMsg); + dTrace("msg:%p, type:%s will be written into vnode-write queue", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pWriteQ, pMsg); break; case SYNC_QUEUE: - dTrace("msg:%p, will be written into vnode-sync queue", pMsg); + dTrace("msg:%p, type:%s will be written into vnode-sync queue", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pSyncQ, pMsg); break; case MERGE_QUEUE: - dTrace("msg:%p, will be written into vnode-merge queue", pMsg); + dTrace("msg:%p, type:%s will be written into vnode-merge queue", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pMergeQ, pMsg); break; default: @@ -317,7 +324,7 @@ int32_t vmProcessMergeMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SVnodesMgmt * pMgmt = pWrapper->pMgmt; + SVnodesMgmt *pMgmt = pWrapper->pMgmt; SSingleWorker *pWorker = &pMgmt->mgmtWorker; dTrace("msg:%p, will be written to vnode-mgmt queue, worker:%s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); @@ -325,7 +332,7 @@ int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SVnodesMgmt * pMgmt = pWrapper->pMgmt; + SVnodesMgmt *pMgmt = pWrapper->pMgmt; SSingleWorker *pWorker = &pMgmt->monitorWorker; dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); @@ -335,7 +342,7 @@ int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueType qtype) { SVnodesMgmt *pMgmt = pWrapper->pMgmt; - SMsgHead * pHead = pRpc->pCont; + SMsgHead *pHead = pRpc->pCont; SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) return -1; diff --git a/source/dnode/mgmt/test/vnode/vnode.cpp b/source/dnode/mgmt/test/vnode/vnode.cpp index 51b7d6818fdce72103f830eb4a9ba57c8b335197..769c484c6ae7d57196e70d569493d27ef7212872 100644 --- a/source/dnode/mgmt/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/test/vnode/vnode.cpp @@ -33,22 +33,18 @@ TEST_F(DndTestVnode, 01_Create_Vnode) { strcpy(createReq.db, "1.d1"); createReq.dbUid = 9527; createReq.vgVersion = 1; - createReq.cacheBlockSize = 16; - createReq.totalBlocks = 10; createReq.daysPerFile = 10; createReq.daysToKeep0 = 3650; createReq.daysToKeep1 = 3650; createReq.daysToKeep2 = 3650; createReq.minRows = 100; createReq.minRows = 4096; - createReq.commitTime = 3600; createReq.fsyncPeriod = 3000; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; createReq.replica = 1; createReq.strict = 1; - createReq.update = 0; createReq.cacheLastRow = 0; createReq.selfIndex = 0; for (int r = 0; r < createReq.replica; ++r) { @@ -75,27 +71,15 @@ TEST_F(DndTestVnode, 01_Create_Vnode) { TEST_F(DndTestVnode, 02_Alter_Vnode) { for (int i = 0; i < 3; ++i) { SAlterVnodeReq alterReq = {0}; - alterReq.vgId = 2; - alterReq.dnodeId = 1; - strcpy(alterReq.db, "1.d1"); - alterReq.dbUid = 9527; alterReq.vgVersion = 2; - alterReq.cacheBlockSize = 16; - alterReq.totalBlocks = 10; alterReq.daysPerFile = 10; alterReq.daysToKeep0 = 3650; alterReq.daysToKeep1 = 3650; alterReq.daysToKeep2 = 3650; - alterReq.minRows = 100; - alterReq.minRows = 4096; - alterReq.commitTime = 3600; alterReq.fsyncPeriod = 3000; alterReq.walLevel = 1; - alterReq.precision = 0; - alterReq.compression = 2; alterReq.replica = 1; alterReq.strict = 1; - alterReq.update = 0; alterReq.cacheLastRow = 0; alterReq.selfIndex = 0; for (int r = 0; r < alterReq.replica; ++r) { diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index 19e202dddc0c56cc7679d7a3d319ab7918e36528..a8bfe91cbf8edfb225510253b9536115b21cea30 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -44,6 +44,11 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw); int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer); +bool mndRebTryStart(); +void mndRebEnd(); +void mndRebCntInc(); +void mndRebCntDec(); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9fc43d601a64b81147f15ecc5775521706aebf15..89638524ae8622930eee95bbf4f01fa2778a9c6b 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -57,11 +57,11 @@ typedef enum { TRN_STAGE_PREPARE = 0, TRN_STAGE_REDO_LOG = 1, TRN_STAGE_REDO_ACTION = 2, - TRN_STAGE_COMMIT = 3, - TRN_STAGE_COMMIT_LOG = 4, - TRN_STAGE_UNDO_ACTION = 5, - TRN_STAGE_UNDO_LOG = 6, - TRN_STAGE_ROLLBACK = 7, + TRN_STAGE_ROLLBACK = 3, + TRN_STAGE_UNDO_ACTION = 4, + TRN_STAGE_UNDO_LOG = 5, + TRN_STAGE_COMMIT = 6, + TRN_STAGE_COMMIT_LOG = 7, TRN_STAGE_FINISHED = 8 } ETrnStage; @@ -72,6 +72,7 @@ typedef enum { TRN_TYPE_DROP_USER = 1003, TRN_TYPE_CREATE_FUNC = 1004, TRN_TYPE_DROP_FUNC = 1005, + TRN_TYPE_CREATE_SNODE = 1006, TRN_TYPE_DROP_SNODE = 1007, TRN_TYPE_CREATE_QNODE = 1008, @@ -91,10 +92,12 @@ typedef enum { TRN_TYPE_CONSUMER_LOST = 1022, TRN_TYPE_CONSUMER_RECOVER = 1023, TRN_TYPE_BASIC_SCOPE_END, + TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_CREATE_DNODE = 2001, TRN_TYPE_DROP_DNODE = 2002, TRN_TYPE_GLOBAL_SCOPE_END, + TRN_TYPE_DB_SCOPE = 3000, TRN_TYPE_CREATE_DB = 3001, TRN_TYPE_ALTER_DB = 3002, @@ -102,6 +105,7 @@ typedef enum { TRN_TYPE_SPLIT_VGROUP = 3004, TRN_TYPE_MERGE_VGROUP = 3015, TRN_TYPE_DB_SCOPE_END, + TRN_TYPE_STB_SCOPE = 4000, TRN_TYPE_CREATE_STB = 4001, TRN_TYPE_ALTER_STB = 4002, @@ -131,7 +135,7 @@ typedef struct { int32_t id; ETrnStage stage; ETrnPolicy policy; - ETrnType transType; + ETrnType type; int32_t code; int32_t failedTimes; void* rpcHandle; @@ -251,32 +255,31 @@ typedef struct { int64_t updateTime; int8_t superUser; int32_t acctId; + int32_t authVersion; SHashObj* readDbs; SHashObj* writeDbs; + SRWLatch lock; } SUserObj; typedef struct { int32_t numOfVgroups; - int32_t cacheBlockSize; - int32_t totalBlocks; + int32_t numOfStables; + int32_t buffer; + int32_t pageSize; + int32_t pages; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; int32_t daysToKeep2; int32_t minRows; int32_t maxRows; - int32_t commitTime; int32_t fsyncPeriod; - int32_t ttl; int8_t walLevel; int8_t precision; int8_t compression; int8_t replications; int8_t strict; - int8_t update; int8_t cacheLastRow; - int8_t streamMode; - int8_t singleSTable; int8_t hashMethod; // default is 1 int32_t numOfRetensions; SArray* pRetensions; @@ -316,7 +319,6 @@ typedef struct { int64_t pointsWritten; int8_t compact; int8_t replica; - int8_t streamMode; SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; } SVgObj; @@ -392,15 +394,12 @@ typedef struct { int8_t type; int8_t replica; int16_t numOfColumns; - int32_t rowSize; int32_t numOfRows; void* pIter; SMnode* pMnode; STableMetaRsp* pMeta; bool sysDbRsp; char db[TSDB_DB_FNAME_LEN]; - int16_t offset[TSDB_MAX_COLUMNS]; - int32_t bytes[TSDB_MAX_COLUMNS]; } SShowObj; typedef struct { @@ -422,6 +421,7 @@ typedef struct { typedef struct { char key[TSDB_PARTITION_KEY_LEN]; + int64_t dbUid; int64_t offset; } SMqOffsetObj; @@ -524,6 +524,7 @@ void* tDecodeSMqConsumerEp(const void* buf, SMqConsumerEp* pEp); typedef struct { char key[TSDB_SUBSCRIBE_KEY_LEN]; SRWLatch lock; + int64_t dbUid; int32_t vgNum; int8_t subType; int8_t withTbName; @@ -560,9 +561,8 @@ int32_t tEncodeSMqSubActionLogObj(void** buf, const SMqSubActionLogO void* tDecodeSMqSubActionLogObj(const void* buf, SMqSubActionLogObj* pLog); typedef struct { - const SMqSubscribeObj* pOldSub; - const SMqTopicObj* pTopic; - const SMqRebSubscribe* pRebInfo; + int32_t oldConsumerNum; + const SMqRebInfo* pRebInfo; } SMqRebInputObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndInfoSchema.h b/source/dnode/mnode/impl/inc/mndInfoSchema.h index 3aea99a909e02796dfbd9862c56339f9f1ba27c4..43d934c43101161bc63a487c644a5f6f3b218348 100644 --- a/source/dnode/mnode/impl/inc/mndInfoSchema.h +++ b/source/dnode/mnode/impl/inc/mndInfoSchema.h @@ -22,18 +22,6 @@ extern "C" { #endif -typedef struct SInfosTableSchema { - const char *name; - const int32_t type; - const int32_t bytes; -} SInfosTableSchema; - -typedef struct SInfosTableMeta { - const char *name; - const SInfosTableSchema *schema; - const int32_t colNum; -} SInfosTableMeta; - int32_t mndInitInfos(SMnode *pMnode); void mndCleanupInfos(SMnode *pMnode); int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp); diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 5083104039a71e75df1c7a803d2f2e8b10391ed3..b96444bebcc7d9df5eff88d7dc878b9fa170be5f 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -47,7 +47,7 @@ typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); -typedef struct SQWorkerMgmt SQHandle; +typedef struct SQWorker SQHandle; typedef struct { const char *name; diff --git a/source/dnode/mnode/impl/inc/mndOffset.h b/source/dnode/mnode/impl/inc/mndOffset.h index 556bfe6a53f3fc9da48d0c471c35215868a468e2..b468496165549bceba4d08ae7cc6d2ef55ef9b6d 100644 --- a/source/dnode/mnode/impl/inc/mndOffset.h +++ b/source/dnode/mnode/impl/inc/mndOffset.h @@ -37,6 +37,8 @@ static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, c return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName); } +int32_t mndDropOffsetByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndPerfSchema.h b/source/dnode/mnode/impl/inc/mndPerfSchema.h index 2c613e253ccc7cafaf6c75729234f90d9fc97e60..19f60229f926583f321ab350f6ac6043f6c9b096 100644 --- a/source/dnode/mnode/impl/inc/mndPerfSchema.h +++ b/source/dnode/mnode/impl/inc/mndPerfSchema.h @@ -22,18 +22,6 @@ extern "C" { #endif -typedef struct SPerfsTableSchema { - char *name; - int32_t type; - int32_t bytes; -} SPerfsTableSchema; - -typedef struct SPerfsTableMeta { - char *name; - const SPerfsTableSchema *schema; - int32_t colNum; -} SPerfsTableMeta; - int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp); int32_t mndInitPerfs(SMnode *pMnode); void mndCleanupPerfs(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/inc/mndShow.h b/source/dnode/mnode/impl/inc/mndShow.h index 03f901e9c412e1ddf466cf336190d4fe7a590b0c..5a3e487e3d350e503186992a17ea9d3a4bc6416e 100644 --- a/source/dnode/mnode/impl/inc/mndShow.h +++ b/source/dnode/mnode/impl/inc/mndShow.h @@ -17,6 +17,7 @@ #define _TD_MND_SHOW_H_ #include "mndInt.h" +#include "systable.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/mnode/impl/inc/mndSma.h b/source/dnode/mnode/impl/inc/mndSma.h index 91c6e24e28d14a157956886b16ad5348193a8ff5..4a80f619d3026c82abeb2c2e9c777073dbf2fa6a 100644 --- a/source/dnode/mnode/impl/inc/mndSma.h +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -26,7 +26,6 @@ int32_t mndInitSma(SMnode *pMnode); void mndCleanupSma(SMnode *pMnode); SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); -int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 82d93a4b9b2cd40fe4c8caed99e6e0011eee8944..a415d39434bca49af308ff149864fb29f3726a0b 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -29,6 +29,7 @@ void mndReleaseStb(SMnode *pMnode, SStbObj *pStb); SSdbRaw *mndStbActionEncode(SStbObj *pStb); int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbs, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen); +int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndSubscribe.h b/source/dnode/mnode/impl/inc/mndSubscribe.h index 7915006f274ce8db1a0e0ae4607da4deb0effe99..5ca672e8dd83808e79302057d08bd41e29635893 100644 --- a/source/dnode/mnode/impl/inc/mndSubscribe.h +++ b/source/dnode/mnode/impl/inc/mndSubscribe.h @@ -31,6 +31,8 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub); int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName); +int32_t mndDropSubByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index fd82c60d37401ecf53eba98d98df6378ec4da54e..e3174a90a24d845fdbae2fa1efaf57f0f33baf8c 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -31,6 +31,8 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic); SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic); SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw); +int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 5b5aff7c86ef204a1e75d6b0d20a295c454ab57c..7bd2bd70ee7fdab00f828a1fe0471c390c568d62 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -36,14 +36,16 @@ typedef struct { typedef enum { TEST_TRANS_START_FUNC = 1, TEST_TRANS_STOP_FUNC = 2, - CONSUME_TRANS_START_FUNC = 3, - CONSUME_TRANS_STOP_FUNC = 4, + MQ_REB_TRANS_START_FUNC = 3, + MQ_REB_TRANS_STOP_FUNC = 4, } ETrnFuncType; typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen); int32_t mndInitTrans(SMnode *pMnode); void mndCleanupTrans(SMnode *pMnode); +STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId); +void mndReleaseTrans(SMnode *pMnode, STrans *pTrans); STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const SRpcMsg *pReq); void mndTransDrop(STrans *pTrans); @@ -59,6 +61,7 @@ void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); void mndTransProcessRsp(SNodeMsg *pRsp); void mndTransPullup(SMnode *pMnode); +int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index e4ecdf328368d94d2a2d37b0ce054129a5ab8217..2140d0fa6750f98b5fd0a785a4a59564fe563922 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -24,9 +24,13 @@ extern "C" { int32_t mndInitUser(SMnode *pMnode); void mndCleanupUser(SMnode *pMnode); -SUserObj *mndAcquireUser(SMnode *pMnode, char *userName); +SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName); void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); +// for trans test +SSdbRaw *mndUserActionEncode(SUserObj *pUser); +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index f42829eddfffddabf55f96ad7ed830831f28378e..8aecc2245402be77a223ee5fd59884381865f07d 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -33,6 +33,7 @@ int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 168bd1db7694a047639a256dac49adb07494ff78..34d1223c0ae896e04776ae95cbe085f5d1b78a6c 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -453,7 +453,7 @@ static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *p colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); char buf[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(buf, pObj->pDnode->ep, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(buf, pObj->pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, buf, false); diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 57326b26ceb818fec893eabcadd10573c8182182..257db0dcd41a3af1d52cb5649155c79cabcbcc9e 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -194,7 +194,7 @@ static int32_t mndRetrieveClusters(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock colDataAppend(pColInfo, numOfRows, (const char*) &pCluster->id, false); char buf[tListLen(pCluster->name) + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(buf, pCluster->name, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(buf, pCluster->name, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, buf, false); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 025f61dc87694d2a6f872f6200470d83445e8b7c..23a87b4691388d7b17ef25f08d3120844d1ca7b1 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -35,7 +35,7 @@ #define MND_CONSUMER_LOST_HB_CNT 3 -static int8_t mqInRebFlag = 0; +static int8_t mqRebLock = 0; static const char *mndConsumerStatusName(int status); @@ -75,6 +75,17 @@ int32_t mndInitConsumer(SMnode *pMnode) { void mndCleanupConsumer(SMnode *pMnode) {} +bool mndRebTryStart() { + int8_t old = atomic_val_compare_exchange_8(&mqRebLock, 0, 1); + return old == 0; +} + +void mndRebEnd() { atomic_sub_fetch_8(&mqRebLock, 1); } + +void mndRebCntInc() { atomic_add_fetch_8(&mqRebLock, 1); } + +void mndRebCntDec() { atomic_sub_fetch_8(&mqRebLock, 1); } + static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg) { SMnode *pMnode = pMsg->pNode; SMqConsumerLostMsg *pLostMsg = pMsg->rpcMsg.pCont; @@ -123,15 +134,15 @@ FAIL: return -1; } -static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { - SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); +static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { + SMqRebInfo *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); if (pRebSub == NULL) { pRebSub = tNewSMqRebSubscribe(key); if (pRebSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebSubscribe)); + taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebInfo)); } return pRebSub; } @@ -143,8 +154,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { void *pIter = NULL; // rebalance cannot be parallel - int8_t old = atomic_val_compare_exchange_8(&mqInRebFlag, 0, 1); - if (old != 0) { + if (!mndRebTryStart()) { mInfo("mq rebalance already in progress, do nothing"); return 0; } @@ -152,7 +162,6 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg)); pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); // TODO set cleanfp - pRebMsg->mqInReb = &mqInRebFlag; // iterate all consumers, find all modification while (1) { @@ -180,7 +189,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { char key[TSDB_SUBSCRIBE_KEY_LEN]; char *removedTopic = taosArrayGetP(pConsumer->currentTopics, i); mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); - SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); + SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); } taosRUnLockLatch(&pConsumer->lock); @@ -191,7 +200,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { char key[TSDB_SUBSCRIBE_KEY_LEN]; char *newTopic = taosArrayGetP(pConsumer->rebNewTopics, i); mndMakeSubscribeKey(key, pConsumer->cgroup, newTopic); - SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); + SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId); } @@ -200,7 +209,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { char key[TSDB_SUBSCRIBE_KEY_LEN]; char *removedTopic = taosArrayGetP(pConsumer->rebRemovedTopics, i); mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); - SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); + SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); } taosRUnLockLatch(&pConsumer->lock); @@ -223,7 +232,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { taosHashCleanup(pRebMsg->rebSubHash); rpcFreeCont(pRebMsg); mTrace("mq rebalance finished, no modification"); - atomic_store_8(&mqInRebFlag, 0); + mndRebEnd(); } return 0; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 7e7e0b58a2a72df41d24e1e8408ccd46242ea894..af7efb5543e17c78b9a909565a1eaab0c4e08015 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -17,12 +17,16 @@ #include "mndDb.h" #include "mndAuth.h" #include "mndDnode.h" +#include "mndOffset.h" #include "mndShow.h" #include "mndSma.h" #include "mndStb.h" +#include "mndSubscribe.h" +#include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "systable.h" #define DB_VER_NUMBER 1 #define DB_RESERVE_SIZE 64 @@ -40,16 +44,17 @@ static int32_t mndProcessCompactDbReq(SNodeMsg *pReq); static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq); -static int32_t mndProcessGetIndexReq(SNodeMsg *pReq); int32_t mndInitDb(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_DB, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndDbActionEncode, - .decodeFp = (SdbDecodeFp)mndDbActionDecode, - .insertFp = (SdbInsertFp)mndDbActionInsert, - .updateFp = (SdbUpdateFp)mndDbActionUpdate, - .deleteFp = (SdbDeleteFp)mndDbActionDelete}; + SSdbTable table = { + .sdbType = SDB_DB, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndDbActionEncode, + .decodeFp = (SdbDecodeFp)mndDbActionDecode, + .insertFp = (SdbInsertFp)mndDbActionInsert, + .updateFp = (SdbUpdateFp)mndDbActionUpdate, + .deleteFp = (SdbDeleteFp)mndDbActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DB, mndProcessCreateDbReq); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_DB, mndProcessAlterDbReq); @@ -57,7 +62,6 @@ int32_t mndInitDb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_USE_DB, mndProcessUseDbReq); mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq); - mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetIndexReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb); @@ -84,33 +88,30 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT32(pRaw, dataPos, pDb->cfgVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->vgVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfVgroups, _OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize, _OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfStables, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.buffer, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.pageSize, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.pages, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep0, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep1, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep2, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.minRows, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRows, _OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.commitTime, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.fsyncPeriod, _OVER) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.ttl, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.walLevel, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.precision, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compression, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.replications, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.strict, _OVER) - SDB_SET_INT8(pRaw, dataPos, pDb->cfg.update, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLastRow, _OVER) - SDB_SET_INT8(pRaw, dataPos, pDb->cfg.streamMode, _OVER) - SDB_SET_INT8(pRaw, dataPos, pDb->cfg.singleSTable, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.hashMethod, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, _OVER) for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { TASSERT(taosArrayGetSize(pDb->cfg.pRetensions) == pDb->cfg.numOfRetensions); SRetention *pRetension = taosArrayGet(pDb->cfg.pRetensions, i); - SDB_SET_INT32(pRaw, dataPos, pRetension->freq, _OVER) - SDB_SET_INT32(pRaw, dataPos, pRetension->keep, _OVER) + SDB_SET_INT64(pRaw, dataPos, pRetension->freq, _OVER) + SDB_SET_INT64(pRaw, dataPos, pRetension->keep, _OVER) SDB_SET_INT8(pRaw, dataPos, pRetension->freqUnit, _OVER) SDB_SET_INT8(pRaw, dataPos, pRetension->keepUnit, _OVER) } @@ -158,26 +159,23 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pDb->cfgVersion, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->vgVersion, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfVgroups, _OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.cacheBlockSize, _OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.totalBlocks, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfStables, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.buffer, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.pageSize, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.pages, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysPerFile, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep0, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep1, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep2, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.minRows, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.maxRows, _OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.commitTime, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.fsyncPeriod, _OVER) - SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.ttl, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.walLevel, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.precision, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.compression, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.replications, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.strict, _OVER) - SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.update, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.cacheLastRow, _OVER) - SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.streamMode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.singleSTable, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.hashMethod, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfRetensions, _OVER) if (pDb->cfg.numOfRetensions > 0) { @@ -185,8 +183,8 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { if (pDb->cfg.pRetensions == NULL) goto _OVER; for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { SRetention retension = {0}; - SDB_GET_INT32(pRaw, dataPos, &retension.freq, _OVER) - SDB_GET_INT32(pRaw, dataPos, &retension.keep, _OVER) + SDB_GET_INT64(pRaw, dataPos, &retension.freq, _OVER) + SDB_GET_INT64(pRaw, dataPos, &retension.keep, _OVER) SDB_GET_INT8(pRaw, dataPos, &retension.freqUnit, _OVER) SDB_GET_INT8(pRaw, dataPos, &retension.keepUnit, _OVER) if (taosArrayPush(pDb->cfg.pRetensions, &retension) == NULL) { @@ -196,6 +194,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { } SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) + taosInitRWLatch(&pDb->lock); terrno = 0; @@ -224,17 +223,29 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { mTrace("db:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); taosWLockLatch(&pOld->lock); - SArray *pOldRetensions = pOld->cfg.pRetensions; pOld->updateTime = pNew->updateTime; pOld->cfgVersion = pNew->cfgVersion; pOld->vgVersion = pNew->vgVersion; - memcpy(&pOld->cfg, &pNew->cfg, sizeof(SDbCfg)); - pNew->cfg.pRetensions = pOldRetensions; + pOld->cfg.buffer = pNew->cfg.buffer; + pOld->cfg.pages = pNew->cfg.pages; + pOld->cfg.pageSize = pNew->cfg.pageSize; + pOld->cfg.daysPerFile = pNew->cfg.daysPerFile; + pOld->cfg.daysToKeep0 = pNew->cfg.daysToKeep0; + pOld->cfg.daysToKeep1 = pNew->cfg.daysToKeep1; + pOld->cfg.daysToKeep2 = pNew->cfg.daysToKeep2; + pOld->cfg.fsyncPeriod = pNew->cfg.fsyncPeriod; + pOld->cfg.walLevel = pNew->cfg.walLevel; + pOld->cfg.strict = pNew->cfg.strict; + pOld->cfg.cacheLastRow = pNew->cfg.cacheLastRow; + pOld->cfg.replications = pNew->cfg.replications; taosWUnLockLatch(&pOld->lock); return 0; } -static int32_t mndGetGlobalVgroupVersion(SMnode *pMnode) { return sdbGetTableVer(pMnode->pSdb, SDB_VGROUP); } +static inline int32_t mndGetGlobalVgroupVersion(SMnode *pMnode) { + SSdb *pSdb = pMnode->pSdb; + return sdbGetTableVer(pSdb, SDB_VGROUP); +} SDbObj *mndAcquireDb(SMnode *pMnode, const char *db) { SSdb *pSdb = pMnode->pSdb; @@ -268,10 +279,10 @@ static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1; - /* - if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1; - if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1; - */ + if (pCfg->numOfStables < TSDB_DB_STREAM_MODE_OFF || pCfg->numOfStables > TSDB_DB_STREAM_MODE_ON) return -1; + if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1; + if (pCfg->pageSize < TSDB_MIN_PAGESIZE_PER_VNODE || pCfg->pageSize > TSDB_MAX_PAGESIZE_PER_VNODE) return -1; + if (pCfg->pages < TSDB_MIN_PAGES_PER_VNODE || pCfg->pages > TSDB_MAX_PAGES_PER_VNODE) return -1; if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1; if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1; if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1; @@ -282,9 +293,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1; if (pCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1; if (pCfg->minRows > pCfg->maxRows) return -1; - if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) return -1; if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1; - // if (pCfg->ttl < TSDB_MIN_TABLE_TTL) return -1; if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1; if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1; if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1; @@ -292,36 +301,30 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; if (pCfg->strict > pCfg->replications) return -1; - if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) return -1; if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; - if (pCfg->streamMode < TSDB_DB_STREAM_MODE_OFF || pCfg->streamMode > TSDB_DB_STREAM_MODE_ON) return -1; - if (pCfg->singleSTable < TSDB_DB_SINGLE_STABLE_ON || pCfg->streamMode > TSDB_DB_SINGLE_STABLE_OFF) return -1; if (pCfg->hashMethod != 1) return -1; return TSDB_CODE_SUCCESS; } static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->numOfVgroups < 0) pCfg->numOfVgroups = TSDB_DEFAULT_VN_PER_DB; - if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; - if (pCfg->totalBlocks < 0) pCfg->totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS; - if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; + if (pCfg->numOfStables < 0) pCfg->numOfStables = TSDB_DEFAULT_DB_SINGLE_STABLE; + if (pCfg->buffer < 0) pCfg->buffer = TSDB_DEFAULT_BUFFER_PER_VNODE; + if (pCfg->pageSize < 0) pCfg->pageSize = TSDB_DEFAULT_PAGES_PER_VNODE; + if (pCfg->pages < 0) pCfg->pages = TSDB_MAX_PAGESIZE_PER_VNODE; + if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DURATION_PER_FILE; if (pCfg->daysToKeep0 < 0) pCfg->daysToKeep0 = TSDB_DEFAULT_KEEP; if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep0; if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep1; if (pCfg->minRows < 0) pCfg->minRows = TSDB_DEFAULT_MINROWS_FBLOCK; if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAXROWS_FBLOCK; - if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME; if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; - if (pCfg->ttl < 0) pCfg->ttl = TSDB_DEFAULT_TABLE_TTL; if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL; if (pCfg->precision < 0) pCfg->precision = TSDB_DEFAULT_PRECISION; if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL; if (pCfg->replications < 0) pCfg->replications = TSDB_DEFAULT_DB_REPLICA; if (pCfg->strict < 0) pCfg->strict = TSDB_DEFAULT_DB_STRICT; - if (pCfg->update < 0) pCfg->update = TSDB_DEFAULT_DB_UPDATE; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; - if (pCfg->streamMode < 0) pCfg->streamMode = TSDB_DEFAULT_DB_STREAM_MODE; - if (pCfg->singleSTable < 0) pCfg->singleSTable = TSDB_DEFAULT_DB_SINGLE_STABLE; if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0; } @@ -447,26 +450,23 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate memcpy(dbObj.createUser, pUser->user, TSDB_USER_LEN); dbObj.cfg = (SDbCfg){ .numOfVgroups = pCreate->numOfVgroups, - .cacheBlockSize = pCreate->cacheBlockSize, - .totalBlocks = pCreate->totalBlocks, + .numOfStables = pCreate->numOfStables, + .buffer = pCreate->buffer, + .pageSize = pCreate->pageSize, + .pages = pCreate->pages, .daysPerFile = pCreate->daysPerFile, .daysToKeep0 = pCreate->daysToKeep0, .daysToKeep1 = pCreate->daysToKeep1, .daysToKeep2 = pCreate->daysToKeep2, .minRows = pCreate->minRows, .maxRows = pCreate->maxRows, - .commitTime = pCreate->commitTime, .fsyncPeriod = pCreate->fsyncPeriod, - .ttl = pCreate->ttl, .walLevel = pCreate->walLevel, .precision = pCreate->precision, .compression = pCreate->compression, .replications = pCreate->replications, .strict = pCreate->strict, - .update = pCreate->update, .cacheLastRow = pCreate->cacheLastRow, - .streamMode = pCreate->streamMode, - .singleSTable = pCreate->singleSTable, .hashMethod = 1, }; @@ -570,8 +570,23 @@ _OVER: static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; - if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) { - pDb->cfg.totalBlocks = pAlter->totalBlocks; + if (pAlter->buffer >= 0 && pAlter->buffer != pDb->cfg.buffer) { + pDb->cfg.buffer = pAlter->buffer; + terrno = 0; + } + + if (pAlter->pages >= 0 && pAlter->pages != pDb->cfg.pages) { + pDb->cfg.pages = pAlter->pages; + terrno = 0; + } + + if (pAlter->pageSize >= 0 && pAlter->pageSize != pDb->cfg.pageSize) { + pDb->cfg.pageSize = pAlter->pageSize; + terrno = 0; + } + + if (pAlter->daysPerFile >= 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) { + pDb->cfg.daysPerFile = pAlter->daysPerFile; terrno = 0; } @@ -636,65 +651,7 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { - SAlterVnodeReq alterReq = {0}; - alterReq.vgVersion = pVgroup->version; - alterReq.totalBlocks = pDb->cfg.totalBlocks; - alterReq.daysToKeep0 = pDb->cfg.daysToKeep0; - alterReq.daysToKeep1 = pDb->cfg.daysToKeep1; - alterReq.daysToKeep2 = pDb->cfg.daysToKeep2; - alterReq.walLevel = pDb->cfg.walLevel; - alterReq.strict = pDb->cfg.strict; - alterReq.cacheLastRow = pDb->cfg.cacheLastRow; - alterReq.replica = pVgroup->replica; - alterReq.selfIndex = -1; - - for (int32_t v = 0; v < pVgroup->replica; ++v) { - SReplica *pReplica = &alterReq.replicas[v]; - SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; - SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pVgidDnode == NULL) { - return NULL; - } - - pReplica->id = pVgidDnode->id; - pReplica->port = pVgidDnode->port; - memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); - mndReleaseDnode(pMnode, pVgidDnode); - - if (pDnode->id == pVgid->dnodeId) { - alterReq.selfIndex = v; - } - } - - if (alterReq.selfIndex == -1) { - terrno = TSDB_CODE_MND_APP_ERROR; - return NULL; - } - - int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); - if (contLen < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - contLen += +sizeof(SMsgHead); - - void *pReq = taosMemoryMalloc(contLen); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - SMsgHead *pHead = pReq; - pHead->contLen = htonl(contLen); - pHead->vgId = htonl(pVgroup->vgId); - - tSerializeSAlterVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &alterReq); - *pContLen = contLen; - return pReq; -} - -static int32_t mndBuilAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { +static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { STransAction action = {0}; SVnodeGid *pVgid = pVgroup->vnodeGid + vn; @@ -730,7 +687,7 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * if (pIter == NULL) break; if (pVgroup->dbUid == pNew->uid) { - if (mndBuilAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { + if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -746,19 +703,19 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndAlterDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_ALTER_DB, &pReq->rpcMsg); - if (pTrans == NULL) goto UPDATE_DB_OVER; + if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to alter db:%s", pTrans->id, pOld->name); mndTransSetDbInfo(pTrans, pOld); - if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto UPDATE_DB_OVER; + if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; -UPDATE_DB_OVER: +_OVER: mndTransDrop(pTrans); return code; } @@ -772,7 +729,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { if (tDeserializeSAlterDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto ALTER_DB_OVER; + goto _OVER; } mDebug("db:%s, start to alter", alterReq.db); @@ -780,24 +737,26 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { pDb = mndAcquireDb(pMnode, alterReq.db); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto ALTER_DB_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto ALTER_DB_OVER; + goto _OVER; } if (mndCheckAlterDropCompactDbAuth(pUser, pDb) != 0) { - goto ALTER_DB_OVER; + goto _OVER; } SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); + dbObj.cfg.numOfRetensions = 0; + dbObj.cfg.pRetensions = NULL; code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); if (code != 0) { - goto ALTER_DB_OVER; + goto _OVER; } dbObj.cfgVersion++; @@ -805,7 +764,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { code = mndAlterDb(pMnode, pReq, pDb, &dbObj); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -ALTER_DB_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to alter since %s", alterReq.db, terrstr()); } @@ -825,36 +784,33 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { if (tDeserializeSDbCfgReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &cfgReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto GET_DB_CFG_OVER; + goto _OVER; } pDb = mndAcquireDb(pMnode, cfgReq.db); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto GET_DB_CFG_OVER; + goto _OVER; } cfgRsp.numOfVgroups = pDb->cfg.numOfVgroups; - cfgRsp.cacheBlockSize = pDb->cfg.cacheBlockSize; - cfgRsp.totalBlocks = pDb->cfg.totalBlocks; + cfgRsp.numOfStables = pDb->cfg.numOfStables; + cfgRsp.buffer = pDb->cfg.buffer; + cfgRsp.pageSize = pDb->cfg.pageSize; + cfgRsp.pages = pDb->cfg.pages; cfgRsp.daysPerFile = pDb->cfg.daysPerFile; cfgRsp.daysToKeep0 = pDb->cfg.daysToKeep0; cfgRsp.daysToKeep1 = pDb->cfg.daysToKeep1; cfgRsp.daysToKeep2 = pDb->cfg.daysToKeep2; cfgRsp.minRows = pDb->cfg.minRows; cfgRsp.maxRows = pDb->cfg.maxRows; - cfgRsp.commitTime = pDb->cfg.commitTime; cfgRsp.fsyncPeriod = pDb->cfg.fsyncPeriod; - cfgRsp.ttl = pDb->cfg.ttl; cfgRsp.walLevel = pDb->cfg.walLevel; cfgRsp.precision = pDb->cfg.precision; cfgRsp.compression = pDb->cfg.compression; cfgRsp.replications = pDb->cfg.replications; cfgRsp.strict = pDb->cfg.strict; - cfgRsp.update = pDb->cfg.update; cfgRsp.cacheLastRow = pDb->cfg.cacheLastRow; - cfgRsp.streamMode = pDb->cfg.streamMode; - cfgRsp.singleSTable = pDb->cfg.singleSTable; cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions; cfgRsp.pRetensions = pDb->cfg.pRetensions; @@ -863,7 +819,7 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; - goto GET_DB_CFG_OVER; + goto _OVER; } tSerializeSDbCfgRsp(pRsp, contLen, &cfgRsp); @@ -873,9 +829,9 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { code = 0; -GET_DB_CFG_OVER: +_OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0) { mError("db:%s, failed to get cfg since %s", cfgReq.db, terrstr()); } @@ -1027,6 +983,9 @@ static int32_t mndDropDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb) { if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER; + /*if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ + /*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ + /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; int32_t rspLen = 0; @@ -1091,7 +1050,8 @@ _OVER: return code; } -void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { +static int32_t mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode) { + int32_t numOfTables = 0; int32_t vindex = 0; SSdb *pSdb = pMnode->pSdb; @@ -1102,8 +1062,7 @@ void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { if (pIter == NULL) break; if (pVgroup->dbUid == pDb->uid) { - *num += pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT; - + numOfTables += pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT; vindex++; } @@ -1111,6 +1070,7 @@ void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { } sdbCancelFetch(pSdb, pIter); + return numOfTables; } static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { @@ -1164,8 +1124,7 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs return -1; } - int32_t numOfTable = 0; - mndGetDBTableNum(pDb, pMnode, &numOfTable); + int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable) { mndBuildDBVgroupInfo(pDb, pMnode, pRsp->pVgroupInfos); @@ -1189,23 +1148,22 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { if (tDeserializeSUseDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &usedbReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto USE_DB_OVER; + goto _OVER; } char *p = strchr(usedbReq.db, '.'); - if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) { + if (p && ((0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcmp(p + 1, TSDB_PERFORMANCE_SCHEMA_DB))))) { memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode); if (usedbReq.vgVersion < vgVersion) { usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); if (usedbRsp.pVgroupInfos == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto USE_DB_OVER; + goto _OVER; } mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); usedbRsp.vgVersion = vgVersion++; - } else { usedbRsp.vgVersion = usedbReq.vgVersion; } @@ -1226,15 +1184,15 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { } else { pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto USE_DB_OVER; + goto _OVER; } if (mndCheckUseDbAuth(pUser, pDb) != 0) { - goto USE_DB_OVER; + goto _OVER; } if (mndExtractDbInfo(pMnode, pDb, &usedbRsp, &usedbReq) < 0) { - goto USE_DB_OVER; + goto _OVER; } code = 0; @@ -1246,7 +1204,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; - goto USE_DB_OVER; + goto _OVER; } tSerializeSUseDbRsp(pRsp, contLen, &usedbRsp); @@ -1254,7 +1212,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { pReq->pRsp = pRsp; pReq->rspLen = contLen; -USE_DB_OVER: +_OVER: if (code != 0) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } @@ -1292,8 +1250,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, continue; } - int32_t numOfTable = 0; - mndGetDBTableNum(pDb, pMnode, &numOfTable); + int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable) { mDebug("db:%s, version & numOfTable not changed", pDbVgVersion->dbFName); @@ -1387,12 +1344,13 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in bool sysDb) { int32_t cols = 0; - char *buf = taosMemoryMalloc(pShow->bytes[cols]); + int32_t bytes = pShow->pMeta->pSchemas[cols].bytes; + char *buf = taosMemoryMalloc(bytes); const char *name = mndGetDbStr(pDb->name); if (name != NULL) { - STR_WITH_MAXSIZE_TO_VARSTR(buf, name, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(buf, name, bytes); } else { - STR_WITH_MAXSIZE_TO_VARSTR(buf, "NULL", pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(buf, "NULL", bytes); } char *status = "ready"; @@ -1430,7 +1388,6 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.replications, false); const char *src = pDb->cfg.strict ? "strict" : "nostrict"; - char b[9 + VARSTR_HEADER_SIZE] = {0}; STR_WITH_SIZE_TO_VARSTR(b, src, strlen(src)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)b, false); @@ -1453,10 +1410,13 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in colDataAppend(pColInfo, rows, (const char *)tmp, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheBlockSize, false); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.buffer, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.pageSize, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.totalBlocks, false); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.pages, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.minRows, false); @@ -1497,21 +1457,14 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)t, false); + // single stable model + int8_t m = 0; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.ttl, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.singleSTable, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.streamMode, false); + colDataAppend(pColInfo, rows, (const char *)&m, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols); colDataAppend(pColInfo, rows, (const char *)b, false); } - - // pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); - // *(int8_t *)pWrite = pDb->cfg.update; } static void setInformationSchemaDbCfg(SDbObj *pDbObj) { @@ -1522,7 +1475,6 @@ static void setInformationSchemaDbCfg(SDbObj *pDbObj) { pDbObj->cfg.numOfVgroups = 0; pDbObj->cfg.strict = 1; pDbObj->cfg.replications = 1; - pDbObj->cfg.update = 1; pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI; } @@ -1534,14 +1486,12 @@ static void setPerfSchemaDbCfg(SDbObj *pDbObj) { pDbObj->cfg.numOfVgroups = 0; pDbObj->cfg.strict = 1; pDbObj->cfg.replications = 1; - pDbObj->cfg.update = 1; pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI; } static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; int32_t *numOfTables = p1; - *numOfTables += pVgroup->numOfTables; return true; } @@ -1591,49 +1541,3 @@ static void mndCancelGetNextDb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } - -static int32_t mndProcessGetIndexReq(SNodeMsg *pReq) { - SUserIndexReq indexReq = {0}; - SMnode *pMnode = pReq->pNode; - int32_t code = -1; - SUserIndexRsp rsp = {0}; - bool exist = false; - - if (tDeserializeSUserIndexReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &indexReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - - code = mndProcessGetSmaReq(pMnode, &indexReq, &rsp, &exist); - if (code) { - goto _OVER; - } - - if (!exist) { - // TODO GET INDEX FROM FULLTEXT - code = -1; - terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; - } else { - int32_t contLen = tSerializeSUserIndexRsp(NULL, 0, &rsp); - void *pRsp = rpcMallocCont(contLen); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; - goto _OVER; - } - - tSerializeSUserIndexRsp(pRsp, contLen, &rsp); - - pReq->pRsp = pRsp; - pReq->rspLen = contLen; - - code = 0; - } - -_OVER: - if (code != 0) { - mError("failed to get index %s since %s", indexReq.indexFName, terrstr()); - } - - return code; -} diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 54f411d71f0aa5dd0f11dfec18112993cbc3dc6e..5bce6ac218e29c1bfda8a323af37af78c39c773a 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -241,7 +241,7 @@ int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) { void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp) { buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); - buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp)); + buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp)); #if 0 int32_t sz; buf = taosDecodeFixedI32(buf, &sz); @@ -277,6 +277,7 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); taosInitRWLatch(&pSubNew->lock); + pSubNew->dbUid = pSub->dbUid; pSubNew->subType = pSub->subType; pSubNew->withTbName = pSub->withTbName; pSubNew->withSchema = pSub->withSchema; @@ -310,6 +311,7 @@ void tDeleteSubscribeObj(SMqSubscribeObj *pSub) { int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { int32_t tlen = 0; tlen += taosEncodeString(buf, pSub->key); + tlen += taosEncodeFixedI64(buf, pSub->dbUid); tlen += taosEncodeFixedI32(buf, pSub->vgNum); tlen += taosEncodeFixedI8(buf, pSub->subType); tlen += taosEncodeFixedI8(buf, pSub->withTbName); @@ -336,6 +338,7 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) { // buf = taosDecodeStringTo(buf, pSub->key); + buf = taosDecodeFixedI64(buf, &pSub->dbUid); buf = taosDecodeFixedI32(buf, &pSub->vgNum); buf = taosDecodeFixedI8(buf, &pSub->subType); buf = taosDecodeFixedI8(buf, &pSub->withTbName); @@ -418,6 +421,9 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI32(pEncoder, pObj->version) < 0) return -1; if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; + if (tEncodeI32(pEncoder, pObj->triggerParam) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->waterMark) < 0) return -1; if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1; if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; @@ -464,6 +470,9 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; + if (tDecodeI32(pDecoder, &pObj->triggerParam) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->waterMark) < 0) return -1; if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 3eeec61dbbe059ffa2e69d3904e5ae8eac6d5a8d..a55cf412622fab0b18fbcde6d86b70cbfe11b92a 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -705,7 +705,7 @@ static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *p colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->id, false); char buf[tListLen(pDnode->ep) + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(buf, pDnode->ep, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(buf, pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, buf, false); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 09d05870192db3e494118d90265309b07c54d567..3ac2951b6f3c81975ba72a8616e6d7a081d4b633 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -314,7 +314,7 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) { goto _OVER; } - if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { + if (createReq.bufSize < 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; goto _OVER; } @@ -517,14 +517,14 @@ static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB cols = 0; char b1[tListLen(pFunc->name) + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(b1, pFunc->name, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(b1, pFunc->name, pShow->pMeta->pSchemas[cols].bytes); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)b1, false); if (pFunc->pComment) { - char *b2 = taosMemoryCalloc(1, pShow->bytes[cols]); - STR_WITH_MAXSIZE_TO_VARSTR(b2, pFunc->pComment, pShow->bytes[cols]); + char *b2 = taosMemoryCalloc(1, pShow->pMeta->pSchemas[cols].bytes); + STR_WITH_MAXSIZE_TO_VARSTR(b2, pFunc->pComment, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)b2, false); @@ -540,7 +540,7 @@ static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB char b3[TSDB_TYPE_STR_MAX_LEN] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(b3, mnodeGenTypeStr(buf, TSDB_TYPE_STR_MAX_LEN, pFunc->outputType, pFunc->outputLen), - pShow->bytes[cols]); + pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)b3, false); diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index 1e708ca321271788ce76554274dfb0d1216a9bda..c9bccaf1d02117dc43a7c631605543243bd6bb3b 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -14,256 +14,10 @@ */ #define _DEFAULT_SOURCE -#include "mndInfoSchema.h" +#include "systable.h" #include "mndInt.h" -#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) -#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) -#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) - -static const SInfosTableSchema dnodesSchema[] = { - {.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, - {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, - {.name = "max_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, - {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, -}; - -static const SInfosTableSchema mnodesSchema[] = { - {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, -}; - -static const SInfosTableSchema modulesSchema[] = { - {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "endpoint", .bytes = 134 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "module", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, -}; - -static const SInfosTableSchema qnodesSchema[] = { - {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, -}; - -static const SInfosTableSchema snodesSchema[] = { - {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, -}; - -static const SInfosTableSchema bnodesSchema[] = { - {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, -}; - -static const SInfosTableSchema clusterSchema[] = { - {.name = "id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, -}; - -static const SInfosTableSchema userDBSchema[] = { - {.name = "name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, - {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "strict", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "wal", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "stream_mode", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - // {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // disable update -}; - -static const SInfosTableSchema userFuncSchema[] = { - {.name = "name", .bytes = TSDB_FUNC_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "comment", .bytes = PATH_MAX - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "aggregate", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "output_type", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "code_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "bufsize", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, -}; - -static const SInfosTableSchema userIdxSchema[] = { - {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "index_database", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "column_name", .bytes = SYSTABLE_SCH_COL_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_VARCHAR}, -}; - -static const SInfosTableSchema userStbsSchema[] = { - {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, -}; - -static const SInfosTableSchema userTblsSchema[] = { - {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "table_comment", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "type", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, -}; - -static const SInfosTableSchema userTblDistSchema[] = { - {.name = "db_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "distributed_histogram", .bytes = 500 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "stddev_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "storage_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "compression_ratio", .bytes = 8, .type = TSDB_DATA_TYPE_DOUBLE}, - {.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, -}; - -static const SInfosTableSchema userUsersSchema[] = { - {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, -}; - -static const SInfosTableSchema grantsSchema[] = { - {.name = "version", .bytes = 8 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "expire time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "storage(GB)", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "databases", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "users", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "accounts", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "connections", .bytes = 11 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "streams", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "cpu cores", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "speed(PPS)", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "querytime", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, -}; - -static const SInfosTableSchema vgroupsSchema[] = { - {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "v1_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "v2_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "v3_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, -}; - -static const SInfosTableSchema smaSchema[] = { - {.name = "sma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, -}; - -static const SInfosTableSchema transSchema[] = { - {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "stage", .bytes = TSDB_TRANS_STAGE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "type", .bytes = TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "last_exec_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "last_error", .bytes = (TSDB_TRANS_ERROR_LEN - 1) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, -}; - -static const SInfosTableSchema configSchema[] = { - {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "value", .bytes = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, -}; - -static const SInfosTableSchema connSchema[] = { - {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "program", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "ip:port", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "login_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, -}; - -static const SInfosTableSchema querySchema[] = { - {.name = "queryId", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "ip:port", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "qid", .bytes = 22 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "time", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "sql_obj_id", .bytes = QUERY_OBJ_ID_SIZE + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "ep", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "stable_query", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, - {.name = "sub_queries", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "sub_query_info", .bytes = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, -}; - -static const SInfosTableMeta infosMeta[] = { - {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)}, - {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)}, - {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)}, - {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)}, - {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, - {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, - {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema)}, - {TSDB_INS_TABLE_USER_DATABASES, userDBSchema, tListLen(userDBSchema)}, - {TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)}, - {TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)}, - {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)}, - {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)}, - {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)}, - {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)}, - {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)}, - {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)}, - {TSDB_INS_TABLE_TRANS, transSchema, tListLen(transSchema)}, - {TSDB_INS_TABLE_SMAS, smaSchema, tListLen(smaSchema)}, - {TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema)}, - {TSDB_INS_TABLE_CONNS, connSchema, tListLen(connSchema)}, - {TSDB_INS_TABLE_QUERIES, querySchema, tListLen(querySchema)}, -}; - -static int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, SSchema **pDst) { +static int32_t mndInitInfosTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) { SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema)); if (NULL == schema) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -289,11 +43,15 @@ static int32_t mndInsInitMeta(SHashObj *hash) { meta.sversion = 1; meta.tversion = 1; - for (int32_t i = 0; i < tListLen(infosMeta); ++i) { - tstrncpy(meta.tbName, infosMeta[i].name, sizeof(meta.tbName)); - meta.numOfColumns = infosMeta[i].colNum; + size_t size = 0; + const SSysTableMeta* pInfosTableMeta = NULL; + getInfosDbMeta(&pInfosTableMeta, &size); + + for (int32_t i = 0; i < size; ++i) { + tstrncpy(meta.tbName, pInfosTableMeta[i].name, sizeof(meta.tbName)); + meta.numOfColumns = pInfosTableMeta[i].colNum; - if (mndInitInfosTableSchema(infosMeta[i].schema, infosMeta[i].colNum, &meta.pSchemas)) { + if (mndInitInfosTableSchema(pInfosTableMeta[i].schema, pInfosTableMeta[i].colNum, &meta.pSchemas)) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index b51c545a6d2739014275a58396054c5b96945836..b7d73581105a2ed8cec123d53824d06b18155e24 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -619,14 +619,14 @@ static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *p colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); char b1[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(b1, pObj->pDnode->ep, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(b1, pObj->pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, b1, false); const char *roles = syncStr(pObj->role); - char *b2 = taosMemoryCalloc(1, strlen(roles) + VARSTR_HEADER_SIZE); - STR_WITH_MAXSIZE_TO_VARSTR(b2, roles, pShow->bytes[cols]); + char *b2 = taosMemoryCalloc(1, 12 + VARSTR_HEADER_SIZE); + STR_WITH_MAXSIZE_TO_VARSTR(b2, roles, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)b2, false); diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index f07534870497c2d8221be2ef784c3523e5334d52..e0ba4076f59fdb1cacbae03333ef7c4a0af1270f 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -231,3 +231,36 @@ static void mndCancelGetNextOffset(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +static int32_t mndSetDropOffsetCommitLogs(SMnode *pMnode, STrans *pTrans, SMqOffsetObj *pOffset) { + SSdbRaw *pCommitRaw = mndOffsetActionEncode(pOffset); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + +int32_t mndDropOffsetByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + int32_t code = -1; + SSdb *pSdb = pMnode->pSdb; + + void *pIter = NULL; + SMqOffsetObj *pOffset = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pOffset); + if (pIter == NULL) break; + + if (pOffset->dbUid != pDb->uid) { + sdbRelease(pSdb, pOffset); + continue; + } + + if (mndSetDropOffsetCommitLogs(pMnode, pTrans, pOffset) < 0) { + goto END; + } + } + + code = 0; +END: + return code; +} diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c index e5374330ad1a1fff1d2026331ce44412620c6062..d7b2e3ec24a89c0a4576753f1e6343cffff9ce02 100644 --- a/source/dnode/mnode/impl/src/mndPerfSchema.c +++ b/source/dnode/mnode/impl/src/mndPerfSchema.c @@ -14,92 +14,11 @@ */ #define _DEFAULT_SOURCE -#include "mndPerfSchema.h" #include "mndInt.h" - -//!!!! Note: only APPEND columns in below tables, NO insert !!!! -static const SPerfsTableSchema connectionsSchema[] = { - {.name = "conn_id", .bytes = 4, .type = TSDB_DATA_TYPE_UINT}, - {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "program", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "login_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, -}; -static const SPerfsTableSchema queriesSchema[] = { - {.name = "query_id", .bytes = 4, .type = TSDB_DATA_TYPE_UBIGINT}, - {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "fqdn", .bytes = TSDB_FQDN_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "exec_time", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "sub_queries", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "sub_query_info", .bytes = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, -}; - -static const SPerfsTableSchema topicSchema[] = { - {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - // TODO config -}; - -static const SPerfsTableSchema consumerSchema[] = { - {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "topics", .bytes = TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "subscribe_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "rebalance_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, -}; - -static const SPerfsTableSchema subscriptionSchema[] = { - {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, -}; - -static const SPerfsTableSchema offsetSchema[] = { - {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "committed_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "current_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "skip_log_cnt", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, -}; - -static const SPerfsTableSchema streamSchema[] = { - {.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "source_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "target_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "target_table", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "watermark", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "trigger", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, -}; - -static const SPerfsTableMeta perfsMeta[] = { - {TSDB_PERFS_TABLE_CONNECTIONS, connectionsSchema, tListLen(connectionsSchema)}, - {TSDB_PERFS_TABLE_QUERIES, queriesSchema, tListLen(queriesSchema)}, - {TSDB_PERFS_TABLE_TOPICS, topicSchema, tListLen(topicSchema)}, - {TSDB_PERFS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)}, - {TSDB_PERFS_TABLE_SUBSCRIPTIONS, subscriptionSchema, tListLen(subscriptionSchema)}, - {TSDB_PERFS_TABLE_OFFSETS, offsetSchema, tListLen(offsetSchema)}, - {TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)}, -}; +#include "systable.h" // connection/application/ -int32_t mndInitPerfsTableSchema(const SPerfsTableSchema *pSrc, int32_t colNum, SSchema **pDst) { +int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) { SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema)); if (NULL == schema) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -126,11 +45,15 @@ int32_t mndPerfsInitMeta(SHashObj *hash) { meta.sversion = 1; meta.tversion = 1; - for (int32_t i = 0; i < tListLen(perfsMeta); ++i) { - strcpy(meta.tbName, perfsMeta[i].name); - meta.numOfColumns = perfsMeta[i].colNum; + size_t size = 0; + const SSysTableMeta* pSysDbTableMeta = NULL; + getPerfDbMeta(&pSysDbTableMeta, &size); + + for (int32_t i = 0; i < size; ++i) { + strcpy(meta.tbName, pSysDbTableMeta[i].name); + meta.numOfColumns = pSysDbTableMeta[i].colNum; - if (mndInitPerfsTableSchema(perfsMeta[i].schema, perfsMeta[i].colNum, &meta.pSchemas)) { + if (mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas)) { return -1; } @@ -166,7 +89,6 @@ int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char } memcpy(pRsp->pSchemas, meta->pSchemas, meta->numOfColumns * sizeof(SSchema)); - return 0; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 7bd22c2de53b17b881b49f70e06bbe7872200a5c..2de337537ffd3d0b64df3840466851668b2ff1a0 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -403,6 +403,16 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb SKv *kv = pIter; switch (kv->key) { + case HEARTBEAT_KEY_USER_AUTHINFO: { + void * rspMsg = NULL; + int32_t rspLen = 0; + mndValidateUserAuthInfo(pMnode, kv->value, kv->valueLen / sizeof(SUserAuthVersion), &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv1 = {.key = HEARTBEAT_KEY_USER_AUTHINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); + } + break; + } case HEARTBEAT_KEY_DBINFO: { void * rspMsg = NULL; int32_t rspLen = 0; @@ -570,38 +580,39 @@ static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int if (pConn == NULL) break; cols = 0; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; +#if 0 + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; *(uint32_t *)pWrite = pConn->id; cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->user, pShow->bytes[cols]); + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->user, pShow->pMeta->pSchemas[cols].bytes); cols++; // app name - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->app, pShow->bytes[cols]); + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->app, pShow->pMeta->pSchemas[cols].bytes); cols++; // app pid - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; *(int32_t *)pWrite = pConn->pid; cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; taosIpPort2String(pConn->ip, pConn->port, ipStr); - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->pMeta->pSchemas[cols].bytes); cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; *(int64_t *)pWrite = pConn->loginTimeMs; cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; if (pConn->lastAccessTimeMs < pConn->loginTimeMs) pConn->lastAccessTimeMs = pConn->loginTimeMs; *(int64_t *)pWrite = pConn->lastAccessTimeMs; cols++; +#endif numOfRows++; } @@ -643,67 +654,67 @@ static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, i SQueryDesc *pDesc = pConn->pQueries + i; cols = 0; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; *(int64_t *)pWrite = htobe64(pDesc->queryId); cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; *(int64_t *)pWrite = htobe64(pConn->id); cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->user, pShow->bytes[cols]); + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->user, pShow->pMeta->pSchemas[cols].bytes); cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; snprintf(str, tListLen(str), "%s:%u", taosIpStr(pConn->ip), pConn->port); - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->pMeta->pSchemas[cols].bytes); cols++; char handleBuf[24] = {0}; snprintf(handleBuf, tListLen(handleBuf), "%" PRIu64, htobe64(pDesc->qId)); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, handleBuf, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, handleBuf, pShow->pMeta->pSchemas[cols].bytes); cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; *(int64_t *)pWrite = htobe64(pDesc->stime); cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; *(int64_t *)pWrite = htobe64(pDesc->useconds); cols++; snprintf(str, tListLen(str), "0x%" PRIx64, htobe64(pDesc->sqlObjId)); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->bytes[cols]); + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->pMeta->pSchemas[cols].bytes); cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; *(int32_t *)pWrite = htonl(pDesc->pid); cols++; char epBuf[TSDB_EP_LEN + 1] = {0}; snprintf(epBuf, tListLen(epBuf), "%s:%u", pDesc->fqdn, pConn->port); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, epBuf, pShow->bytes[cols]); + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, epBuf, pShow->pMeta->pSchemas[cols].bytes); cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; *(bool *)pWrite = pDesc->stableQuery; cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; *(int32_t *)pWrite = htonl(pDesc->numOfSub); cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->subSqlInfo, pShow->bytes[cols]); + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->subSqlInfo, pShow->pMeta->pSchemas[cols].bytes); cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->sql, pShow->bytes[cols]); + pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->sql, pShow->pMeta->pSchemas[cols].bytes); cols++; numOfRows++; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index cec5933ba32b2e2f3fbcfb8a5ab48a4f8109ad18..ae223c34b48254a4e9b75b87d236661174c10cca 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -517,7 +517,7 @@ static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *p colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); char ep[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(ep, pObj->pDnode->ep, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(ep, pObj->pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)ep, false); diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index d184e354c44223cf7bf2e99e022caa45dd9476fc..7aa55e21091df78f11475a7d245507d15eeb80fa 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -20,7 +20,7 @@ int32_t mndProcessQueryMsg(SNodeMsg *pReq) { SMnode *pMnode = pReq->pNode; - SReadHandle handle = {0}; + SReadHandle handle = {.mnd = pMnode}; mTrace("msg:%p, in query queue is processing", pReq); switch (pReq->rpcMsg.msgType) { diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 3f4b2fe145ea5121cf42a56275daf285de16279d..9274d35b8b5d4746bc474e61c4afabee87a01842 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -204,6 +204,8 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p pTask->smaSink.smaId = pStream->smaId; } else { pTask->sinkType = TASK_SINK__TABLE; + pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); + ASSERT(pTask->tbSink.pSchemaWrapper); } // dispatch @@ -242,6 +244,7 @@ int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStr pTask->smaSink.smaId = pStream->smaId; } else { pTask->sinkType = TASK_SINK__TABLE; + pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); } // // dispatch @@ -308,8 +311,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // sink part if (level == 0) { // only for inplace - pTask->sinkType = TASK_SINK__SHOW; - pTask->showSink.reserved = 0; + pTask->sinkType = TASK_SINK__NONE; if (!hasExtraSink) { #if 1 if (pStream->createdBy == STREAM_CREATED_BY__SMA) { @@ -317,6 +319,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { pTask->smaSink.smaId = pStream->smaId; } else { pTask->sinkType = TASK_SINK__TABLE; + pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); } #endif } @@ -368,8 +371,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { pTask->sourceType = TASK_SOURCE__PIPE; // sink part - pTask->sinkType = TASK_SINK__SHOW; - /*pTask->sinkType = TASK_SINK__NONE;*/ + pTask->sinkType = TASK_SINK__NONE; // dispatch part ASSERT(hasExtraSink); @@ -456,7 +458,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { pTask->sourceType = TASK_SOURCE__MERGE; // sink part - pTask->sinkType = TASK_SINK__SHOW; + pTask->sinkType = TASK_SINK__NONE; // dispatch part pTask->dispatchType = TASK_DISPATCH__NONE; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 94366a241d697025f0d0682d8b2bab24df12ad76..be333d154a4b39d2cf7fd9f65878663a03dda184 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndShow.h" +#include "systable.h" #define SHOW_STEP_SIZE 100 @@ -47,7 +48,7 @@ void mndCleanupShow(SMnode *pMnode) { } } -static int32_t convertToRetrieveType(char* name, int32_t len) { +static int32_t convertToRetrieveType(char *name, int32_t len) { int32_t type = -1; if (strncasecmp(name, TSDB_INS_TABLE_DNODES, len) == 0) { @@ -72,8 +73,6 @@ static int32_t convertToRetrieveType(char* name, int32_t len) { // type = TSDB_MGMT_TABLE_INDEX; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, len) == 0) { type = TSDB_MGMT_TABLE_STB; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, len) == 0) { - type = TSDB_MGMT_TABLE_STREAMS; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) { type = TSDB_MGMT_TABLE_TABLE; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, len) == 0) { @@ -84,26 +83,28 @@ static int32_t convertToRetrieveType(char* name, int32_t len) { type = TSDB_MGMT_TABLE_GRANTS; } else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, len) == 0) { type = TSDB_MGMT_TABLE_VGROUP; - } else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, len) == 0) { + } else if (strncasecmp(name, TSDB_PERFS_TABLE_CONSUMERS, len) == 0) { type = TSDB_MGMT_TABLE_CONSUMERS; - } else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, len) == 0) { + } else if (strncasecmp(name, TSDB_PERFS_TABLE_SUBSCRIBES, len) == 0) { type = TSDB_MGMT_TABLE_SUBSCRIBES; - } else if (strncasecmp(name, TSDB_INS_TABLE_TRANS, len) == 0) { + } else if (strncasecmp(name, TSDB_PERFS_TABLE_TRANS, len) == 0) { type = TSDB_MGMT_TABLE_TRANS; - } else if (strncasecmp(name, TSDB_INS_TABLE_SMAS, len) == 0) { + } else if (strncasecmp(name, TSDB_PERFS_TABLE_SMAS, len) == 0) { type = TSDB_MGMT_TABLE_SMAS; } else if (strncasecmp(name, TSDB_INS_TABLE_CONFIGS, len) == 0) { type = TSDB_MGMT_TABLE_CONFIGS; - } else if (strncasecmp(name, TSDB_INS_TABLE_CONNS, len) == 0) { + } else if (strncasecmp(name, TSDB_PERFS_TABLE_CONNECTIONS, len) == 0) { type = TSDB_MGMT_TABLE_CONNS; - } else if (strncasecmp(name, TSDB_INS_TABLE_QUERIES, len) == 0) { + } else if (strncasecmp(name, TSDB_PERFS_TABLE_QUERIES, len) == 0) { type = TSDB_MGMT_TABLE_QUERIES; - } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, len) == 0) { + } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, len) == 0) { type = TSDB_MGMT_TABLE_VNODES; } else if (strncasecmp(name, TSDB_PERFS_TABLE_TOPICS, len) == 0) { type = TSDB_MGMT_TABLE_TOPICS; + } else if (strncasecmp(name, TSDB_PERFS_TABLE_STREAMS, len) == 0) { + type = TSDB_MGMT_TABLE_STREAMS; } else { -// ASSERT(0); + // ASSERT(0); } return type; @@ -115,12 +116,12 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) { int64_t showId = atomic_add_fetch_64(&pMgmt->showId, 1); if (showId == 0) atomic_add_fetch_64(&pMgmt->showId, 1); - int32_t size = sizeof(SShowObj); + int32_t size = sizeof(SShowObj); SShowObj showObj = {0}; - showObj.id = showId; + showObj.id = showId; showObj.pMnode = pMnode; - showObj.type = convertToRetrieveType(pReq->tb, tListLen(pReq->tb)); + showObj.type = convertToRetrieveType(pReq->tb, tListLen(pReq->tb)); memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN); int32_t keepTime = tsShellActivityTimer * 6 * 1000; @@ -208,16 +209,6 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) { pShow->pMeta = pMeta; pShow->numOfColumns = pShow->pMeta->numOfColumns; - int32_t offset = 0; - - for (int32_t i = 0; i < pShow->pMeta->numOfColumns; ++i) { - pShow->offset[i] = offset; - - int32_t bytes = pShow->pMeta->pSchemas[i].bytes; - pShow->rowSize += bytes; - pShow->bytes[i] = bytes; - offset += bytes; - } } else { pShow = mndAcquireShowObj(pMnode, retrieveReq.showId); if (pShow == NULL) { diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 0e3e8b68ccd23dfd408b92dfc3fd306716a32a3c..8619df978baf436d9ac213768189d202be4b5696 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -40,6 +40,7 @@ static int32_t mndProcessMCreateSmaReq(SNodeMsg *pReq); static int32_t mndProcessMDropSmaReq(SNodeMsg *pReq); static int32_t mndProcessVCreateSmaRsp(SNodeMsg *pRsp); static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp); +static int32_t mndProcessGetSmaReq(SNodeMsg *pReq); static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); @@ -56,6 +57,7 @@ int32_t mndInitSma(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessMDropSmaReq); mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndProcessVCreateSmaRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndProcessVDropSmaRsp); + mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetSmaReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveSma); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelGetNextSma); @@ -686,7 +688,7 @@ _OVER: return code; } -int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) { +static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) { int32_t code = -1; SSmaObj *pSma = NULL; @@ -715,6 +717,51 @@ int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexR } mndReleaseSma(pMnode, pSma); + return code; +} + +static int32_t mndProcessGetSmaReq(SNodeMsg *pReq) { + SUserIndexReq indexReq = {0}; + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SUserIndexRsp rsp = {0}; + bool exist = false; + + if (tDeserializeSUserIndexReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &indexReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + code = mndGetSma(pMnode, &indexReq, &rsp, &exist); + if (code) { + goto _OVER; + } + + if (!exist) { + // TODO GET INDEX FROM FULLTEXT + code = -1; + terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; + } else { + int32_t contLen = tSerializeSUserIndexRsp(NULL, 0, &rsp); + void *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = -1; + goto _OVER; + } + + tSerializeSUserIndexRsp(pRsp, contLen, &rsp); + + pReq->pRsp = pRsp; + pReq->rspLen = contLen; + + code = 0; + } + +_OVER: + if (code != 0) { + mError("failed to get index %s since %s", indexReq.indexFName, terrstr()); + } return code; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index b7d0ed8f5afa7558e72fc43a8645dba15b2a4892..58db4772e783fc74c9e2d4ef9119e60ae84444f4 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -463,7 +463,7 @@ static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *p colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); char ep[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(ep, pObj->pDnode->ep, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(ep, pObj->pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)ep, false); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 41bcef25d1b671b60f1c0e1311e025783e1ac5a2..8ae0d5d19c83e78de136b887ccf5f2c1a2175428 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -428,17 +428,23 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt } static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { - SName name = {0}; + SName name = {0}; + SVDropStbReq req = {0}; + int32_t contLen = 0; + int32_t ret = 0; + SMsgHead *pHead = NULL; + SCoder coder = {0}; + tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - SVDropTbReq req = {0}; - req.ver = 0; req.name = (char *)tNameGetTableName(&name); - req.type = TD_SUPER_TABLE; req.suid = pStb->uid; - int32_t contLen = tSerializeSVDropTbReq(NULL, &req) + sizeof(SMsgHead); - SMsgHead *pHead = taosMemoryMalloc(contLen); + tEncodeSize(tEncodeSVDropStbReq, &req, contLen, ret); + if (ret < 0) return NULL; + + contLen += sizeof(SMsgHead); + pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -448,7 +454,10 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, pHead->vgId = htonl(pVgroup->vgId); void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); - tSerializeSVDropTbReq(&pBuf, &req); + + tCoderInit(&coder, TD_LITTLE_ENDIAN, pBuf, contLen - sizeof(SMsgHead), TD_ENCODER); + tEncodeSVDropStbReq(&coder, &req); + tCoderClear(&coder); *pContLen = contLen; return pHead; @@ -670,8 +679,8 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre memcpy(stbObj.pAst2, pCreate->pAst2, stbObj.ast2Len); } - stbObj.pColumns = taosMemoryMalloc(stbObj.numOfColumns * sizeof(SSchema)); - stbObj.pTags = taosMemoryMalloc(stbObj.numOfTags * sizeof(SSchema)); + stbObj.pColumns = taosMemoryCalloc(1, stbObj.numOfColumns * sizeof(SSchema)); + stbObj.pTags = taosMemoryCalloc(1, stbObj.numOfTags * sizeof(SSchema)); if (stbObj.pColumns == NULL || stbObj.pTags == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -722,7 +731,6 @@ _OVER: static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { SMnode *pMnode = pReq->pNode; int32_t code = -1; - SStbObj *pTopicStb = NULL; SStbObj *pStb = NULL; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -753,12 +761,6 @@ static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { goto _OVER; } - pTopicStb = mndAcquireStb(pMnode, createReq.name); - if (pTopicStb != NULL) { - terrno = TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC; - goto _OVER; - } - pDb = mndAcquireDbByStb(pMnode, createReq.name); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -774,6 +776,13 @@ static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { goto _OVER; } + int32_t numOfStbs = -1; + mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs); + if (pDb->cfg.numOfStables == 1 && numOfStbs != 0) { + terrno = TSDB_CODE_MND_SINGLE_STB_MODE_DB; + goto _OVER; + } + code = mndCreateStb(pMnode, pReq, &createReq, pDb); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -783,7 +792,6 @@ _OVER: } mndReleaseStb(pMnode, pStb); - mndReleaseStb(pMnode, pTopicStb); mndReleaseDb(pMnode, pDb); mndReleaseUser(pMnode, pUser); tFreeSMCreateStbReq(&createReq); @@ -1095,7 +1103,7 @@ static int32_t mndSetAlterStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); if (pRedoRaw == NULL) return -1; if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_UPDATING) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; return 0; } @@ -1427,7 +1435,6 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa pRsp->numOfColumns = pStb->numOfColumns; pRsp->precision = pDb->cfg.precision; pRsp->tableType = TSDB_SUPER_TABLE; - pRsp->update = pDb->cfg.update; pRsp->sversion = pStb->version; pRsp->suid = pStb->uid; pRsp->tuid = pStb->uid; @@ -1580,7 +1587,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int return 0; } -static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) { +int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) { SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); if (pDb == NULL) { diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index db5c725f5e2cafd77e52c08211184d577794cd41..5c6e2ce77114e30b94f4176ccb8fd02f93cb49ea 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -308,6 +308,8 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe streamObj.smaId = 0; /*streamObj.physicalPlan = "";*/ streamObj.logicalPlan = "not implemented"; + streamObj.trigger = pCreate->triggerType; + streamObj.waterMark = pCreate->watermark; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); if (pTrans == NULL) { @@ -431,7 +433,7 @@ static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *p SStreamObj *pStream = NULL; while (numOfRows < rows) { - pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pStream); + pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream); if (pShow->pIter == NULL) break; SColumnInfoData *pColInfo; @@ -471,8 +473,13 @@ static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *p pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->trigger, false); + + numOfRows++; + sdbRelease(pSdb, pStream); } - return 0; + + pShow->numOfRows += numOfRows; + return numOfRows; } static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index aeecab34cb0e61a3251dc7506ec5b174a69109d9..545caea03d350366f6849cc1807519c3849951fe 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -80,6 +80,7 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pSub->dbUid = pTopic->dbUid; pSub->subType = pTopic->subType; pSub->withTbName = pTopic->withTbName; pSub->withSchema = pTopic->withSchema; @@ -144,6 +145,10 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SM int32_t vgId = pRebVg->pVgEp->vgId; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); + if (pVgObj == NULL) { + taosMemoryFree(buf); + return -1; + } STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgObj); @@ -170,27 +175,20 @@ static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup) return 0; } -static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { - SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); +static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { + SMqRebInfo *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); if (pRebSub == NULL) { pRebSub = tNewSMqRebSubscribe(key); if (pRebSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebSubscribe)); + taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebInfo)); } return pRebSub; } static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { - if (pInput->pTopic != NULL) { - // create subscribe - pOutput->pSub = mndCreateSub(pMnode, pInput->pTopic, pInput->pRebInfo->key); - ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) == 0); - } else { - pOutput->pSub = tCloneSubscribeObj(pInput->pOldSub); - } int32_t totalVgNum = pOutput->pSub->vgNum; mInfo("mq rebalance subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum); @@ -241,12 +239,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } // 3. calc vg number of each consumer - int32_t oldSz = 0; - if (pInput->pOldSub) { - oldSz = taosHashGetSize(pInput->pOldSub->consumerHash); - } - int32_t afterRebConsumerNum = - oldSz + taosArrayGetSize(pInput->pRebInfo->newConsumers) - taosArrayGetSize(pInput->pRebInfo->removedConsumers); + int32_t afterRebConsumerNum = pInput->oldConsumerNum + taosArrayGetSize(pInput->pRebInfo->newConsumers) - + taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t minVgCnt = 0; int32_t imbConsumerNum = 0; // calc num @@ -452,7 +446,10 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SNodeMsg *pMsg, const SMqRebO } // 4. TODO commit log: modification log - // 5. execution + // 5. set cb + mndTransSetCb(pTrans, MQ_REB_TRANS_START_FUNC, MQ_REB_TRANS_STOP_FUNC, NULL, 0); + + // 6. execution if (mndTransPrepare(pMnode, pTrans) != 0) goto REB_FAIL; mndTransDrop(pTrans); @@ -481,22 +478,34 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) { rebOutput.touchedConsumers = taosArrayInit(0, sizeof(void *)); rebOutput.rebVgs = taosArrayInit(0, sizeof(SMqRebOutputVg)); - SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter; - SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); + SMqRebInfo *pRebInfo = (SMqRebInfo *)pIter; + SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebInfo->key); + + rebInput.pRebInfo = pRebInfo; if (pSub == NULL) { // split sub key and extract topic char topic[TSDB_TOPIC_FNAME_LEN]; char cgroup[TSDB_CGROUP_LEN]; - mndSplitSubscribeKey(pRebSub->key, topic, cgroup); + mndSplitSubscribeKey(pRebInfo->key, topic, cgroup); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); ASSERT(pTopic); taosRLockLatch(&pTopic->lock); - rebInput.pTopic = pTopic; - } - rebInput.pRebInfo = pRebSub; - rebInput.pOldSub = pSub; + rebOutput.pSub = mndCreateSub(pMnode, pTopic, pRebInfo->key); + ASSERT(taosHashGetSize(rebOutput.pSub->consumerHash) == 0); + + taosRUnLockLatch(&pTopic->lock); + mndReleaseTopic(pMnode, pTopic); + + rebInput.oldConsumerNum = 0; + } else { + taosRLockLatch(&pSub->lock); + rebInput.oldConsumerNum = taosHashGetSize(pSub->consumerHash); + rebOutput.pSub = tCloneSubscribeObj(pSub); + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); + } // TODO replace assert with error check ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0); @@ -506,21 +515,15 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) { /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/ // TODO replace assert with error check - ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0); - - if (rebInput.pTopic) { - SMqTopicObj *pTopic = (SMqTopicObj *)rebInput.pTopic; - taosRUnLockLatch(&pTopic->lock); - mndReleaseTopic(pMnode, pTopic); - } else { - mndReleaseSubscribe(pMnode, pSub); + if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) { + mError("persist rebalance output error, possibly vnode splitted or dropped"); } } // reset flag - atomic_store_8(pReq->mqInReb, 0); mInfo("mq rebalance completed successfully"); taosHashCleanup(pReq->rebSubHash); + mndRebEnd(); return 0; } @@ -584,7 +587,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t tlen; SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER); - buf = taosMemoryMalloc(tlen + 1); + buf = taosMemoryMalloc(tlen); if (buf == NULL) goto SUB_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER); @@ -670,3 +673,36 @@ static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } + +static int32_t mndSetDropSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { + SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + +int32_t mndDropSubByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + int32_t code = -1; + SSdb *pSdb = pMnode->pSdb; + + void *pIter = NULL; + SMqSubscribeObj *pSub = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pSub); + if (pIter == NULL) break; + + if (pSub->dbUid != pDb->uid) { + sdbRelease(pSdb, pSub); + continue; + } + + if (mndSetDropSubCommitLogs(pMnode, pTrans, pSub) < 0) { + goto END; + } + } + + code = 0; +END: + return code; +} diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 731ee69105846f0ec9d44c6d9837281ea8952f95..0a8d1cee4ad873bf79f85ffbe44ce1f94ddb9511 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -38,6 +38,8 @@ static int32_t mndProcessDropTopicInRsp(SNodeMsg *pRsp); static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); +static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); + int32_t mndInitTopic(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_TOPIC, .keyType = SDB_KEY_BINARY, @@ -553,7 +555,41 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB return numOfRows; } +static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic) { + SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + int32_t code = -1; + SSdb *pSdb = pMnode->pSdb; + + void *pIter = NULL; + SMqTopicObj *pTopic = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); + if (pIter == NULL) break; + + if (pTopic->dbUid != pDb->uid) { + sdbRelease(pSdb, pTopic); + continue; + } + + if (mndSetDropTopicCommitLogs(pMnode, pTrans, pTopic) < 0) { + goto END; + } + } + + code = 0; +END: + return code; +} diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index c08b0f6db94b19f270f608690bca875c0664a763..43e27359841c04b110478d8b6c6d1a0713098d72 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -16,14 +16,15 @@ #define _DEFAULT_SOURCE #include "mndTrans.h" #include "mndAuth.h" +#include "mndConsumer.h" #include "mndDb.h" #include "mndShow.h" #include "mndSync.h" #include "mndUser.h" -#define MND_TRANS_VER_NUMBER 1 -#define MND_TRANS_ARRAY_SIZE 8 -#define MND_TRANS_RESERVE_SIZE 64 +#define TRANS_VER_NUMBER 1 +#define TRANS_ARRAY_SIZE 8 +#define TRANS_RESERVE_SIZE 64 static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); @@ -62,13 +63,15 @@ static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter); int32_t mndInitTrans(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_TRANS, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndTransActionEncode, - .decodeFp = (SdbDecodeFp)mndTransActionDecode, - .insertFp = (SdbInsertFp)mndTransActionInsert, - .updateFp = (SdbUpdateFp)mndTransActionUpdate, - .deleteFp = (SdbDeleteFp)mndTransActionDelete}; + SSdbTable table = { + .sdbType = SDB_TRANS, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndTransActionEncode, + .decodeFp = (SdbDecodeFp)mndTransActionDecode, + .insertFp = (SdbInsertFp)mndTransActionInsert, + .updateFp = (SdbUpdateFp)mndTransActionUpdate, + .deleteFp = (SdbDeleteFp)mndTransActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq); @@ -83,7 +86,7 @@ void mndCleanupTrans(SMnode *pMnode) {} static SSdbRaw *mndTransActionEncode(STrans *pTrans) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t rawDataLen = sizeof(STrans) + MND_TRANS_RESERVE_SIZE; + int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE; int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs); int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs); int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs); @@ -115,78 +118,89 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { rawDataLen += (sizeof(STransAction) + pAction->contLen); } - SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, MND_TRANS_VER_NUMBER, rawDataLen); + SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TRANS_VER_NUMBER, rawDataLen); if (pRaw == NULL) { mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); return NULL; } int32_t dataPos = 0; - SDB_SET_INT32(pRaw, dataPos, pTrans->id, TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->policy, TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->stage, TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->transType, TRANS_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, TRANS_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pTrans->dbUid, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, redoLogNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, undoLogNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, commitLogNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, redoActionNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, undoActionNum, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->id, _OVER) + + ETrnStage stage = pTrans->stage; + if (stage == TRN_STAGE_REDO_LOG || stage == TRN_STAGE_REDO_ACTION) { + stage = TRN_STAGE_PREPARE; + } else if (stage == TRN_STAGE_UNDO_ACTION || stage == TRN_STAGE_UNDO_LOG) { + stage = TRN_STAGE_ROLLBACK; + } else if (stage == TRN_STAGE_COMMIT_LOG || stage == TRN_STAGE_FINISHED) { + stage = TRN_STAGE_COMMIT; + } else { + } + + SDB_SET_INT16(pRaw, dataPos, stage, _OVER) + SDB_SET_INT16(pRaw, dataPos, pTrans->policy, _OVER) + SDB_SET_INT16(pRaw, dataPos, pTrans->type, _OVER) + SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pTrans->dbUid, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_INT32(pRaw, dataPos, redoLogNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, undoLogNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, commitLogNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, redoActionNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, undoActionNum, _OVER) for (int32_t i = 0; i < redoLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->redoLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); - SDB_SET_INT32(pRaw, dataPos, len, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, _OVER) } for (int32_t i = 0; i < undoLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->undoLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); - SDB_SET_INT32(pRaw, dataPos, len, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, _OVER) } for (int32_t i = 0; i < commitLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->commitLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); - SDB_SET_INT32(pRaw, dataPos, len, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, _OVER) } for (int32_t i = 0; i < redoActionNum; ++i) { STransAction *pAction = taosArrayGet(pTrans->redoActions, i); - SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pAction->msgType, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->contLen, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) + SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) } for (int32_t i = 0; i < undoActionNum; ++i) { STransAction *pAction = taosArrayGet(pTrans->undoActions, i); - SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pAction->msgType, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->contLen, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) + SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, _OVER) } - SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pTrans->paramLen, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, _OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, _OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->paramLen, _OVER) if (pTrans->param != NULL) { - SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, _OVER) } - SDB_SET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER) - SDB_SET_DATALEN(pRaw, dataPos, TRANS_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; -TRANS_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("trans:%d, failed to encode to raw:%p len:%d since %s", pTrans->id, pRaw, dataPos, terrstr()); sdbFreeRaw(pRaw); @@ -213,38 +227,38 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; STransAction action = {0}; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TRANS_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != MND_TRANS_VER_NUMBER) { + if (sver != TRANS_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto TRANS_DECODE_OVER; + goto _OVER; } pRow = sdbAllocRow(sizeof(STrans)); - if (pRow == NULL) goto TRANS_DECODE_OVER; + if (pRow == NULL) goto _OVER; pTrans = sdbGetRowObj(pRow); - if (pTrans == NULL) goto TRANS_DECODE_OVER; + if (pTrans == NULL) goto _OVER; - SDB_GET_INT32(pRaw, dataPos, &pTrans->id, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->id, _OVER) - int16_t type = 0; - int16_t policy = 0; int16_t stage = 0; - SDB_GET_INT16(pRaw, dataPos, &policy, TRANS_DECODE_OVER) - SDB_GET_INT16(pRaw, dataPos, &stage, TRANS_DECODE_OVER) - SDB_GET_INT16(pRaw, dataPos, &type, TRANS_DECODE_OVER) - pTrans->policy = policy; + int16_t policy = 0; + int16_t type = 0; + SDB_GET_INT16(pRaw, dataPos, &stage, _OVER) + SDB_GET_INT16(pRaw, dataPos, &policy, _OVER) + SDB_GET_INT16(pRaw, dataPos, &type, _OVER) pTrans->stage = stage; - pTrans->transType = type; - SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, TRANS_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pTrans->dbUid, TRANS_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &redoLogNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &undoLogNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &commitLogNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &redoActionNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &undoActionNum, TRANS_DECODE_OVER) + pTrans->policy = policy; + pTrans->type = type; + SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pTrans->dbUid, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_INT32(pRaw, dataPos, &redoLogNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &undoLogNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &commitLogNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &redoActionNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &undoActionNum, _OVER) pTrans->redoLogs = taosArrayInit(redoLogNum, sizeof(void *)); pTrans->undoLogs = taosArrayInit(undoLogNum, sizeof(void *)); @@ -252,79 +266,79 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { pTrans->redoActions = taosArrayInit(redoActionNum, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(undoActionNum, sizeof(STransAction)); - if (pTrans->redoLogs == NULL) goto TRANS_DECODE_OVER; - if (pTrans->undoLogs == NULL) goto TRANS_DECODE_OVER; - if (pTrans->commitLogs == NULL) goto TRANS_DECODE_OVER; - if (pTrans->redoActions == NULL) goto TRANS_DECODE_OVER; - if (pTrans->undoActions == NULL) goto TRANS_DECODE_OVER; + if (pTrans->redoLogs == NULL) goto _OVER; + if (pTrans->undoLogs == NULL) goto _OVER; + if (pTrans->commitLogs == NULL) goto _OVER; + if (pTrans->redoActions == NULL) goto _OVER; + if (pTrans->undoActions == NULL) goto _OVER; for (int32_t i = 0; i < redoLogNum; ++i) { - SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) pData = taosMemoryMalloc(dataLen); - if (pData == NULL) goto TRANS_DECODE_OVER; + if (pData == NULL) goto _OVER; mTrace("raw:%p, is created", pData); - SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->redoLogs, &pData) == NULL) goto TRANS_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, _OVER); + if (taosArrayPush(pTrans->redoLogs, &pData) == NULL) goto _OVER; pData = NULL; } for (int32_t i = 0; i < undoLogNum; ++i) { - SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) pData = taosMemoryMalloc(dataLen); - if (pData == NULL) goto TRANS_DECODE_OVER; + if (pData == NULL) goto _OVER; mTrace("raw:%p, is created", pData); - SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->undoLogs, &pData) == NULL) goto TRANS_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, _OVER); + if (taosArrayPush(pTrans->undoLogs, &pData) == NULL) goto _OVER; pData = NULL; } for (int32_t i = 0; i < commitLogNum; ++i) { - SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) pData = taosMemoryMalloc(dataLen); - if (pData == NULL) goto TRANS_DECODE_OVER; + if (pData == NULL) goto _OVER; mTrace("raw:%p, is created", pData); - SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->commitLogs, &pData) == NULL) goto TRANS_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, _OVER); + if (taosArrayPush(pTrans->commitLogs, &pData) == NULL) goto _OVER; pData = NULL; } for (int32_t i = 0; i < redoActionNum; ++i) { - SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), TRANS_DECODE_OVER); - SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); + SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) action.pCont = taosMemoryMalloc(action.contLen); - if (action.pCont == NULL) goto TRANS_DECODE_OVER; - SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto TRANS_DECODE_OVER; + if (action.pCont == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); + if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; action.pCont = NULL; } for (int32_t i = 0; i < undoActionNum; ++i) { - SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), TRANS_DECODE_OVER); - SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); + SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) action.pCont = taosMemoryMalloc(action.contLen); - if (action.pCont == NULL) goto TRANS_DECODE_OVER; - SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto TRANS_DECODE_OVER; + if (action.pCont == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); + if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; action.pCont = NULL; } - SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, _OVER) if (pTrans->paramLen != 0) { pTrans->param = taosMemoryMalloc(pTrans->paramLen); - SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, _OVER); } - SDB_GET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) terrno = 0; -TRANS_DECODE_OVER: +_OVER: if (terrno != 0) { mError("trans:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, terrstr()); mndTransDropData(pTrans); @@ -399,6 +413,16 @@ static const char *mndTransType(ETrnType type) { return "subscribe"; case TRN_TYPE_REBALANCE: return "rebalance"; + case TRN_TYPE_COMMIT_OFFSET: + return "commit-offset"; + case TRN_TYPE_CREATE_STREAM: + return "create-stream"; + case TRN_TYPE_DROP_STREAM: + return "drop-stream"; + case TRN_TYPE_CONSUMER_LOST: + return "consumer-lost"; + case TRN_TYPE_CONSUMER_RECOVER: + return "consumer-recover"; case TRN_TYPE_CREATE_DNODE: return "create-qnode"; case TRN_TYPE_DROP_DNODE: @@ -442,13 +466,16 @@ static TransCbFp mndTransGetCbFp(ETrnFuncType ftype) { return mndTransTestStartFunc; case TEST_TRANS_STOP_FUNC: return mndTransTestStopFunc; + case MQ_REB_TRANS_START_FUNC: + return mndRebCntInc; + case MQ_REB_TRANS_STOP_FUNC: + return mndRebCntDec; default: return NULL; } } static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { - // pTrans->stage = TRN_STAGE_PREPARE; mTrace("trans:%d, perform insert action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage)); if (pTrans->startFunc > 0) { @@ -510,16 +537,15 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { return 0; } -static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { - SSdb *pSdb = pMnode->pSdb; - STrans *pTrans = sdbAcquire(pSdb, SDB_TRANS, &transId); +STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { + STrans *pTrans = sdbAcquire(pMnode->pSdb, SDB_TRANS, &transId); if (pTrans == NULL) { terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; } return pTrans; } -static void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { +void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pTrans); } @@ -535,16 +561,16 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S pTrans->id = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; - pTrans->transType = type; + pTrans->type = type; pTrans->createdTime = taosGetTimestampMs(); pTrans->rpcHandle = pReq->handle; pTrans->rpcAHandle = pReq->ahandle; pTrans->rpcRefId = pReq->refId; - pTrans->redoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); - pTrans->undoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); - pTrans->commitLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); - pTrans->redoActions = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(STransAction)); - pTrans->undoActions = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(STransAction)); + pTrans->redoLogs = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->undoLogs = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->commitLogs = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); + pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL || pTrans->redoActions == NULL || pTrans->undoActions == NULL) { @@ -553,7 +579,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S return NULL; } - mDebug("trans:%d, local var is created, data:%p", pTrans->id, pTrans); + mDebug("trans:%d, local object is created, data:%p", pTrans->id, pTrans); return pTrans; } @@ -580,14 +606,14 @@ static void mndTransDropActions(SArray *pArray) { void mndTransDrop(STrans *pTrans) { if (pTrans != NULL) { mndTransDropData(pTrans); - mDebug("trans:%d, local var is freed, data:%p", pTrans->id, pTrans); + mDebug("trans:%d, local object is freed, data:%p", pTrans->id, pTrans); taosMemoryFreeClear(pTrans); } } static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw) { if (pArray == NULL || pRaw == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = TSDB_CODE_INVALID_PARA; return -1; } @@ -669,27 +695,27 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { } static bool mndIsBasicTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_BASIC_SCOPE && pTrans->stage < TRN_TYPE_BASIC_SCOPE_END; + return pTrans->type > TRN_TYPE_BASIC_SCOPE && pTrans->type < TRN_TYPE_BASIC_SCOPE_END; } static bool mndIsGlobalTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_GLOBAL_SCOPE && pTrans->stage < TRN_TYPE_GLOBAL_SCOPE_END; + return pTrans->type > TRN_TYPE_GLOBAL_SCOPE && pTrans->type < TRN_TYPE_GLOBAL_SCOPE_END; } static bool mndIsDbTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_DB_SCOPE && pTrans->stage < TRN_TYPE_DB_SCOPE_END; + return pTrans->type > TRN_TYPE_DB_SCOPE && pTrans->type < TRN_TYPE_DB_SCOPE_END; } static bool mndIsStbTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_STB_SCOPE && pTrans->stage < TRN_TYPE_STB_SCOPE_END; + return pTrans->type > TRN_TYPE_STB_SCOPE && pTrans->type < TRN_TYPE_STB_SCOPE_END; } -static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewTrans) { - if (mndIsBasicTrans(pNewTrans)) return 0; - +static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNewTrans) { STrans *pTrans = NULL; void *pIter = NULL; - int32_t code = 0; + bool conflict = false; + + if (mndIsBasicTrans(pNewTrans)) return conflict; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans); @@ -698,42 +724,35 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT if (mndIsGlobalTrans(pNewTrans)) { if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - code = -1; - break; + conflict = true; + } else { } } - if (mndIsDbTrans(pNewTrans)) { - if (mndIsBasicTrans(pTrans)) continue; + else if (mndIsDbTrans(pNewTrans)) { if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - code = -1; - break; - } - if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { + conflict = true; + } else if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - code = -1; - break; + conflict = true; } + } else { } } - if (mndIsStbTrans(pNewTrans)) { - if (mndIsBasicTrans(pTrans)) continue; + else if (mndIsStbTrans(pNewTrans)) { if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - code = -1; - break; - } - if (mndIsDbTrans(pTrans)) { + conflict = true; + } else if (mndIsDbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - code = -1; - break; + conflict = true; } + } else { } - if (mndIsStbTrans(pTrans)) continue; } sdbRelease(pMnode->pSdb, pTrans); @@ -741,12 +760,12 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT sdbCancelFetch(pMnode->pSdb, pIter); sdbRelease(pMnode->pSdb, pTrans); - return code; + return conflict; } int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { - if (mndCheckTransCanBeStartedInParallel(pMnode, pTrans) != 0) { - terrno = TSDB_CODE_MND_TRANS_CANT_PARALLEL; + if (mndCheckTransConflict(pMnode, pTrans)) { + terrno = TSDB_CODE_MND_TRANS_CONFLICT; mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); return -1; } @@ -800,7 +819,8 @@ static int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) { } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { - bool sendRsp = false; + bool sendRsp = false; + int32_t code = pTrans->code; if (pTrans->stage == TRN_STAGE_FINISHED) { sendRsp = true; @@ -809,12 +829,12 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { if (pTrans->policy == TRN_POLICY_ROLLBACK) { if (pTrans->stage == TRN_STAGE_UNDO_LOG || pTrans->stage == TRN_STAGE_UNDO_ACTION || pTrans->stage == TRN_STAGE_ROLLBACK) { + if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; sendRsp = true; } - } - - if (pTrans->policy == TRN_POLICY_RETRY) { + } else { if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 0) { + if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; sendRsp = true; } } @@ -826,14 +846,16 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } taosMemoryFree(pTrans->rpcRsp); - mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, + mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); - SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, - .code = pTrans->code, - .ahandle = pTrans->rpcAHandle, - .refId = pTrans->rpcRefId, - .pCont = rpcCont, - .contLen = pTrans->rpcRspLen}; + SRpcMsg rspMsg = { + .handle = pTrans->rpcHandle, + .ahandle = pTrans->rpcAHandle, + .refId = pTrans->rpcRefId, + .code = code, + .pCont = rpcCont, + .contLen = pTrans->rpcRspLen, + }; tmsgSendRsp(&rspMsg); pTrans->rpcHandle = NULL; pTrans->rpcRsp = NULL; @@ -850,7 +872,7 @@ void mndTransProcessRsp(SNodeMsg *pRsp) { STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans == NULL) { mError("trans:%d, failed to get transId from vnode rsp since %s", transId, terrstr()); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } SArray *pArray = NULL; @@ -860,18 +882,18 @@ void mndTransProcessRsp(SNodeMsg *pRsp) { pArray = pTrans->undoActions; } else { mError("trans:%d, invalid trans stage:%d while recv action rsp", pTrans->id, pTrans->stage); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } if (pArray == NULL) { mError("trans:%d, invalid trans stage:%d", transId, pTrans->stage); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } int32_t actionNum = taosArrayGetSize(pTrans->redoActions); if (action < 0 || action >= actionNum) { mError("trans:%d, invalid action:%d", transId, action); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } STransAction *pAction = taosArrayGet(pArray, action); @@ -887,7 +909,7 @@ void mndTransProcessRsp(SNodeMsg *pRsp) { pAction->acceptableCode); mndTransExecute(pMnode, pTrans); -HANDLE_ACTION_RSP_OVER: +_OVER: mndReleaseTrans(pMnode, pTrans); } @@ -897,27 +919,41 @@ static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) { if (arraySize == 0) return 0; + int32_t code = 0; for (int32_t i = 0; i < arraySize; ++i) { SSdbRaw *pRaw = taosArrayGetP(pArray, i); - int32_t code = sdbWriteWithoutFree(pSdb, pRaw); - if (code != 0) { - return code; + if (sdbWriteWithoutFree(pSdb, pRaw) != 0) { + code = ((terrno != 0) ? terrno : -1); } } - return 0; + terrno = code; + return code; } static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteLogs(pMnode, pTrans->redoLogs); + int32_t code = mndTransExecuteLogs(pMnode, pTrans->redoLogs); + if (code != 0) { + mError("failed to execute redoLogs since %s", terrstr()); + } + return code; } static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteLogs(pMnode, pTrans->undoLogs); + int32_t code = mndTransExecuteLogs(pMnode, pTrans->undoLogs); + if (code != 0) { + mError("failed to execute undoLogs since %s, return success", terrstr()); + } + + return 0; // return success in any case } static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteLogs(pMnode, pTrans->commitLogs); + int32_t code = mndTransExecuteLogs(pMnode, pTrans->commitLogs); + if (code != 0) { + mError("failed to execute commitLogs since %s", terrstr()); + } + return code; } static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { @@ -961,6 +997,12 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr pAction->msgReceived = 0; pAction->errCode = 0; } else { + pAction->msgSent = 0; + pAction->msgReceived = 0; + pAction->errCode = terrno; + if (terrno == TSDB_CODE_INVALID_PTR || terrno == TSDB_CODE_NODE_OFFLINE) { + rpcFreeCont(rpcMsg.pCont); + } mError("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr()); return -1; } @@ -1007,11 +1049,19 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA } static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); + int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("failed to execute redoActions since %s", terrstr()); + } + return code; } static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); + int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("failed to execute undoActions since %s", terrstr()); + } + return code; } static bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { @@ -1078,8 +1128,8 @@ static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { } else { pTrans->code = terrno; if (pTrans->policy == TRN_POLICY_ROLLBACK) { - pTrans->stage = TRN_STAGE_REDO_ACTION; - mError("trans:%d, stage from commit to redoAction since %s, failedTimes:%d", pTrans->id, terrstr(), + pTrans->stage = TRN_STAGE_UNDO_ACTION; + mError("trans:%d, stage from commit to undoAction since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = true; } else { @@ -1104,7 +1154,7 @@ static bool mndTransPerformCommitLogStage(SMnode *pMnode, STrans *pTrans) { } else { pTrans->code = terrno; pTrans->failedTimes++; - mError("trans:%d, stage keep on commitLog since %s", pTrans->id, terrstr()); + mError("trans:%d, stage keep on commitLog since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = false; } @@ -1140,7 +1190,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { continueExec = false; } else { pTrans->failedTimes++; - mError("trans:%d, stage keep on undoAction since %s", pTrans->id, terrstr()); + mError("trans:%d, stage keep on undoAction since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = false; } @@ -1157,7 +1207,7 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { continueExec = true; } else { pTrans->failedTimes++; - mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr()); + mError("trans:%d, stage keep on rollback since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = false; } @@ -1230,7 +1280,7 @@ static int32_t mndProcessTransReq(SNodeMsg *pReq) { return 0; } -static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { +int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { SArray *pArray = NULL; if (pTrans->stage == TRN_STAGE_REDO_ACTION) { pArray = pTrans->redoActions; @@ -1248,14 +1298,14 @@ static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { if (pAction == NULL) continue; if (pAction->msgReceived == 0) { - mInfo("trans:%d, action:%d set processed", pTrans->id, i); + mInfo("trans:%d, action:%d set processed for kill msg received", pTrans->id, i); pAction->msgSent = 1; pAction->msgReceived = 1; pAction->errCode = 0; } if (pAction->errCode != 0) { - mInfo("trans:%d, action:%d set processed, errCode from %s to success", pTrans->id, i, + mInfo("trans:%d, action:%d set processed for kill msg received, errCode from %s to success", pTrans->id, i, tstrerror(pAction->errCode)); pAction->msgSent = 1; pAction->msgReceived = 1; @@ -1276,19 +1326,19 @@ static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { if (tDeserializeSKillTransReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto KILL_OVER; + goto _OVER; } mInfo("trans:%d, start to kill", killReq.transId); pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto KILL_OVER; + goto _OVER; } if (!pUser->superUser) { terrno = TSDB_CODE_MND_NO_RIGHTS; - goto KILL_OVER; + goto _OVER; } pTrans = mndAcquireTrans(pMnode, killReq.transId); @@ -1300,7 +1350,7 @@ static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { code = mndKillTrans(pMnode, pTrans); -KILL_OVER: +_OVER: if (code != 0) { mError("trans:%d, failed to kill since %s", killReq.transId, terrstr()); return -1; @@ -1346,25 +1396,25 @@ static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->createdTime, false); char stage[TSDB_TRANS_STAGE_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(stage, mndTransStr(pTrans->stage), pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(stage, mndTransStr(pTrans->stage), pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)stage, false); char dbname[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(dbname, mndGetDbStr(pTrans->dbname), pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(dbname, mndGetDbStr(pTrans->dbname), pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)dbname, false); - char transType[TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(dbname, mndTransType(pTrans->transType), pShow->bytes[cols]); + char type[TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(dbname, mndTransType(pTrans->type), pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)transType, false); + colDataAppend(pColInfo, numOfRows, (const char *)type, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->lastExecTime, false); char lastError[TSDB_TRANS_ERROR_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(dbname, pTrans->lastError, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(dbname, pTrans->lastError, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)lastError, false); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index b71e5e4f1750f01e994375d372e942775c694b6c..8b5fa06faf4f3d67f1d74f80e2bee2dc4eb87382 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -25,7 +25,6 @@ #define USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); -static SSdbRaw *mndUserActionEncode(SUserObj *pUser); static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw); static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser); static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser); @@ -90,7 +89,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { return 0; } -static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { +SSdbRaw *mndUserActionEncode(SUserObj *pUser) { terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); @@ -187,6 +186,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { } SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) + taosInitRWLatch(&pUser->lock); terrno = 0; @@ -229,16 +229,17 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { mTrace("user:%s, perform update action, old row:%p new row:%p", pOld->user, pOld, pNew); - memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); + taosWLockLatch(&pOld->lock); pOld->updateTime = pNew->updateTime; - + memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); TSWAP(pOld->readDbs, pNew->readDbs); TSWAP(pOld->writeDbs, pNew->writeDbs); + taosWUnLockLatch(&pOld->lock); return 0; } -SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) { +SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName) { SSdb *pSdb = pMnode->pSdb; SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName); if (pUser == NULL) { @@ -276,9 +277,6 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate } sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - char *param = strdup("====> test code to be deleted later <====="); - mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); - if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); @@ -430,8 +428,12 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { } memcpy(&newUser, pUser, sizeof(SUserObj)); + + taosRLockLatch(&pUser->lock); newUser.readDbs = mndDupDbHash(pUser->readDbs); newUser.writeDbs = mndDupDbHash(pUser->writeDbs); + taosRUnLockLatch(&pUser->lock); + if (newUser.readDbs == NULL || newUser.writeDbs == NULL) { goto _OVER; } @@ -455,13 +457,16 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB) { if (taosHashRemove(newUser.readDbs, alterReq.dbname, len) != 0) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_READ_DB) { taosHashClear(newUser.readDbs); + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB) { if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; @@ -471,13 +476,16 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB) { if (taosHashRemove(newUser.writeDbs, alterReq.dbname, len) != 0) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_WRITE_DB) { taosHashClear(newUser.writeDbs); + newUser.authVersion++; } else { terrno = TSDB_CODE_MND_INVALID_ALTER_OPER; goto _OVER; @@ -580,6 +588,38 @@ _OVER: return code; } +static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { + memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); + pRsp->superAuth = pUser->superUser; + pRsp->version = pUser->authVersion; + taosRLockLatch(&pUser->lock); + pRsp->readDbs = mndDupDbHash(pUser->readDbs); + pRsp->writeDbs = mndDupDbHash(pUser->writeDbs); + taosRUnLockLatch(&pUser->lock); + pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (NULL == pRsp->createdDbs) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + while (1) { + SDbObj *pDb = NULL; + pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb); + if (pIter == NULL) break; + + if (strcmp(pDb->createUser, pUser->user) == 0) { + int32_t len = strlen(pDb->name) + 1; + taosHashPut(pRsp->createdDbs, pDb->name, len, pDb->name, len); + } + + sdbRelease(pSdb, pDb); + } + + return 0; +} + static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { SMnode *pMnode = pReq->pNode; int32_t code = -1; @@ -600,25 +640,9 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { goto _OVER; } - memcpy(authRsp.user, pUser->user, TSDB_USER_LEN); - authRsp.superAuth = pUser->superUser; - authRsp.readDbs = mndDupDbHash(pUser->readDbs); - authRsp.writeDbs = mndDupDbHash(pUser->writeDbs); - - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - while (1) { - SDbObj *pDb = NULL; - pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb); - if (pIter == NULL) break; - - if (strcmp(pDb->createUser, pUser->user) == 0) { - int32_t len = strlen(pDb->name) + 1; - taosHashPut(authRsp.readDbs, pDb->name, len, pDb->name, len); - taosHashPut(authRsp.writeDbs, pDb->name, len, pDb->name, len); - } - - sdbRelease(pSdb, pDb); + code = mndSetUserAuthRsp(pMnode, pUser, &authRsp); + if (code) { + goto _OVER; } int32_t contLen = tSerializeSGetUserAuthRsp(NULL, 0, &authRsp); @@ -635,6 +659,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { code = 0; _OVER: + mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&authRsp); @@ -657,7 +682,7 @@ static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols); char name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->pMeta->pSchemas[cols].bytes); colDataAppend(pColInfo, numOfRows, (const char *)name, false); @@ -685,3 +710,72 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen) { + SUserAuthBatchRsp batchRsp = {0}; + batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserAuthRsp)); + if (batchRsp.pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + int32_t code = 0; + for (int32_t i = 0; i < numOfUses; ++i) { + SUserObj *pUser = mndAcquireUser(pMnode, pUsers[i].user); + if (pUser == NULL) { + mError("user:%s, failed to auth user since %s", pUsers[i].user, terrstr()); + continue; + } + + if (pUser->authVersion <= pUsers[i].version) { + mndReleaseUser(pMnode, pUser); + continue; + } + + SGetUserAuthRsp rsp = {0}; + code = mndSetUserAuthRsp(pMnode, pUser, &rsp); + if (code) { + mndReleaseUser(pMnode, pUser); + tFreeSGetUserAuthRsp(&rsp); + goto _OVER; + } + + + taosArrayPush(batchRsp.pArray, &rsp); + mndReleaseUser(pMnode, pUser); + } + + if (taosArrayGetSize(batchRsp.pArray) <= 0) { + *ppRsp = NULL; + *pRspLen = 0; + + tFreeSUserAuthBatchRsp(&batchRsp); + return 0; + } + + int32_t rspLen = tSerializeSUserAuthBatchRsp(NULL, 0, &batchRsp); + void *pRsp = taosMemoryMalloc(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tFreeSUserAuthBatchRsp(&batchRsp); + return -1; + } + tSerializeSUserAuthBatchRsp(pRsp, rspLen, &batchRsp); + + *ppRsp = pRsp; + *pRspLen = rspLen; + + tFreeSUserAuthBatchRsp(&batchRsp); + return 0; + +_OVER: + + *ppRsp = NULL; + *pRspLen = 0; + + tFreeSUserAuthBatchRsp(&batchRsp); + return code; +} + + + diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index aab9ac61a24b760bfae8c206b763ba1b597dea5e..d1e4be116197ea1089a4a223d9c9810cd5d9cf3e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -21,8 +21,8 @@ #include "mndShow.h" #include "mndTrans.h" -#define TSDB_VGROUP_VER_NUMBER 1 -#define TSDB_VGROUP_RESERVE_SIZE 64 +#define VGROUP_VER_NUMBER 1 +#define VGROUP_RESERVE_SIZE 64 static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); @@ -34,19 +34,21 @@ static int32_t mndProcessAlterVnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessDropVnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessCompactVnodeRsp(SNodeMsg *pRsp); -static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); -static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); int32_t mndInitVgroup(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_VGROUP, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, - .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, - .insertFp = (SdbInsertFp)mndVgroupActionInsert, - .updateFp = (SdbUpdateFp)mndVgroupActionUpdate, - .deleteFp = (SdbDeleteFp)mndVgroupActionDelete}; + SSdbTable table = { + .sdbType = SDB_VGROUP, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, + .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, + .insertFp = (SdbInsertFp)mndVgroupActionInsert, + .updateFp = (SdbUpdateFp)mndVgroupActionUpdate, + .deleteFp = (SdbDeleteFp)mndVgroupActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp); @@ -66,29 +68,29 @@ void mndCleanupVgroup(SMnode *pMnode) {} SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUMBER, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); - if (pRaw == NULL) goto VG_ENCODE_OVER; + SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, VGROUP_VER_NUMBER, sizeof(SVgObj) + VGROUP_RESERVE_SIZE); + if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_INT32(pRaw, dataPos, pVgroup->vgId, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->version, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd, VG_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid, VG_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pVgroup->replica, VG_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->vgId, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->version, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid, _OVER) + SDB_SET_INT8(pRaw, dataPos, pVgroup->replica, _OVER) for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; - SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId, VG_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId, _OVER) } - SDB_SET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE, VG_ENCODE_OVER) - SDB_SET_DATALEN(pRaw, dataPos, VG_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, VGROUP_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; -VG_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("vgId:%d, failed to encode to raw:%p since %s", pVgroup->vgId, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -103,41 +105,41 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto VG_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != TSDB_VGROUP_VER_NUMBER) { + if (sver != VGROUP_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto VG_DECODE_OVER; + goto _OVER; } SSdbRow *pRow = sdbAllocRow(sizeof(SVgObj)); - if (pRow == NULL) goto VG_DECODE_OVER; + if (pRow == NULL) goto _OVER; SVgObj *pVgroup = sdbGetRowObj(pRow); - if (pVgroup == NULL) goto VG_DECODE_OVER; + if (pVgroup == NULL) goto _OVER; int32_t dataPos = 0; - SDB_GET_INT32(pRaw, dataPos, &pVgroup->vgId, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->createdTime, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->updateTime, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->version, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashBegin, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashEnd, VG_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->dbUid, VG_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pVgroup->replica, VG_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->vgId, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->updateTime, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->version, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashBegin, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashEnd, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->dbUid, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pVgroup->replica, _OVER) for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; - SDB_GET_INT32(pRaw, dataPos, &pVgid->dnodeId, VG_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgid->dnodeId, _OVER) if (pVgroup->replica == 1) { pVgid->role = TAOS_SYNC_STATE_LEADER; } } - SDB_GET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE, VG_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, VGROUP_RESERVE_SIZE, _OVER) terrno = 0; -VG_DECODE_OVER: +_OVER: if (terrno != 0) { mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup->vgId, pRaw, terrstr()); taosMemoryFreeClear(pRow); @@ -190,25 +192,24 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg memcpy(createReq.db, pDb->name, TSDB_DB_FNAME_LEN); createReq.dbUid = pDb->uid; createReq.vgVersion = pVgroup->version; - createReq.cacheBlockSize = pDb->cfg.cacheBlockSize; - createReq.totalBlocks = pDb->cfg.totalBlocks; + createReq.numOfStables = pDb->cfg.numOfStables; + createReq.buffer = pDb->cfg.buffer; + createReq.pageSize = pDb->cfg.pageSize; + createReq.pages = pDb->cfg.pages; createReq.daysPerFile = pDb->cfg.daysPerFile; createReq.daysToKeep0 = pDb->cfg.daysToKeep0; createReq.daysToKeep1 = pDb->cfg.daysToKeep1; createReq.daysToKeep2 = pDb->cfg.daysToKeep2; createReq.minRows = pDb->cfg.minRows; createReq.maxRows = pDb->cfg.maxRows; - createReq.commitTime = pDb->cfg.commitTime; createReq.fsyncPeriod = pDb->cfg.fsyncPeriod; createReq.walLevel = pDb->cfg.walLevel; createReq.precision = pDb->cfg.precision; createReq.compression = pDb->cfg.compression; createReq.strict = pDb->cfg.strict; - createReq.update = pDb->cfg.update; createReq.cacheLastRow = pDb->cfg.cacheLastRow; createReq.replica = pVgroup->replica; createReq.selfIndex = -1; - createReq.streamMode = pVgroup->streamMode; createReq.hashBegin = pVgroup->hashBegin; createReq.hashEnd = pVgroup->hashEnd; createReq.hashMethod = pDb->cfg.hashMethod; @@ -255,6 +256,68 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg return pReq; } +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { + SAlterVnodeReq alterReq = {0}; + alterReq.vgVersion = pVgroup->version; + alterReq.buffer = pDb->cfg.buffer; + alterReq.pages = pDb->cfg.pages; + alterReq.pageSize = pDb->cfg.pageSize; + alterReq.daysPerFile = pDb->cfg.daysPerFile; + alterReq.daysToKeep0 = pDb->cfg.daysToKeep0; + alterReq.daysToKeep1 = pDb->cfg.daysToKeep1; + alterReq.daysToKeep2 = pDb->cfg.daysToKeep2; + alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod; + alterReq.walLevel = pDb->cfg.walLevel; + alterReq.strict = pDb->cfg.strict; + alterReq.cacheLastRow = pDb->cfg.cacheLastRow; + alterReq.replica = pVgroup->replica; + alterReq.selfIndex = -1; + + for (int32_t v = 0; v < pVgroup->replica; ++v) { + SReplica *pReplica = &alterReq.replicas[v]; + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; + SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pVgidDnode == NULL) { + return NULL; + } + + pReplica->id = pVgidDnode->id; + pReplica->port = pVgidDnode->port; + memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); + mndReleaseDnode(pMnode, pVgidDnode); + + if (pDnode->id == pVgid->dnodeId) { + alterReq.selfIndex = v; + } + } + + if (alterReq.selfIndex == -1) { + terrno = TSDB_CODE_MND_APP_ERROR; + return NULL; + } + + int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); + if (contLen < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + contLen += +sizeof(SMsgHead); + + void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + SMsgHead *pHead = pReq; + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + tSerializeSAlterVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &alterReq); + *pContLen = contLen; + return pReq; +} + void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { SDropVnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; @@ -373,12 +436,12 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { pVgroups = taosMemoryCalloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); if (pVgroups == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto ALLOC_VGROUP_OVER; + goto _OVER; } pArray = mndBuildDnodesArray(pMnode); if (pArray == NULL) { - goto ALLOC_VGROUP_OVER; + goto _OVER; } mDebug("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), @@ -398,7 +461,6 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { pVgroup->createdTime = taosGetTimestampMs(); pVgroup->updateTime = pVgroups->createdTime; pVgroup->version = 1; - pVgroup->streamMode = pDb->cfg.streamMode; pVgroup->hashBegin = hashMin + hashInterval * v; if (v == pDb->cfg.numOfVgroups - 1) { pVgroup->hashEnd = hashMax; @@ -412,7 +474,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { if (mndGetAvailableDnode(pMnode, pVgroup, pArray) != 0) { terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; - goto ALLOC_VGROUP_OVER; + goto _OVER; } allocedVgroups++; @@ -423,7 +485,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { mDebug("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); -ALLOC_VGROUP_OVER: +_OVER: if (code != 0) taosMemoryFree(pVgroups); taosArrayDestroy(pArray); return code; @@ -494,7 +556,7 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep return 0; } -static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -535,14 +597,13 @@ static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* // default 3 replica for (int32_t i = 0; i < 3; ++i) { - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (i < pVgroup->replica) { colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vnodeGid[i].dnodeId, false); char buf1[20] = {0}; const char *role = syncStr(pVgroup->vnodeGid[i].role); - STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)buf1, false); @@ -599,13 +660,12 @@ int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { return numOfVnodes; } -static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SVgObj *pVgroup = NULL; int32_t cols = 0; -// int32_t dnodeId = pShow->replica; while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index b38fd6178cdb49507e6f2861eb77547377519a3f..e2814e95f0fbd3bc0dba38c8973e3cd08af2858a 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -43,9 +43,6 @@ #include "mndUser.h" #include "mndVgroup.h" -#define MQ_TIMER_MS 2000 -#define TRNAS_TIMER_MS 6000 - static void *mndBuildTimerMsg(int32_t *pContLen) { SMTimerReq timerReq = {0}; @@ -68,7 +65,7 @@ static void mndPullupTrans(void *param, void *tmrId) { tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } - taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); + taosTmrReset(mndPullupTrans, tsTransPullupMs, pMnode, pMnode->timer, &pMnode->transTimer); } static void mndCalMqRebalance(void *param, void *tmrId) { @@ -84,7 +81,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) { tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } - taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); + taosTmrReset(mndCalMqRebalance, tsMaRebalanceMs, pMnode, pMnode->timer, &pMnode->mqTimer); } static void mndPullupTelem(void *param, void *tmrId) { @@ -106,12 +103,12 @@ static int32_t mndInitTimer(SMnode *pMnode) { return -1; } - if (taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) { + if (taosTmrReset(mndPullupTrans, tsTransPullupMs, pMnode, pMnode->timer, &pMnode->transTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer)) { + if (taosTmrReset(mndCalMqRebalance, tsMaRebalanceMs, pMnode, pMnode->timer, &pMnode->mqTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 6e72ce89d3512a471b4038a836114866b6d6309d..3c5bf5c0835893fe945b33ad5b5f9535030ad5ad 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -35,27 +35,24 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { SCreateDbReq createReq = {0}; strcpy(createReq.db, "1.d1"); createReq.numOfVgroups = 2; - createReq.cacheBlockSize = 16; - createReq.totalBlocks = 10; + createReq.buffer = -1; + createReq.pageSize = -1; + createReq.pages = -1; createReq.daysPerFile = 1000; createReq.daysToKeep0 = 3650; createReq.daysToKeep1 = 3650; createReq.daysToKeep2 = 3650; createReq.minRows = 100; createReq.maxRows = 4096; - createReq.commitTime = 3600; createReq.fsyncPeriod = 3000; - createReq.ttl = 1; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.update = 0; createReq.cacheLastRow = 0; createReq.ignoreExist = 1; - createReq.streamMode = 0; - createReq.singleSTable = 0; + createReq.numOfStables = 0; createReq.numOfRetensions = 0; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); @@ -76,7 +73,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { { SAlterDbReq alterdbReq = {0}; strcpy(alterdbReq.db, "1.d1"); - alterdbReq.totalBlocks = 12; + alterdbReq.buffer = 12; alterdbReq.daysToKeep0 = 300; alterdbReq.daysToKeep1 = 400; alterdbReq.daysToKeep2 = 500; @@ -129,27 +126,24 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { SCreateDbReq createReq = {0}; strcpy(createReq.db, "1.d2"); createReq.numOfVgroups = 2; - createReq.cacheBlockSize = 16; - createReq.totalBlocks = 10; + createReq.buffer = -1; + createReq.pageSize = -1; + createReq.pages = -1; createReq.daysPerFile = 1000; createReq.daysToKeep0 = 3650; createReq.daysToKeep1 = 3650; createReq.daysToKeep2 = 3650; createReq.minRows = 100; createReq.maxRows = 4096; - createReq.commitTime = 3600; createReq.fsyncPeriod = 3000; - createReq.ttl = 1; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.update = 0; createReq.cacheLastRow = 0; createReq.ignoreExist = 1; - createReq.streamMode = 0; - createReq.singleSTable = 0; + createReq.numOfStables = 0; createReq.numOfRetensions = 0; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 0473fa375e7ceef9bb8d0b3702b83e8684363871..c8f832160b4addc852fcf3f610dcd8d608cfef29 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -24,6 +24,7 @@ class MndTestFunc : public ::testing::Test { void SetCode(SCreateFuncReq* pReq, const char* pCode, int32_t size); void SetComment(SCreateFuncReq* pReq, const char* pComment); + void SetBufSize(SCreateFuncReq* pReq, int32_t size); }; Testbase MndTestFunc::test; @@ -40,6 +41,10 @@ void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) { strcpy(pReq->pComment, pComment); } +void MndTestFunc::SetBufSize(SCreateFuncReq* pReq, int32_t size) { + pReq->bufSize = size; +} + TEST_F(MndTestFunc, 01_Show_Func) { test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); EXPECT_EQ(test.GetShowRows(), 0); @@ -96,6 +101,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { strcpy(createReq.name, "f1"); SetCode(&createReq, "code1", 6); SetComment(&createReq, "comment1"); + SetBufSize(&createReq, -1); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); diff --git a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp index 606db251d2f062ab632f3ab8601153abea695b1b..b93adf99305492fec346a1fb981aef7e4e55979f 100644 --- a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp +++ b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp @@ -15,8 +15,31 @@ class MndTestSdb : public ::testing::Test { protected: - static void SetUpTestSuite() {} - static void TearDownTestSuite() {} + static void SetUpTestSuite() { + dDebugFlag = 143; + vDebugFlag = 0; + mDebugFlag = 143; + cDebugFlag = 0; + jniDebugFlag = 0; + tmrDebugFlag = 135; + uDebugFlag = 135; + rpcDebugFlag = 143; + qDebugFlag = 0; + wDebugFlag = 0; + sDebugFlag = 0; + tsdbDebugFlag = 0; + tsLogEmbedded = 1; + tsAsyncLog = 0; + + const char *path = "/tmp/td"; + taosRemoveDir(path); + taosMkDir(path); + tstrncpy(tsLogDir, path, PATH_MAX); + if (taosInitLog("taosdlog", 1) != 0) { + printf("failed to init log file\n"); + } + } + static void TearDownTestSuite() { taosCloseLog(); } public: void SetUp() override {} @@ -26,6 +49,8 @@ class MndTestSdb : public ::testing::Test { typedef struct SMnode { int32_t v100; int32_t v200; + int32_t insertTimes; + int32_t deleteTimes; SSdb *pSdb; } SMnode; @@ -80,6 +105,48 @@ SSdbRaw *strEncode(SStrObj *pObj) { return pRaw; } +SSdbRaw *i32Encode(SI32Obj *pObj) { + int32_t dataPos = 0; + SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, 2, sizeof(SI32Obj)); + + sdbSetRawInt32(pRaw, dataPos, pObj->key); + dataPos += sizeof(pObj->key); + sdbSetRawInt8(pRaw, dataPos, pObj->v8); + dataPos += sizeof(pObj->v8); + sdbSetRawInt16(pRaw, dataPos, pObj->v16); + dataPos += sizeof(pObj->v16); + sdbSetRawInt32(pRaw, dataPos, pObj->v32); + dataPos += sizeof(pObj->v32); + sdbSetRawInt64(pRaw, dataPos, pObj->v64); + dataPos += sizeof(pObj->v64); + sdbSetRawBinary(pRaw, dataPos, pObj->vstr, sizeof(pObj->vstr)); + dataPos += sizeof(pObj->vstr); + sdbSetRawDataLen(pRaw, dataPos); + + return pRaw; +} + +SSdbRaw *i64Encode(SI64Obj *pObj) { + int32_t dataPos = 0; + SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, 3, sizeof(SI64Obj)); + + sdbSetRawInt64(pRaw, dataPos, pObj->key); + dataPos += sizeof(pObj->key); + sdbSetRawInt8(pRaw, dataPos, pObj->v8); + dataPos += sizeof(pObj->v8); + sdbSetRawInt16(pRaw, dataPos, pObj->v16); + dataPos += sizeof(pObj->v16); + sdbSetRawInt32(pRaw, dataPos, pObj->v32); + dataPos += sizeof(pObj->v32); + sdbSetRawInt64(pRaw, dataPos, pObj->v64); + dataPos += sizeof(pObj->v64); + sdbSetRawBinary(pRaw, dataPos, pObj->vstr, sizeof(pObj->vstr)); + dataPos += sizeof(pObj->vstr); + sdbSetRawDataLen(pRaw, dataPos); + + return pRaw; +} + SSdbRow *strDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; @@ -108,9 +175,103 @@ SSdbRow *strDecode(SSdbRaw *pRaw) { return pRow; } -int32_t strInsert(SSdb *pSdb, SStrObj *pObj) { return 0; } +SSdbRow *i32Decode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + if (sver != 2) return NULL; + + SSdbRow *pRow = sdbAllocRow(sizeof(SI32Obj)); + if (pRow == NULL) return NULL; + + SI32Obj *pObj = (SI32Obj *)sdbGetRowObj(pRow); + if (pObj == NULL) return NULL; + + int32_t dataPos = 0; + sdbGetRawInt32(pRaw, dataPos, &pObj->key); + dataPos += sizeof(pObj->key); + sdbGetRawInt8(pRaw, dataPos, &pObj->v8); + dataPos += sizeof(pObj->v8); + sdbGetRawInt16(pRaw, dataPos, &pObj->v16); + dataPos += sizeof(pObj->v16); + sdbGetRawInt32(pRaw, dataPos, &pObj->v32); + dataPos += sizeof(pObj->v32); + sdbGetRawInt64(pRaw, dataPos, &pObj->v64); + dataPos += sizeof(pObj->v64); + sdbGetRawBinary(pRaw, dataPos, pObj->vstr, sizeof(pObj->vstr)); + dataPos += sizeof(pObj->vstr); + + return pRow; +} + +SSdbRow *i64Decode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + if (sver != 3) return NULL; + + SSdbRow *pRow = sdbAllocRow(sizeof(SI64Obj)); + if (pRow == NULL) return NULL; + + SI64Obj *pObj = (SI64Obj *)sdbGetRowObj(pRow); + if (pObj == NULL) return NULL; + + int32_t dataPos = 0; + sdbGetRawInt64(pRaw, dataPos, &pObj->key); + dataPos += sizeof(pObj->key); + sdbGetRawInt8(pRaw, dataPos, &pObj->v8); + dataPos += sizeof(pObj->v8); + sdbGetRawInt16(pRaw, dataPos, &pObj->v16); + dataPos += sizeof(pObj->v16); + sdbGetRawInt32(pRaw, dataPos, &pObj->v32); + dataPos += sizeof(pObj->v32); + sdbGetRawInt64(pRaw, dataPos, &pObj->v64); + dataPos += sizeof(pObj->v64); + sdbGetRawBinary(pRaw, dataPos, pObj->vstr, sizeof(pObj->vstr)); + dataPos += sizeof(pObj->vstr); + + return pRow; +} + +int32_t strInsert(SSdb *pSdb, SStrObj *pObj) { + SMnode *pMnode = pSdb->pMnode; + pMnode->insertTimes++; + return 0; +} + +int32_t i32Insert(SSdb *pSdb, SI32Obj *pObj) { + SMnode *pMnode = pSdb->pMnode; + pMnode->insertTimes++; + return 0; +} + +int32_t i64Insert(SSdb *pSdb, SI64Obj *pObj) { + SMnode *pMnode = pSdb->pMnode; + pMnode->insertTimes++; + return 0; +} + +int32_t strDelete(SSdb *pSdb, SStrObj *pObj, bool callFunc) { + if (callFunc) { + SMnode *pMnode = pSdb->pMnode; + pMnode->deleteTimes++; + } + return 0; +} -int32_t strDelete(SSdb *pSdb, SStrObj *pObj, bool callFunc) { return 0; } +int32_t i32Delete(SSdb *pSdb, SI32Obj *pObj, bool callFunc) { + if (callFunc) { + SMnode *pMnode = pSdb->pMnode; + pMnode->deleteTimes++; + } + return 0; +} + +int32_t i64Delete(SSdb *pSdb, SI64Obj *pObj, bool callFunc) { + if (callFunc) { + SMnode *pMnode = pSdb->pMnode; + pMnode->deleteTimes++; + } + return 0; +} int32_t strUpdate(SSdb *pSdb, SStrObj *pOld, SStrObj *pNew) { pOld->v8 = pNew->v8; @@ -121,6 +282,24 @@ int32_t strUpdate(SSdb *pSdb, SStrObj *pOld, SStrObj *pNew) { return 0; } +int32_t i32Update(SSdb *pSdb, SI32Obj *pOld, SI32Obj *pNew) { + pOld->v8 = pNew->v8; + pOld->v16 = pNew->v16; + pOld->v32 = pNew->v32; + pOld->v64 = pNew->v64; + strcpy(pOld->vstr, pNew->vstr); + return 0; +} + +int32_t i64Update(SSdb *pSdb, SI64Obj *pOld, SI64Obj *pNew) { + pOld->v8 = pNew->v8; + pOld->v16 = pNew->v16; + pOld->v32 = pNew->v32; + pOld->v64 = pNew->v64; + strcpy(pOld->vstr, pNew->vstr); + return 0; +} + void strSetDefault(SStrObj *pObj, int32_t index) { memset(pObj, 0, sizeof(SStrObj)); snprintf(pObj->key, sizeof(pObj->key), "k%d", index * 1000); @@ -131,6 +310,26 @@ void strSetDefault(SStrObj *pObj, int32_t index) { snprintf(pObj->vstr, sizeof(pObj->vstr), "v%d", index * 1000); } +void i32SetDefault(SI32Obj *pObj, int32_t index) { + memset(pObj, 0, sizeof(SI32Obj)); + pObj->key = index; + pObj->v8 = index; + pObj->v16 = index; + pObj->v32 = index * 1000; + pObj->v64 = index * 1000; + snprintf(pObj->vstr, sizeof(pObj->vstr), "v%d", index * 1000); +} + +void i64SetDefault(SI64Obj *pObj, int32_t index) { + memset(pObj, 0, sizeof(SI64Obj)); + pObj->key = index; + pObj->v8 = index; + pObj->v16 = index; + pObj->v32 = index * 1000; + pObj->v64 = index * 1000; + snprintf(pObj->vstr, sizeof(pObj->vstr), "v%d", index * 1000); +} + int32_t strDefault(SMnode *pMnode) { SStrObj strObj; SSdbRaw *pRaw = NULL; @@ -144,6 +343,8 @@ int32_t strDefault(SMnode *pMnode) { pRaw = strEncode(&strObj); sdbSetRawStatus(pRaw, SDB_STATUS_READY); if (sdbWriteWithoutFree(pMnode->pSdb, pRaw) != 0) return -1; + + EXPECT_EQ(sdbGetRawTotalSize(pRaw), 79); sdbFreeRaw(pRaw); return 0; @@ -151,28 +352,32 @@ int32_t strDefault(SMnode *pMnode) { bool sdbTraverseSucc1(SMnode *pMnode, SStrObj *pObj, int32_t *p1, int32_t *p2, int32_t *p3) { if (pObj->v8 == 1) { - *p1 = *p2 + *p3 + pObj->v8; + *p1 += *p2 + *p3 + pObj->v8; } return true; } bool sdbTraverseSucc2(SMnode *pMnode, SStrObj *pObj, int32_t *p1, int32_t *p2, int32_t *p3) { - *p1 = *p2 + *p3 + pObj->v8; + *p1 += *p2 + *p3 + pObj->v8; return true; } bool sdbTraverseFail(SMnode *pMnode, SStrObj *pObj, int32_t *p1, int32_t *p2, int32_t *p3) { - *p1 = *p2 + *p3; + *p1 += *p2 + *p3; return false; } -TEST_F(MndTestSdb, 01_Write) { - void *pIter; - int32_t num; - SStrObj *pObj; - SMnode mnode; - SSdb *pSdb; +TEST_F(MndTestSdb, 01_Write_Str) { + void *pIter = NULL; + int32_t num = 0; + SStrObj *pObj = NULL; + SMnode mnode = {0}; + SSdb *pSdb = NULL; SSdbOpt opt = {0}; + SStrObj strObj = {0}; + SI32Obj i32Obj = {0}; + SI64Obj i64Obj = {0}; + SSdbRaw *pRaw = NULL; int32_t p1 = 0; int32_t p2 = 111; int32_t p3 = 222; @@ -183,42 +388,64 @@ TEST_F(MndTestSdb, 01_Write) { opt.path = "/tmp/mnode_test_sdb"; taosRemoveDir(opt.path); - SSdbTable strTable = { - .sdbType = SDB_USER, - .keyType = SDB_KEY_BINARY, - .deployFp = (SdbDeployFp)strDefault, - .encodeFp = (SdbEncodeFp)strEncode, - .decodeFp = (SdbDecodeFp)strDecode, - .insertFp = (SdbInsertFp)strInsert, - .updateFp = (SdbUpdateFp)strUpdate, - .deleteFp = (SdbDeleteFp)strDelete, - }; + SSdbTable strTable1; + memset(&strTable1, 0, sizeof(SSdbTable)); + strTable1.sdbType = SDB_USER; + strTable1.keyType = SDB_KEY_BINARY; + strTable1.deployFp = (SdbDeployFp)strDefault; + strTable1.encodeFp = (SdbEncodeFp)strEncode; + strTable1.decodeFp = (SdbDecodeFp)strDecode; + strTable1.insertFp = (SdbInsertFp)strInsert; + strTable1.updateFp = (SdbUpdateFp)strUpdate; + strTable1.deleteFp = (SdbDeleteFp)strDelete; + + SSdbTable strTable2; + memset(&strTable2, 0, sizeof(SSdbTable)); + strTable2.sdbType = SDB_VGROUP; + strTable2.keyType = SDB_KEY_INT32; + strTable2.encodeFp = (SdbEncodeFp)i32Encode; + strTable2.decodeFp = (SdbDecodeFp)i32Decode; + strTable2.insertFp = (SdbInsertFp)i32Insert; + strTable2.updateFp = (SdbUpdateFp)i32Update; + strTable2.deleteFp = (SdbDeleteFp)i32Delete; + + SSdbTable strTable3; + memset(&strTable3, 0, sizeof(SSdbTable)); + strTable3.sdbType = SDB_CONSUMER; + strTable3.keyType = SDB_KEY_INT64; + strTable3.encodeFp = (SdbEncodeFp)i64Encode; + strTable3.decodeFp = (SdbDecodeFp)i64Decode; + strTable3.insertFp = (SdbInsertFp)i64Insert; + strTable3.updateFp = (SdbUpdateFp)i64Update; + strTable3.deleteFp = (SdbDeleteFp)i64Delete; pSdb = sdbInit(&opt); mnode.pSdb = pSdb; ASSERT_NE(pSdb, nullptr); - ASSERT_EQ(sdbSetTable(pSdb, strTable), 0); + ASSERT_EQ(sdbSetTable(pSdb, strTable1), 0); + ASSERT_EQ(sdbSetTable(pSdb, strTable2), 0); + ASSERT_EQ(sdbSetTable(pSdb, strTable3), 0); ASSERT_EQ(sdbDeploy(pSdb), 0); -#if 0 + pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k1000"); ASSERT_NE(pObj, nullptr); EXPECT_STREQ(pObj->key, "k1000"); EXPECT_STREQ(pObj->vstr, "v1000"); - EXPECT_EQ(pObj->v8, 1); - EXPECT_EQ(pObj->v16, 1); - EXPECT_EQ(pObj->v32, 1000); - EXPECT_EQ(pObj->v64, 1000); + ASSERT_EQ(pObj->v8, 1); + ASSERT_EQ(pObj->v16, 1); + ASSERT_EQ(pObj->v32, 1000); + ASSERT_EQ(pObj->v64, 1000); sdbRelease(pSdb, pObj); pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k2000"); ASSERT_NE(pObj, nullptr); EXPECT_STREQ(pObj->key, "k2000"); EXPECT_STREQ(pObj->vstr, "v2000"); - EXPECT_EQ(pObj->v8, 2); - EXPECT_EQ(pObj->v16, 2); - EXPECT_EQ(pObj->v32, 2000); - EXPECT_EQ(pObj->v64, 2000); + ASSERT_EQ(pObj->v8, 2); + ASSERT_EQ(pObj->v16, 2); + ASSERT_EQ(pObj->v32, 2000); + ASSERT_EQ(pObj->v64, 2000); sdbRelease(pSdb, pObj); pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k200"); @@ -233,7 +460,7 @@ TEST_F(MndTestSdb, 01_Write) { num++; sdbRelease(pSdb, pObj); } while (1); - EXPECT_EQ(num, 2); + ASSERT_EQ(num, 2); do { pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pObj); @@ -248,51 +475,254 @@ TEST_F(MndTestSdb, 01_Write) { p1 = 0; p2 = 111; p3 = 222; - sdbTraverse(pSdb, SDB_USER, (sdbTraverseFp)sdbTraverseSucc2, &p1, &p2, &p3); - EXPECT_EQ(p1, 334); + sdbTraverse(pSdb, SDB_USER, (sdbTraverseFp)sdbTraverseSucc1, &p1, &p2, &p3); + ASSERT_EQ(p1, 334); p1 = 0; p2 = 111; p3 = 222; sdbTraverse(pSdb, SDB_USER, (sdbTraverseFp)sdbTraverseSucc2, &p1, &p2, &p3); - EXPECT_EQ(p1, 669); + ASSERT_EQ(p1, 669); p1 = 0; p2 = 111; p3 = 222; sdbTraverse(pSdb, SDB_USER, (sdbTraverseFp)sdbTraverseFail, &p1, &p2, &p3); - EXPECT_EQ(p1, 333); + ASSERT_EQ(p1, 333); + + ASSERT_EQ(sdbGetSize(pSdb, SDB_USER), 2); + ASSERT_EQ(sdbGetMaxId(pSdb, SDB_USER), -1); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_USER), 2 ); + ASSERT_EQ(sdbUpdateVer(pSdb, 0), -1); + ASSERT_EQ(sdbUpdateVer(pSdb, 1), 0); + ASSERT_EQ(sdbUpdateVer(pSdb, -1), -1); + ASSERT_EQ(mnode.insertTimes, 2); + ASSERT_EQ(mnode.deleteTimes, 0); + + { + // insert, call func + strSetDefault(&strObj, 3); + pRaw = strEncode(&strObj); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); + pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k3000"); + ASSERT_NE(pObj, nullptr); + EXPECT_STREQ(pObj->key, "k3000"); + EXPECT_STREQ(pObj->vstr, "v3000"); + ASSERT_EQ(pObj->v8, 3); + ASSERT_EQ(pObj->v16, 3); + ASSERT_EQ(pObj->v32, 3000); + ASSERT_EQ(pObj->v64, 3000); + sdbRelease(pSdb, pObj); - EXPECT_EQ(sdbGetSize(pSdb, SDB_USER), 2); - EXPECT_EQ(sdbGetMaxId(pSdb, SDB_USER), -1); - EXPECT_EQ(sdbGetTableVer(pSdb, SDB_USER), 2); - EXPECT_EQ(sdbUpdateVer(pSdb, 0), 2); - EXPECT_EQ(sdbUpdateVer(pSdb, 1), 3); - EXPECT_EQ(sdbUpdateVer(pSdb, -1), 2); + ASSERT_EQ(sdbGetSize(pSdb, SDB_USER), 3); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_USER), 3); + ASSERT_EQ(sdbGetMaxId(pSdb, SDB_USER), -1); + + // update, call func + strSetDefault(&strObj, 3); + strObj.v8 = 4; + pRaw = strEncode(&strObj); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); + pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k3000"); + ASSERT_NE(pObj, nullptr); + EXPECT_STREQ(pObj->key, "k3000"); + EXPECT_STREQ(pObj->vstr, "v3000"); + ASSERT_EQ(pObj->v8, 4); + ASSERT_EQ(pObj->v16, 3); + ASSERT_EQ(pObj->v32, 3000); + ASSERT_EQ(pObj->v64, 3000); + sdbRelease(pSdb, pObj); - // insert, call func + ASSERT_EQ(sdbGetSize(pSdb, SDB_USER), 3); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_USER), 4); + ASSERT_EQ(sdbUpdateVer(pSdb, 0), -1); + ASSERT_EQ(sdbUpdateVer(pSdb, 1), 0); + ASSERT_EQ(sdbUpdateVer(pSdb, -1), -1); + ASSERT_EQ(mnode.insertTimes, 3); + ASSERT_EQ(mnode.deleteTimes, 0); + + // delete, call func 2 + strSetDefault(&strObj, 3); + strObj.v16 = 4; + pRaw = strEncode(&strObj); + sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); + ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); + pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k3000"); + ASSERT_EQ(pObj, nullptr); + ASSERT_EQ(sdbGetSize(pSdb, SDB_USER), 2); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_USER), 5); + ASSERT_EQ(mnode.insertTimes, 3); + ASSERT_EQ(mnode.deleteTimes, 1); + } - // update, call func + { + int32_t key = 4; + i32SetDefault(&i32Obj, key); + pRaw = i32Encode(&i32Obj); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); + SI32Obj *pI32Obj = (SI32Obj *)sdbAcquire(pSdb, SDB_VGROUP, &key); + ASSERT_NE(pI32Obj, nullptr); + ASSERT_EQ(pI32Obj->key, key); + ASSERT_EQ(pI32Obj->v8, 4); + ASSERT_EQ(pI32Obj->v16, 4); + ASSERT_EQ(pI32Obj->v32, 4000); + ASSERT_EQ(pI32Obj->v64, 4000); + sdbRelease(pSdb, pI32Obj); + + ASSERT_EQ(sdbGetSize(pSdb, SDB_VGROUP), 1); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_VGROUP), 1); + ASSERT_EQ(sdbGetMaxId(pSdb, SDB_VGROUP), 5); + + i32SetDefault(&i32Obj, key); + i32Obj.v8 = 5; + pRaw = i32Encode(&i32Obj); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); + pI32Obj = (SI32Obj *)sdbAcquire(pSdb, SDB_VGROUP, &key); + ASSERT_NE(pI32Obj, nullptr); + ASSERT_EQ(pI32Obj->key, key); + ASSERT_EQ(pI32Obj->v8, 5); + ASSERT_EQ(pI32Obj->v16, 4); + ASSERT_EQ(pI32Obj->v32, 4000); + ASSERT_EQ(pI32Obj->v64, 4000); + sdbRelease(pSdb, pI32Obj); + + ASSERT_EQ(sdbGetSize(pSdb, SDB_VGROUP), 1); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_VGROUP), 2); + ASSERT_EQ(mnode.insertTimes, 4); + ASSERT_EQ(mnode.deleteTimes, 1); + + // delete, call func 2 + key = 4; + i32SetDefault(&i32Obj, key); + pRaw = i32Encode(&i32Obj); + sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); + ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); + pI32Obj = (SI32Obj *)sdbAcquire(pSdb, SDB_VGROUP, &key); + ASSERT_EQ(pI32Obj, nullptr); + ASSERT_EQ(sdbGetSize(pSdb, SDB_VGROUP), 0); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_VGROUP), 3); + ASSERT_EQ(sdbGetMaxId(pSdb, SDB_VGROUP), 5); + ASSERT_EQ(mnode.insertTimes, 4); + ASSERT_EQ(mnode.deleteTimes, 2); + + key = 6; + i32SetDefault(&i32Obj, key); + pRaw = i32Encode(&i32Obj); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); + pI32Obj = (SI32Obj *)sdbAcquire(pSdb, SDB_VGROUP, &key); + ASSERT_NE(pI32Obj, nullptr); + ASSERT_EQ(pI32Obj->key, key); + ASSERT_EQ(pI32Obj->v8, 6); + ASSERT_EQ(pI32Obj->v16, 6); + ASSERT_EQ(pI32Obj->v32, 6000); + ASSERT_EQ(pI32Obj->v64, 6000); + sdbRelease(pSdb, pI32Obj); + + ASSERT_EQ(sdbGetSize(pSdb, SDB_VGROUP), 1); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_VGROUP), 4); + ASSERT_EQ(sdbGetMaxId(pSdb, SDB_VGROUP), 7); + ASSERT_EQ(mnode.insertTimes, 5); + ASSERT_EQ(mnode.deleteTimes, 2); + } - // delete, call func 2 + { + int64_t key = 4; + i64SetDefault(&i64Obj, key); + pRaw = i64Encode(&i64Obj); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); + SI64Obj *pI64Obj = (SI64Obj *)sdbAcquire(pSdb, SDB_CONSUMER, &key); + ASSERT_NE(pI64Obj, nullptr); + ASSERT_EQ(pI64Obj->key, key); + ASSERT_EQ(pI64Obj->v8, 4); + ASSERT_EQ(pI64Obj->v16, 4); + ASSERT_EQ(pI64Obj->v32, 4000); + ASSERT_EQ(pI64Obj->v64, 4000); + sdbRelease(pSdb, pI64Obj); + + ASSERT_EQ(sdbGetSize(pSdb, SDB_CONSUMER), 1); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_CONSUMER), 1); + ASSERT_EQ(sdbGetMaxId(pSdb, SDB_CONSUMER), -1); + + i64SetDefault(&i64Obj, key); + i64Obj.v8 = 5; + pRaw = i64Encode(&i64Obj); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); + pI64Obj = (SI64Obj *)sdbAcquire(pSdb, SDB_CONSUMER, &key); + ASSERT_NE(pI64Obj, nullptr); + ASSERT_EQ(pI64Obj->key, key); + ASSERT_EQ(pI64Obj->v8, 5); + ASSERT_EQ(pI64Obj->v16, 4); + ASSERT_EQ(pI64Obj->v32, 4000); + ASSERT_EQ(pI64Obj->v64, 4000); + sdbRelease(pSdb, pI64Obj); + + ASSERT_EQ(sdbGetSize(pSdb, SDB_CONSUMER), 1); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_CONSUMER), 2); + ASSERT_EQ(mnode.insertTimes, 6); + ASSERT_EQ(mnode.deleteTimes, 2); + + // delete, call func 2 + key = 4; + i64SetDefault(&i64Obj, key); + pRaw = i64Encode(&i64Obj); + sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); + ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); + pObj = (SStrObj *)sdbAcquire(pSdb, SDB_CONSUMER, &key); + ASSERT_EQ(pObj, nullptr); + ASSERT_EQ(sdbGetSize(pSdb, SDB_CONSUMER), 0); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_CONSUMER), 3); + ASSERT_EQ(sdbGetMaxId(pSdb, SDB_CONSUMER), -1); + ASSERT_EQ(mnode.insertTimes, 6); + ASSERT_EQ(mnode.deleteTimes, 3); + + key = 7; + i64SetDefault(&i64Obj, key); + pRaw = i64Encode(&i64Obj); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); + pI64Obj = (SI64Obj *)sdbAcquire(pSdb, SDB_CONSUMER, &key); + ASSERT_NE(pI64Obj, nullptr); + ASSERT_EQ(pI64Obj->key, key); + ASSERT_EQ(pI64Obj->v8, 7); + ASSERT_EQ(pI64Obj->v16, 7); + ASSERT_EQ(pI64Obj->v32, 7000); + ASSERT_EQ(pI64Obj->v64, 7000); + sdbRelease(pSdb, pI64Obj); + + ASSERT_EQ(sdbGetSize(pSdb, SDB_CONSUMER), 1); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_CONSUMER), 4); + ASSERT_EQ(sdbGetMaxId(pSdb, SDB_CONSUMER), -1); + ASSERT_EQ(mnode.insertTimes, 7); + ASSERT_EQ(mnode.deleteTimes, 3); + } // write version - - // sdb Write ver - - // sdbRead -#endif + ASSERT_EQ(sdbUpdateVer(pSdb, 1), 0); + ASSERT_EQ(sdbUpdateVer(pSdb, 1), 1); + ASSERT_EQ(sdbWriteFile(pSdb), 0); ASSERT_EQ(sdbWriteFile(pSdb), 0); + sdbCleanup(pSdb); + ASSERT_EQ(mnode.insertTimes, 7); + ASSERT_EQ(mnode.deleteTimes, 7); } -TEST_F(MndTestSdb, 01_Read) { - void *pIter; - int32_t num; - SStrObj *pObj; - SMnode mnode; - SSdb *pSdb; +TEST_F(MndTestSdb, 01_Read_Str) { + void *pIter = NULL; + int32_t num = 0; + SStrObj *pObj = NULL; + SMnode mnode = {0}; + SSdb *pSdb = NULL; SSdbOpt opt = {0}; + SStrObj strObj = {0}; + SSdbRaw *pRaw = NULL; int32_t p1 = 0; int32_t p2 = 111; int32_t p3 = 222; @@ -301,22 +731,174 @@ TEST_F(MndTestSdb, 01_Read) { mnode.v200 = 200; opt.pMnode = &mnode; opt.path = "/tmp/mnode_test_sdb"; - taosRemoveDir(opt.path); - SSdbTable strTable = { - .sdbType = SDB_USER, - .keyType = SDB_KEY_BINARY, - .deployFp = (SdbDeployFp)strDefault, - .encodeFp = (SdbEncodeFp)strEncode, - .decodeFp = (SdbDecodeFp)strDecode, - .insertFp = (SdbInsertFp)strInsert, - .updateFp = (SdbUpdateFp)strDelete, - .deleteFp = (SdbDeleteFp)strUpdate, - }; + SSdbTable strTable1; + memset(&strTable1, 0, sizeof(SSdbTable)); + strTable1.sdbType = SDB_USER; + strTable1.keyType = SDB_KEY_BINARY; + strTable1.deployFp = (SdbDeployFp)strDefault; + strTable1.encodeFp = (SdbEncodeFp)strEncode; + strTable1.decodeFp = (SdbDecodeFp)strDecode; + strTable1.insertFp = (SdbInsertFp)strInsert; + strTable1.updateFp = (SdbUpdateFp)strUpdate; + strTable1.deleteFp = (SdbDeleteFp)strDelete; + + SSdbTable strTable2; + memset(&strTable2, 0, sizeof(SSdbTable)); + strTable2.sdbType = SDB_VGROUP; + strTable2.keyType = SDB_KEY_INT32; + strTable2.encodeFp = (SdbEncodeFp)i32Encode; + strTable2.decodeFp = (SdbDecodeFp)i32Decode; + strTable2.insertFp = (SdbInsertFp)i32Insert; + strTable2.updateFp = (SdbUpdateFp)i32Update; + strTable2.deleteFp = (SdbDeleteFp)i32Delete; + + SSdbTable strTable3; + memset(&strTable3, 0, sizeof(SSdbTable)); + strTable3.sdbType = SDB_CONSUMER; + strTable3.keyType = SDB_KEY_INT64; + strTable3.encodeFp = (SdbEncodeFp)i64Encode; + strTable3.decodeFp = (SdbDecodeFp)i64Decode; + strTable3.insertFp = (SdbInsertFp)i64Insert; + strTable3.updateFp = (SdbUpdateFp)i64Update; + strTable3.deleteFp = (SdbDeleteFp)i64Delete; pSdb = sdbInit(&opt); mnode.pSdb = pSdb; - + ASSERT_NE(pSdb, nullptr); + ASSERT_NE(pSdb, nullptr); + ASSERT_EQ(sdbSetTable(pSdb, strTable1), 0); + ASSERT_EQ(sdbSetTable(pSdb, strTable2), 0); + ASSERT_EQ(sdbSetTable(pSdb, strTable3), 0); ASSERT_EQ(sdbReadFile(pSdb), 0); + + ASSERT_EQ(sdbGetSize(pSdb, SDB_USER), 2); + ASSERT_EQ(sdbGetMaxId(pSdb, SDB_USER), -1); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_USER), 5); + ASSERT_EQ(sdbUpdateVer(pSdb, 0), 1); + ASSERT_EQ(mnode.insertTimes, 4); + ASSERT_EQ(mnode.deleteTimes, 0); + + pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k1000"); + ASSERT_NE(pObj, nullptr); + EXPECT_STREQ(pObj->key, "k1000"); + EXPECT_STREQ(pObj->vstr, "v1000"); + ASSERT_EQ(pObj->v8, 1); + ASSERT_EQ(pObj->v16, 1); + ASSERT_EQ(pObj->v32, 1000); + ASSERT_EQ(pObj->v64, 1000); + sdbRelease(pSdb, pObj); + + pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k2000"); + ASSERT_NE(pObj, nullptr); + EXPECT_STREQ(pObj->key, "k2000"); + EXPECT_STREQ(pObj->vstr, "v2000"); + ASSERT_EQ(pObj->v8, 2); + ASSERT_EQ(pObj->v16, 2); + ASSERT_EQ(pObj->v32, 2000); + ASSERT_EQ(pObj->v64, 2000); + sdbRelease(pSdb, pObj); + + pObj = (SStrObj *)sdbAcquire(pSdb, SDB_USER, "k200"); + ASSERT_EQ(pObj, nullptr); + + pIter = NULL; + num = 0; + do { + pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pObj); + if (pIter == NULL) break; + ASSERT_NE(pObj, nullptr); + num++; + sdbRelease(pSdb, pObj); + } while (1); + ASSERT_EQ(num, 2); + + do { + pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pObj); + if (pIter == NULL) break; + if (strcmp(pObj->key, "k1000") == 0) { + sdbCancelFetch(pSdb, pIter); + break; + } + } while (1); + EXPECT_STREQ(pObj->key, "k1000"); + + p1 = 0; + p2 = 111; + p3 = 222; + sdbTraverse(pSdb, SDB_USER, (sdbTraverseFp)sdbTraverseSucc1, &p1, &p2, &p3); + ASSERT_EQ(p1, 334); + + p1 = 0; + p2 = 111; + p3 = 222; + sdbTraverse(pSdb, SDB_USER, (sdbTraverseFp)sdbTraverseSucc2, &p1, &p2, &p3); + ASSERT_EQ(p1, 669); + + p1 = 0; + p2 = 111; + p3 = 222; + sdbTraverse(pSdb, SDB_USER, (sdbTraverseFp)sdbTraverseFail, &p1, &p2, &p3); + ASSERT_EQ(p1, 333); + + int32_t i32key = 6; + SI32Obj *pI32Obj = (SI32Obj *)sdbAcquire(pSdb, SDB_VGROUP, &i32key); + ASSERT_NE(pI32Obj, nullptr); + ASSERT_EQ(pI32Obj->key, 6); + ASSERT_EQ(pI32Obj->v8, 6); + ASSERT_EQ(pI32Obj->v16, 6); + ASSERT_EQ(pI32Obj->v32, 6000); + ASSERT_EQ(pI32Obj->v64, 6000); + sdbRelease(pSdb, pI32Obj); + + ASSERT_EQ(sdbGetSize(pSdb, SDB_VGROUP), 1); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_VGROUP), 4); + ASSERT_EQ(sdbGetMaxId(pSdb, SDB_VGROUP), 7); + + int64_t i64key = 7; + SI64Obj *pI64Obj = (SI64Obj *)sdbAcquire(pSdb, SDB_CONSUMER, &i64key); + ASSERT_NE(pI64Obj, nullptr); + ASSERT_EQ(pI64Obj->key, 7); + ASSERT_EQ(pI64Obj->v8, 7); + ASSERT_EQ(pI64Obj->v16, 7); + ASSERT_EQ(pI64Obj->v32, 7000); + ASSERT_EQ(pI64Obj->v64, 7000); + sdbRelease(pSdb, pI64Obj); + + ASSERT_EQ(sdbGetSize(pSdb, SDB_CONSUMER), 1); + ASSERT_EQ(sdbGetTableVer(pSdb, SDB_CONSUMER), 4); + + ASSERT_EQ(mnode.insertTimes, 4); + ASSERT_EQ(mnode.deleteTimes, 0); + + { + SI32Obj i32Obj = {0}; + int32_t key = 6; + i32SetDefault(&i32Obj, key); + pRaw = i32Encode(&i32Obj); + sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING); + ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); + pI32Obj = (SI32Obj *)sdbAcquire(pSdb, SDB_VGROUP, &key); + ASSERT_EQ(pI32Obj, nullptr); + int32_t code = terrno; + ASSERT_EQ(code, TSDB_CODE_SDB_OBJ_DROPPING); + } + + { + SI32Obj i32Obj = {0}; + int32_t key = 8; + i32SetDefault(&i32Obj, key); + pRaw = i32Encode(&i32Obj); + EXPECT_NE(sdbSetRawStatus(pRaw, SDB_STATUS_INIT), 0); + sdbSetRawStatus(pRaw, SDB_STATUS_CREATING); + ASSERT_EQ(sdbWrite(pSdb, pRaw), 0); + pI32Obj = (SI32Obj *)sdbAcquire(pSdb, SDB_VGROUP, &key); + ASSERT_EQ(pI32Obj, nullptr); + int32_t code = terrno; + ASSERT_EQ(code, TSDB_CODE_SDB_OBJ_CREATING); + } + sdbCleanup(pSdb); + ASSERT_EQ(mnode.insertTimes, 5); + ASSERT_EQ(mnode.deleteTimes, 5); } \ No newline at end of file diff --git a/source/dnode/mnode/impl/test/sma/sma.cpp b/source/dnode/mnode/impl/test/sma/sma.cpp index 54a154e684b50abf4fea155bb5dd8da1efc47e78..4dc4e0477947d647961acd1859720f1013d6da45 100644 --- a/source/dnode/mnode/impl/test/sma/sma.cpp +++ b/source/dnode/mnode/impl/test/sma/sma.cpp @@ -40,24 +40,22 @@ void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { SCreateDbReq createReq = {0}; strcpy(createReq.db, dbname); createReq.numOfVgroups = 2; - createReq.cacheBlockSize = 16; - createReq.totalBlocks = 10; + createReq.buffer = -1; + createReq.pageSize = -1; + createReq.pages = -1; createReq.daysPerFile = 10 * 1440; createReq.daysToKeep0 = 3650 * 1440; createReq.daysToKeep1 = 3650 * 1440; createReq.daysToKeep2 = 3650 * 1440; createReq.minRows = 100; createReq.maxRows = 4096; - createReq.commitTime = 3600; createReq.fsyncPeriod = 3000; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.update = 0; createReq.cacheLastRow = 0; - createReq.ttl = 1; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index 9270d38e1148f925bfb6a7d754041d0f679ea05c..220a73def60780863819e1aa8a54412b2af565c7 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -41,24 +41,22 @@ void* MndTestStb::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { SCreateDbReq createReq = {0}; strcpy(createReq.db, dbname); createReq.numOfVgroups = 2; - createReq.cacheBlockSize = 16; - createReq.totalBlocks = 10; + createReq.buffer = -1; + createReq.pageSize = -1; + createReq.pages = -1; createReq.daysPerFile = 1000; createReq.daysToKeep0 = 3650; createReq.daysToKeep1 = 3650; createReq.daysToKeep2 = 3650; createReq.minRows = 100; createReq.maxRows = 4096; - createReq.commitTime = 3600; createReq.fsyncPeriod = 3000; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.update = 0; createReq.cacheLastRow = 0; - createReq.ttl = 1; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); @@ -343,7 +341,6 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { EXPECT_EQ(metaRsp.numOfTags, 3); EXPECT_EQ(metaRsp.precision, TSDB_TIME_PRECISION_MILLI); EXPECT_EQ(metaRsp.tableType, TSDB_SUPER_TABLE); - EXPECT_EQ(metaRsp.update, 0); EXPECT_EQ(metaRsp.sversion, 1); EXPECT_EQ(metaRsp.tversion, 0); EXPECT_GT(metaRsp.suid, 0); diff --git a/source/dnode/mnode/impl/test/topic/topic.cpp b/source/dnode/mnode/impl/test/topic/topic.cpp index 917cb23fc9b4cd2cec2204a2b0f6494a638f9b57..eccc1b99d3f3dc8189712776e4ee852c6169cdfa 100644 --- a/source/dnode/mnode/impl/test/topic/topic.cpp +++ b/source/dnode/mnode/impl/test/topic/topic.cpp @@ -33,24 +33,22 @@ void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { SCreateDbReq createReq = {0}; strcpy(createReq.db, dbname); createReq.numOfVgroups = 2; - createReq.cacheBlockSize = 16; - createReq.totalBlocks = 10; + createReq.buffer = -1; + createReq.pageSize = -1; + createReq.pages = -1; createReq.daysPerFile = 10 * 1440; createReq.daysToKeep0 = 3650 * 1440; createReq.daysToKeep1 = 3650 * 1440; createReq.daysToKeep2 = 3650 * 1440; createReq.minRows = 100; createReq.maxRows = 4096; - createReq.commitTime = 3600; createReq.fsyncPeriod = 3000; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.update = 0; createReq.cacheLastRow = 0; - createReq.ttl = 1; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); diff --git a/source/dnode/mnode/impl/test/trans/CMakeLists.txt b/source/dnode/mnode/impl/test/trans/CMakeLists.txt index fa0ef9f26392d977ea4c091d5a714bb7b2a466b4..3931433c1934f7316a59813d26c2eaf1f8c387b1 100644 --- a/source/dnode/mnode/impl/test/trans/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/trans/CMakeLists.txt @@ -1,11 +1,37 @@ -aux_source_directory(. MNODE_TRANS_TEST_SRC) -add_executable(transTest ${MNODE_TRANS_TEST_SRC}) +add_executable(transTest1 "") +target_sources(transTest1 + PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/trans1.cpp" +) target_link_libraries( - transTest + transTest1 PUBLIC sut ) +target_include_directories( + transTest1 + PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" +) +add_test( + NAME transTest1 + COMMAND transTest1 +) +add_executable(transTest2 "") +target_sources(transTest2 + PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/trans2.cpp" +) +target_link_libraries( + transTest2 + PUBLIC dnode mnode gtest_main +) +target_include_directories( + transTest2 + PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" +) add_test( - NAME transTest - COMMAND transTest + NAME transTest2 + COMMAND transTest2 ) diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans1.cpp similarity index 93% rename from source/dnode/mnode/impl/test/trans/trans.cpp rename to source/dnode/mnode/impl/test/trans/trans1.cpp index bcf6fe8536e5468114bcbc9d736ebac328eabe27..07bf9a2bcf35dba70c2cc79e3803c075baa77731 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans1.cpp @@ -11,10 +11,10 @@ #include "sut.h" -class MndTestTrans : public ::testing::Test { +class MndTestTrans1 : public ::testing::Test { protected: static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_trans", 9013); + test.Init("/tmp/mnode_test_trans1", 9013); const char* fqdn = "localhost"; const char* firstEp = "localhost:9013"; server2.Start("/tmp/mnode_test_trans2", fqdn, 9020, firstEp); @@ -26,7 +26,7 @@ class MndTestTrans : public ::testing::Test { } static void KillThenRestartServer() { - char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; + char file[PATH_MAX] = "/tmp/mnode_test_trans1/mnode/data/sdb.data"; TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); int32_t size = 3 * 1024 * 1024; void* buffer = taosMemoryMalloc(size); @@ -60,10 +60,10 @@ class MndTestTrans : public ::testing::Test { void TearDown() override {} }; -Testbase MndTestTrans::test; -TestServer MndTestTrans::server2; +Testbase MndTestTrans1::test; +TestServer MndTestTrans1::server2; -TEST_F(MndTestTrans, 00_Create_User_Crash) { +TEST_F(MndTestTrans1, 00_Create_User_Crash) { { test.SendShowReq(TSDB_MGMT_TABLE_TRANS, "trans", ""); EXPECT_EQ(test.GetShowRows(), 0); @@ -83,7 +83,7 @@ TEST_F(MndTestTrans, 00_Create_User_Crash) { } } -TEST_F(MndTestTrans, 01_Create_User_Crash) { +TEST_F(MndTestTrans1, 01_Create_User_Crash) { { SCreateUserReq createReq = {0}; strcpy(createReq.user, "u1"); @@ -107,7 +107,7 @@ TEST_F(MndTestTrans, 01_Create_User_Crash) { EXPECT_EQ(test.GetShowRows(), 2); } -TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) { +TEST_F(MndTestTrans1, 02_Create_Qnode1_Crash) { { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; @@ -142,7 +142,7 @@ TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) { } } -TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { +TEST_F(MndTestTrans1, 03_Create_Qnode2_Crash) { { SCreateDnodeReq createReq = {0}; strcpy(createReq.fqdn, "localhost"); diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b3cbcb6898324ce647a8d025e4ee0a3c65df18cc --- /dev/null +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -0,0 +1,513 @@ +/** + * @file trans.cpp + * @author slguan (slguan@taosdata.com) + * @brief MNODE module trans tests + * @version 1.0 + * @date 2022-05-02 + * + * @copyright Copyright (c) 2022 + * + */ + +#include + +#include "mndTrans.h" +#include "mndUser.h" +#include "tcache.h" + +void reportStartup(SMgmtWrapper *pWrapper, const char *name, const char *desc) {} + +class MndTestTrans2 : public ::testing::Test { + protected: + static void InitLog() { + dDebugFlag = 143; + vDebugFlag = 0; + mDebugFlag = 207; + cDebugFlag = 0; + jniDebugFlag = 0; + tmrDebugFlag = 135; + uDebugFlag = 135; + rpcDebugFlag = 143; + qDebugFlag = 0; + wDebugFlag = 0; + sDebugFlag = 0; + tsdbDebugFlag = 0; + tsLogEmbedded = 1; + tsAsyncLog = 0; + + const char *logpath = "/tmp/td"; + taosRemoveDir(logpath); + taosMkDir(logpath); + tstrncpy(tsLogDir, logpath, PATH_MAX); + if (taosInitLog("taosdlog", 1) != 0) { + printf("failed to init log file\n"); + } + } + + static void InitMnode() { + static SMsgCb msgCb = {0}; + msgCb.reportStartupFp = reportStartup; + msgCb.pWrapper = (SMgmtWrapper *)(&msgCb); // hack + tmsgSetDefaultMsgCb(&msgCb); + + SMnodeOpt opt = {0}; + opt.deploy = 1; + opt.replica = 1; + opt.replicas[0].id = 1; + opt.replicas[0].port = 9040; + strcpy(opt.replicas[0].fqdn, "localhost"); + opt.msgCb = msgCb; + + tsTransPullupMs = 1000; + + const char *mnodepath = "/tmp/mnode_test_trans"; + taosRemoveDir(mnodepath); + pMnode = mndOpen(mnodepath, &opt); + mndStart(pMnode); + } + + static void SetUpTestSuite() { + InitLog(); + walInit(); + InitMnode(); + } + + static void TearDownTestSuite() { + mndStop(pMnode); + mndClose(pMnode); + walCleanUp(); + taosCloseLog(); + taosStopCacheRefreshWorker(); + } + + static SMnode *pMnode; + + public: + void SetUp() override {} + void TearDown() override {} + + int32_t CreateUserLog(const char *acct, const char *user, ETrnType type, SDbObj *pDb) { + SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); + tstrncpy(userObj.user, user, TSDB_USER_LEN); + tstrncpy(userObj.acct, acct, TSDB_USER_LEN); + userObj.createdTime = taosGetTimestampMs(); + userObj.updateTime = userObj.createdTime; + userObj.superUser = 1; + + SRpcMsg rpcMsg = {0}; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, type, &rpcMsg); + SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); + mndTransAppendRedolog(pTrans, pRedoRaw); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); + mndTransAppendUndolog(pTrans, pUndoRaw); + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + char *param = strdup("====> test log <====="); + mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + + if (pDb != NULL) { + mndTransSetDbInfo(pTrans, pDb); + } + + int32_t code = mndTransPrepare(pMnode, pTrans); + mndTransDrop(pTrans); + + return code; + } + + int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction, ETrnPolicy policy, ETrnType type, + SDbObj *pDb) { + SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); + tstrncpy(userObj.user, user, TSDB_USER_LEN); + tstrncpy(userObj.acct, acct, TSDB_USER_LEN); + userObj.createdTime = taosGetTimestampMs(); + userObj.updateTime = userObj.createdTime; + userObj.superUser = 1; + + SRpcMsg rpcMsg = {0}; + STrans *pTrans = mndTransCreate(pMnode, policy, type, &rpcMsg); + SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); + mndTransAppendRedolog(pTrans, pRedoRaw); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); + mndTransAppendUndolog(pTrans, pUndoRaw); + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + char *param = strdup("====> test action <====="); + mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + + { + STransAction action = {0}; + action.epSet.inUse = 0; + action.epSet.numOfEps = 1; + action.epSet.eps[0].port = 9040; + strcpy(action.epSet.eps[0].fqdn, "localhost"); + + int32_t contLen = 1024; + void *pReq = taosMemoryCalloc(1, contLen); + strcpy((char *)pReq, "hello world redo"); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_CREATE_MNODE; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + mndTransAppendRedoAction(pTrans, &action); + } + + if (hasUndoAction) { + STransAction action = {0}; + action.epSet.inUse = 0; + action.epSet.numOfEps = 1; + action.epSet.eps[0].port = 9040; + strcpy(action.epSet.eps[0].fqdn, "localhost"); + + int32_t contLen = 1024; + void *pReq = taosMemoryCalloc(1, contLen); + strcpy((char *)pReq, "hello world undo"); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_CREATE_MNODE; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + mndTransAppendUndoAction(pTrans, &action); + } + + { + void *pRsp = taosMemoryCalloc(1, 256); + strcpy((char *)pRsp, "simple rsponse"); + mndTransSetRpcRsp(pTrans, pRsp, 256); + } + + if (pDb != NULL) { + mndTransSetDbInfo(pTrans, pDb); + } + + int32_t code = mndTransPrepare(pMnode, pTrans); + mndTransDrop(pTrans); + + return code; + } + + int32_t CreateUserGlobal(const char *acct, const char *user) { + SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); + tstrncpy(userObj.user, user, TSDB_USER_LEN); + tstrncpy(userObj.acct, acct, TSDB_USER_LEN); + userObj.createdTime = taosGetTimestampMs(); + userObj.updateTime = userObj.createdTime; + userObj.superUser = 1; + + SRpcMsg rpcMsg = {0}; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg); + SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); + mndTransAppendRedolog(pTrans, pRedoRaw); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); + mndTransAppendUndolog(pTrans, pUndoRaw); + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + char *param = strdup("====> test log <====="); + mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + + int32_t code = mndTransPrepare(pMnode, pTrans); + mndTransDrop(pTrans); + + return code; + } +}; + +SMnode *MndTestTrans2::pMnode; + +TEST_F(MndTestTrans2, 01_Log) { + const char *acct = "root"; + const char *acct_invalid = "root1"; + const char *user1 = "log1"; + const char *user2 = "log2"; + SUserObj *pUser1 = NULL; + SUserObj *pUser2 = NULL; + + ASSERT_NE(pMnode, nullptr); + + EXPECT_EQ(CreateUserLog(acct, user1, TRN_TYPE_CREATE_USER, NULL), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + + // failed to create user and rollback + EXPECT_EQ(CreateUserLog(acct_invalid, user2, TRN_TYPE_CREATE_USER, NULL), 0); + pUser2 = mndAcquireUser(pMnode, user2); + ASSERT_EQ(pUser2, nullptr); + + mndTransPullup(pMnode); +} + +TEST_F(MndTestTrans2, 02_Action) { + const char *acct = "root"; + const char *acct_invalid = "root1"; + const char *user1 = "action1"; + const char *user2 = "action2"; + SUserObj *pUser1 = NULL; + SUserObj *pUser2 = NULL; + STrans *pTrans = NULL; + int32_t transId = 0; + int32_t action = 0; + + ASSERT_NE(pMnode, nullptr); + + { + // failed to create user and rollback + EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + // create user, and fake a response + { + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + transId = 4; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } + } + + { + EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_RETRY, TRN_TYPE_CREATE_USER, NULL), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + { + transId = 5; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_REDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->redoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + rspMsg.rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } + + { + transId = 5; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + EXPECT_EQ(pTrans->stage, TRN_STAGE_REDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 2); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->redoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } + } + + { + EXPECT_EQ(CreateUserAction(acct, user2, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); + SUserObj *pUser2 = (SUserObj *)sdbAcquire(pMnode->pSdb, SDB_USER, user2); + ASSERT_NE(pUser2, nullptr); + mndReleaseUser(pMnode, pUser2); + + { + transId = 6; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + rspMsg.rpcMsg.code = 0; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser2 = mndAcquireUser(pMnode, user2); + ASSERT_NE(pUser2, nullptr); + mndReleaseUser(pMnode, pUser2); + } + + { + transId = 6; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 2); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser2 = mndAcquireUser(pMnode, user2); + ASSERT_EQ(pUser2, nullptr); + mndReleaseUser(pMnode, pUser2); + } + } +} + +TEST_F(MndTestTrans2, 03_Kill) { + const char *acct = "root"; + const char *user1 = "kill1"; + const char *user2 = "kill2"; + SUserObj *pUser1 = NULL; + SUserObj *pUser2 = NULL; + STrans *pTrans = NULL; + int32_t transId = 0; + int32_t action = 0; + + ASSERT_NE(pMnode, nullptr); + + { + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + transId = 7; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + mndKillTrans(pMnode, pTrans); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } +} + +TEST_F(MndTestTrans2, 04_Conflict) { + const char *acct = "root"; + const char *user1 = "conflict1"; + const char *user2 = "conflict2"; + const char *user3 = "conflict3"; + const char *user4 = "conflict4"; + const char *user5 = "conflict5"; + const char *user6 = "conflict6"; + const char *user7 = "conflict7"; + const char *user8 = "conflict8"; + SUserObj *pUser = NULL; + STrans *pTrans = NULL; + int32_t transId = 0; + int32_t code = 0; + + ASSERT_NE(pMnode, nullptr); + + { + SDbObj dbObj1 = {0}; + dbObj1.uid = 9521; + strcpy(dbObj1.name, "db"); + SDbObj dbObj2 = {0}; + dbObj2.uid = 9522; + strcpy(dbObj2.name, "conflict db"); + + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &dbObj1), 0); + pUser = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser, nullptr); + mndReleaseUser(pMnode, pUser); + + transId = 8; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + + // stb scope + EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DNODE, NULL), -1); + code = terrno; + EXPECT_EQ(code, TSDB_CODE_MND_TRANS_CONFLICT); + + EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DB, &dbObj2), 0); + EXPECT_EQ(CreateUserLog(acct, user3, TRN_TYPE_CREATE_STB, &dbObj1), 0); + + // db scope + pTrans->type = TRN_TYPE_CREATE_DB; + EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DNODE, NULL), -1); + EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DB, &dbObj2), 0); + EXPECT_EQ(CreateUserLog(acct, user5, TRN_TYPE_CREATE_STB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user5, TRN_TYPE_CREATE_STB, &dbObj2), 0); + + // global scope + pTrans->type = TRN_TYPE_CREATE_DNODE; + EXPECT_EQ(CreateUserLog(acct, user6, TRN_TYPE_CREATE_DNODE, NULL), 0); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj2), -1); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_STB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_STB, &dbObj2), -1); + + // global scope + pTrans->type = TRN_TYPE_CREATE_USER; + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj1), 0); + EXPECT_EQ(CreateUserLog(acct, user8, TRN_TYPE_CREATE_DB, &dbObj2), 0); + + mndKillTrans(pMnode, pTrans); + mndReleaseTrans(pMnode, pTrans); + + pUser = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser, nullptr); + mndReleaseUser(pMnode, pUser); + } +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index 79464db134c1b1b6f842a68a72351ddfabf92716..ee961e9a275aac5e259aff5f6d93021e24231f33 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -286,24 +286,22 @@ TEST_F(MndTestUser, 03_Alter_User) { SCreateDbReq createReq = {0}; strcpy(createReq.db, "1.d2"); createReq.numOfVgroups = 2; - createReq.cacheBlockSize = 16; - createReq.totalBlocks = 10; + createReq.buffer = -1; + createReq.pageSize = -1; + createReq.pages = -1; createReq.daysPerFile = 10 * 1440; createReq.daysToKeep0 = 3650 * 1440; createReq.daysToKeep1 = 3650 * 1440; createReq.daysToKeep2 = 3650 * 1440; createReq.minRows = 100; createReq.maxRows = 4096; - createReq.commitTime = 3600; createReq.fsyncPeriod = 3000; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.update = 0; createReq.cacheLastRow = 0; - createReq.ttl = 1; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); diff --git a/source/dnode/mnode/sdb/inc/sdbInt.h b/source/dnode/mnode/sdb/inc/sdbInt.h index 563fc72d0028acb7edf0d5c91b34e8903b5b1038..23c0f8a01c19ffa1b64dd15fd6d38dc893c8b130 100644 --- a/source/dnode/mnode/sdb/inc/sdbInt.h +++ b/source/dnode/mnode/sdb/inc/sdbInt.h @@ -31,8 +31,6 @@ extern "C" { #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }} #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} -#define SDB_MAX_SIZE (32 * 1024) - typedef struct SSdbRaw { int8_t type; int8_t status; diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index d180271175dc557a531b4f8f2f040ed2f914abd8..51f40c12cd65a61e83f0395d6f6dc4783f9f03de 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -28,12 +28,12 @@ SSdb *sdbInit(SSdbOpt *pOption) { return NULL; } - char path[PATH_MAX + 100]; - snprintf(path, PATH_MAX + 100, "%s%sdata", pOption->path, TD_DIRSEP); + char path[PATH_MAX + 100] = {0}; + snprintf(path, sizeof(path), "%s%sdata", pOption->path, TD_DIRSEP); pSdb->currDir = strdup(path); - snprintf(path, PATH_MAX + 100, "%s%ssync", pOption->path, TD_DIRSEP); + snprintf(path, sizeof(path), "%s%ssync", pOption->path, TD_DIRSEP); pSdb->syncDir = strdup(path); - snprintf(path, PATH_MAX + 100, "%s%stmp", pOption->path, TD_DIRSEP); + snprintf(path, sizeof(path), "%s%stmp", pOption->path, TD_DIRSEP); pSdb->tmpDir = strdup(path); if (pSdb->currDir == NULL || pSdb->currDir == NULL || pSdb->currDir == NULL) { sdbCleanup(pSdb); @@ -50,7 +50,7 @@ SSdb *sdbInit(SSdbOpt *pOption) { for (ESdbType i = 0; i < SDB_MAX; ++i) { taosInitRWLatch(&pSdb->locks[i]); pSdb->maxId[i] = 0; - pSdb->tableVer[i] = -1; + pSdb->tableVer[i] = 0; pSdb->keyTypes[i] = SDB_KEY_INT32; } @@ -99,7 +99,7 @@ void sdbCleanup(SSdb *pSdb) { taosHashClear(hash); taosHashCleanup(hash); pSdb->hashObjs[i] = NULL; - mDebug("sdb table:%d is cleaned up", i); + mDebug("sdb table:%s is cleaned up", sdbTableName(i)); } taosMemoryFree(pSdb); diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 33383802bee2620016772d9d2d1fc4da9b65b333..f8bd14813c7a4e5707e7c7beb20dabb23d23d6e2 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -18,7 +18,7 @@ #include "tchecksum.h" #include "wal.h" -#define SDB_TABLE_SIZE 24 +#define SDB_TABLE_SIZE 24 #define SDB_RESERVE_SIZE 512 static int32_t sdbRunDeployFp(SSdb *pSdb) { @@ -50,7 +50,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { } for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { - int64_t maxId = -1; + int64_t maxId = 0; ret = taosReadFile(pFile, &maxId, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -66,7 +66,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { } for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { - int64_t ver = -1; + int64_t ver = 0; ret = taosReadFile(pFile, &ver, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -102,7 +102,7 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { } for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { - int64_t maxId = -1; + int64_t maxId = 0; if (i < SDB_MAX) { maxId = pSdb->maxId[i]; } @@ -113,7 +113,7 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { } for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { - int64_t ver = -1; + int64_t ver = 0; if (i < SDB_MAX) { ver = pSdb->tableVer[i]; } @@ -165,6 +165,9 @@ int32_t sdbReadFile(SSdb *pSdb) { return -1; } + int64_t tableVer[SDB_MAX] = {0}; + memcpy(tableVer, pSdb->tableVer, sizeof(tableVer)); + while (1) { readLen = sizeof(SSdbRaw); ret = taosReadFile(pFile, pRaw, readLen); @@ -206,15 +209,16 @@ int32_t sdbReadFile(SSdb *pSdb) { code = sdbWriteWithoutFree(pSdb, pRaw); if (code != 0) { mError("failed to read file:%s since %s", file, terrstr()); - goto PARSE_SDB_DATA_ERROR; + goto _OVER; } } code = 0; pSdb->lastCommitVer = pSdb->curVer; + memcpy(pSdb->tableVer, tableVer, sizeof(tableVer)); mDebug("read file:%s successfully, ver:%" PRId64, file, pSdb->lastCommitVer); -PARSE_SDB_DATA_ERROR: +_OVER: taosCloseFile(&pFile); sdbFreeRaw(pRaw); @@ -259,7 +263,13 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { SSdbRow **ppRow = taosHashIterate(hash, NULL); while (ppRow != NULL) { SSdbRow *pRow = *ppRow; - if (pRow == NULL || pRow->status != SDB_STATUS_READY) { + if (pRow == NULL) { + ppRow = taosHashIterate(hash, ppRow); + continue; + } + + if (pRow->status != SDB_STATUS_READY && pRow->status != SDB_STATUS_DROPPING) { + sdbPrintOper(pSdb, pRow, "not-write"); ppRow = taosHashIterate(hash, ppRow); continue; } diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index b34c2a82d87dfb526d2f2f3f5bc44c404f7a21f1..94008b2f7cc69690d1a4de7707dcb868044659af 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "sdbInt.h" -static void sdbCheck(SSdb *pSdb, SSdbRow *pRow); +static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow); const char *sdbTableName(ESdbType type) { switch (type) { @@ -65,12 +65,10 @@ const char *sdbTableName(ESdbType type) { } } -static const char *sdbStatusStr(ESdbStatus status) { +static const char *sdbStatusName(ESdbStatus status) { switch (status) { case SDB_STATUS_CREATING: return "creating"; - case SDB_STATUS_UPDATING: - return "updating"; case SDB_STATUS_DROPPING: return "dropping"; case SDB_STATUS_READY: @@ -89,13 +87,13 @@ void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper) { if (keyType == SDB_KEY_BINARY) { mTrace("%s:%s, ref:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), (char *)pRow->pObj, pRow->refCount, oper, - pRow->pObj, sdbStatusStr(pRow->status)); + pRow->pObj, sdbStatusName(pRow->status)); } else if (keyType == SDB_KEY_INT32) { mTrace("%s:%d, ref:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), *(int32_t *)pRow->pObj, pRow->refCount, - oper, pRow->pObj, sdbStatusStr(pRow->status)); + oper, pRow->pObj, sdbStatusName(pRow->status)); } else if (keyType == SDB_KEY_INT64) { mTrace("%s:%" PRId64 ", ref:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj, - pRow->refCount, oper, pRow->pObj, sdbStatusStr(pRow->status)); + pRow->refCount, oper, pRow->pObj, sdbStatusName(pRow->status)); } else { } } @@ -116,7 +114,7 @@ static SHashObj *sdbGetHash(SSdb *pSdb, int32_t type) { } static int32_t sdbGetkeySize(SSdb *pSdb, ESdbType type, const void *pKey) { - int32_t keySize; + int32_t keySize = 0; EKeyType keyType = pSdb->keyTypes[type]; if (keyType == SDB_KEY_INT32) { @@ -149,7 +147,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { taosWUnLockLatch(pLock); sdbFreeRow(pSdb, pRow, false); - terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } @@ -183,18 +181,18 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pNewRow, int32_t keySize) { SRWLatch *pLock = &pSdb->locks[pNewRow->type]; - taosRLockLatch(pLock); + taosWLockLatch(pLock); SSdbRow **ppOldRow = taosHashGet(hash, pNewRow->pObj, keySize); if (ppOldRow == NULL || *ppOldRow == NULL) { - taosRUnLockLatch(pLock); + taosWUnLockLatch(pLock); return sdbInsertRow(pSdb, hash, pRaw, pNewRow, keySize); } SSdbRow *pOldRow = *ppOldRow; pOldRow->status = pRaw->status; sdbPrintOper(pSdb, pOldRow, "update"); - taosRUnLockLatch(pLock); + taosWUnLockLatch(pLock); int32_t code = 0; SdbUpdateFp updateFp = pSdb->updateFps[pNewRow->type]; @@ -230,8 +228,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * pSdb->tableVer[pOldRow->type]++; sdbFreeRow(pSdb, pRow, false); - sdbCheck(pSdb, pOldRow); - // sdbRelease(pSdb, pOldRow->pObj); + sdbCheckRow(pSdb, pOldRow); return 0; } @@ -254,7 +251,6 @@ int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw) { case SDB_STATUS_CREATING: code = sdbInsertRow(pSdb, hash, pRaw, pRow, keySize); break; - case SDB_STATUS_UPDATING: case SDB_STATUS_READY: case SDB_STATUS_DROPPING: code = sdbUpdateRow(pSdb, hash, pRaw, pRow, keySize); @@ -295,7 +291,6 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { SSdbRow *pRow = *ppRow; switch (pRow->status) { case SDB_STATUS_READY: - case SDB_STATUS_UPDATING: atomic_add_fetch_32(&pRow->refCount, 1); pRet = pRow->pObj; sdbPrintOper(pSdb, pRow, "acquire"); @@ -315,9 +310,9 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { return pRet; } -static void sdbCheck(SSdb *pSdb, SSdbRow *pRow) { +static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) { SRWLatch *pLock = &pSdb->locks[pRow->type]; - taosRLockLatch(pLock); + taosWLockLatch(pLock); int32_t ref = atomic_load_32(&pRow->refCount); sdbPrintOper(pSdb, pRow, "check"); @@ -325,7 +320,7 @@ static void sdbCheck(SSdb *pSdb, SSdbRow *pRow) { sdbFreeRow(pSdb, pRow, true); } - taosRUnLockLatch(pLock); + taosWUnLockLatch(pLock); } void sdbRelease(SSdb *pSdb, void *pObj) { @@ -335,7 +330,7 @@ void sdbRelease(SSdb *pSdb, void *pObj) { if (pRow->type >= SDB_MAX) return; SRWLatch *pLock = &pSdb->locks[pRow->type]; - taosRLockLatch(pLock); + taosWLockLatch(pLock); int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); sdbPrintOper(pSdb, pRow, "release"); @@ -343,7 +338,7 @@ void sdbRelease(SSdb *pSdb, void *pObj) { sdbFreeRow(pSdb, pRow, true); } - taosRUnLockLatch(pLock); + taosWUnLockLatch(pLock); } void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { @@ -355,16 +350,6 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { SRWLatch *pLock = &pSdb->locks[type]; taosRLockLatch(pLock); -#if 0 - if (pIter != NULL) { - SSdbRow *pLastRow = *(SSdbRow **)pIter; - int32_t ref = atomic_load_32(&pLastRow->refCount); - if (ref <= 0 && pLastRow->status == SDB_STATUS_DROPPED) { - sdbFreeRow(pSdb, pLastRow); - } - } -#endif - SSdbRow **ppRow = taosHashIterate(hash, pIter); while (ppRow != NULL) { SSdbRow *pRow = *ppRow; diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index 45464c3bd35835d5d9c41dd8c78949c46fa7de97..d09198f66f87453d88f67e26766a1183830999be 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -134,6 +134,11 @@ int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status) { return -1; } + if (status == SDB_STATUS_INIT) { + terrno = TSDB_CODE_INVALID_PARA; + return -1; + } + pRaw->status = status; return 0; } diff --git a/source/dnode/qnode/inc/qndInt.h b/source/dnode/qnode/inc/qndInt.h index 307bf3efb8913d4fae4966797835e432ebb051c5..c18a43c4fb20df1b939ee0d9f0ef8813abba58c6 100644 --- a/source/dnode/qnode/inc/qndInt.h +++ b/source/dnode/qnode/inc/qndInt.h @@ -29,7 +29,7 @@ extern "C" { #endif -typedef struct SQWorkerMgmt SQHandle; +typedef struct SQWorker SQHandle; typedef struct SQnode { int32_t qndId; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 724c7b312ea2870f8a03cdb6e113ba82e685f546..58e00ee34a3029550b952d1c2a02eca178f6b7a3 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -8,7 +8,6 @@ target_sources( "src/vnd/vnodeBufPool.c" "src/vnd/vnodeCfg.c" "src/vnd/vnodeCommit.c" - "src/vnd/vnodeInt.c" "src/vnd/vnodeQuery.c" "src/vnd/vnodeStateMgr.c" "src/vnd/vnodeModule.c" @@ -31,9 +30,9 @@ target_sources( "src/tsdb/tsdbFS.c" "src/tsdb/tsdbOpen.c" "src/tsdb/tsdbMemTable.c" + "src/tsdb/tsdbMemTable2.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" - "src/tsdb/tsdbScan.c" "src/tsdb/tsdbSma.c" "src/tsdb/tsdbWrite.c" diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 4213ce78bd36fd7a4cfa6828e496825bfc80b226..62deb200c95e716c07dd0bc146cd44f9221f0123 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -32,7 +32,7 @@ #include "tmsg.h" #include "trow.h" -#include "tdbInt.h" +#include "tdb.h" #ifdef __cplusplus extern "C" { @@ -57,9 +57,6 @@ int vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); -int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg); -int32_t vnodeCompact(SVnode *pVnode); -int32_t vnodeSync(SVnode *pVnode); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int vnodeValidateTableHash(SVnode *pVnode, char *tableFName); @@ -92,16 +89,16 @@ int metaTbCursorNext(SMTbCursor *pTbCur); #endif // tsdb -typedef struct STsdb STsdb; +// typedef struct STsdb STsdb; typedef void *tsdbReaderT; #define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 #define BLOCK_LOAD_TABLE_SEQ_ORDER 2 #define BLOCK_LOAD_TABLE_RR_ORDER 3 -tsdbReaderT *tsdbQueryTables(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, +tsdbReaderT *tsdbQueryTables(SVnode *pVnode, SQueryTableDataCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId); -tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *groupList, uint64_t qId, +tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableGroupInfo *groupList, uint64_t qId, void *pMemRef); int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo); bool isTsdbCacheLastRow(tsdbReaderT *pReader); @@ -111,12 +108,12 @@ int32_t tsdbQuerySTableByTagCond(void *pMeta, uint64_t uid, TSKEY skey, con int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle); bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle); void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); -int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SColumnDataAgg **pBlockStatis); +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SColumnDataAgg ***pBlockStatis, bool* allHave); SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList); void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond *pCond); void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); -int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo); +int32_t tsdbGetTableGroupFromIdList(SVnode *pVnode, SArray *pTableIdList, STableGroupInfo *pGroupInfo); // tq @@ -146,9 +143,19 @@ struct STsdbCfg { int32_t keep0; int32_t keep1; int32_t keep2; - SArray *retentions; + // TODO: save to tsdb cfg file + int8_t type; // ETsdbType + SRetention retentions[TSDB_RETENTION_MAX]; }; +typedef enum { + TSDB_TYPE_TSDB = 0, // TSDB + TSDB_TYPE_TSMA = 1, // TSMA + TSDB_TYPE_RSMA_L0 = 2, // RSMA Level 0 + TSDB_TYPE_RSMA_L1 = 3, // RSMA Level 1 + TSDB_TYPE_RSMA_L2 = 4, // RSMA Level 2 +} ETsdbType; + struct SVnodeCfg { int32_t vgId; char dbname[TSDB_DB_NAME_LEN]; @@ -157,7 +164,6 @@ struct SVnodeCfg { int32_t szCache; uint64_t szBuf; bool isHeap; - int8_t streamMode; bool isWeak; STsdbCfg tsdbCfg; SWalCfg walCfg; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 0001a4323139447ae8ed836eb7e9495c48df9d1c..5970fdbe92ca8ad0ae8d3ef7d102f22d7902fd6b 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -43,7 +43,6 @@ int metaEncodeEntry(SCoder* pCoder, const SMetaEntry* pME); int metaDecodeEntry(SCoder* pCoder, SMetaEntry* pME); // metaTable ================== -int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); // metaQuery ================== int metaGetTableEntryByVersion(SMetaReader* pReader, int64_t version, tb_uid_t uid); @@ -104,8 +103,6 @@ typedef struct { #if 1 -// int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); -int metaDropTable(SMeta* pMeta, tb_uid_t uid); SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid); void metaCloseSmaCursor(SMSmaCursor* pSmaCur); int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4fc94da06a223b9c090a78b144aec467a25bf4a0..2716692bd71d57fb4ebdc24c79dc9c968abee3ac 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -40,7 +40,6 @@ typedef struct STable STable; int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable); void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable); -int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows); int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); @@ -72,8 +71,8 @@ struct STsdb { char *path; SVnode *pVnode; bool repoLocked; + int8_t level; // retention level TdThreadMutex mutex; - STsdbCfg config; STsdbMemTable *mem; STsdbMemTable *imem; SRtn rtn; @@ -185,7 +184,8 @@ struct STsdbFS { }; #define REPO_ID(r) TD_VID((r)->pVnode) -#define REPO_CFG(r) (&(r)->config) +#define REPO_CFG(r) (&(r)->pVnode->config.tsdbCfg) +#define REPO_LEVEL(r) ((r)->level) #define REPO_FS(r) ((r)->fs) #define REPO_META(r) ((r)->pVnode->pMeta) #define REPO_TFS(r) ((r)->pVnode->pTfs) @@ -534,23 +534,6 @@ static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) { } } -// tsdbDBDef -// typedef struct SDBFile SDBFile; -// typedef DB_ENV* TDBEnv; - -// struct SDBFile { -// int32_t fid; -// DB* pDB; -// char* path; -// }; - -// int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF); -// void tsdbCloseDBF(SDBFile* pDBF); -// int32_t tsdbOpenBDBEnv(DB_ENV** ppEnv, const char* path); -// void tsdbCloseBDBEnv(DB_ENV* pEnv); -// int32_t tsdbSaveSmaToDB(SDBFile* pDBF, void* key, uint32_t keySize, void* data, uint32_t dataSize); -// void* tsdbGetSmaDataByKey(SDBFile* pDBF, void* key, uint32_t keySize, uint32_t* valueSize); - // tsdbFile #define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_DELIMITER 0xF00AFA0F diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index 162d733cc3654bb2d743334ff6758508c7c98683..5215812ac577223b45450a4183c50a7911e4e917 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -40,7 +40,6 @@ static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) { return TSDB_CODE_SUCCESS; } - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index b5066799ca172711c3511e93cbe59520fc5093db..3724a98a5ea908619edbcd33c5ec978a7eccd7ea 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -25,7 +25,7 @@ #include "tcompare.h" #include "tcompression.h" #include "tdatablock.h" -#include "tdbInt.h" +#include "tdb.h" #include "tencode.h" #include "tfs.h" #include "tglobal.h" @@ -46,18 +46,21 @@ extern "C" { #endif -typedef struct SVnodeInfo SVnodeInfo; -typedef struct SMeta SMeta; -typedef struct STsdb STsdb; -typedef struct STQ STQ; -typedef struct SVState SVState; -typedef struct SVBufPool SVBufPool; -typedef struct SQWorkerMgmt SQHandle; - -#define VNODE_META_DIR "meta" -#define VNODE_TSDB_DIR "tsdb" -#define VNODE_TQ_DIR "tq" -#define VNODE_WAL_DIR "wal" +typedef struct SVnodeInfo SVnodeInfo; +typedef struct SMeta SMeta; +typedef struct STsdb STsdb; +typedef struct STQ STQ; +typedef struct SVState SVState; +typedef struct SVBufPool SVBufPool; +typedef struct SQWorker SQHandle; + +#define VNODE_META_DIR "meta" +#define VNODE_TSDB_DIR "tsdb" +#define VNODE_TQ_DIR "tq" +#define VNODE_WAL_DIR "wal" +#define VNODE_TSMA_DIR "tsma" +#define VNODE_RSMA1_DIR "rsma1" +#define VNODE_RSMA2_DIR "rsma2" // vnd.h void* vnodeBufPoolMalloc(SVBufPool* pPool, int size); @@ -72,7 +75,9 @@ int metaClose(SMeta* pMeta); int metaBegin(SMeta* pMeta); int metaCommit(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); +int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); +int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); int metaGetTableEntryByName(SMetaReader* pReader, const char* name); @@ -87,14 +92,20 @@ int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); // tsdb -int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb); -int tsdbClose(STsdb* pTsdb); -int tsdbBegin(STsdb* pTsdb); -int tsdbCommit(STsdb* pTsdb); -int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version); -int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); -int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); -int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); +int tsdbOpen(SVnode* pVnode, int8_t type); +int tsdbClose(STsdb* pTsdb); +int tsdbBegin(STsdb* pTsdb); +int tsdbCommit(STsdb* pTsdb); +int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version); +int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); +int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); +int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); +int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, int32_t* pAffectedRows); +tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, + uint64_t taskId); +tsdbReaderT tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, + void* pMemRef); +int32_t tsdbGetTableGroupFromIdListT(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo); // tq STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal); @@ -114,7 +125,7 @@ int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, t int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore); void tsdbUidStoreDestory(STbUidStore* pStore); void* tsdbUidStoreFree(STbUidStore* pStore); -int32_t tsdbTriggerRSma(STsdb* pTsdb, SMeta* pMeta, void* pMsg, int32_t inputType); +int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType); typedef struct { int8_t streamType; // sma or other @@ -155,6 +166,8 @@ struct SVnode { SVBufPool* onRecycle; SMeta* pMeta; STsdb* pTsdb; + STsdb* pRSma1; + STsdb* pRSma2; SWal* pWal; STQ* pTq; SSink* pSink; @@ -163,15 +176,24 @@ struct SVnode { SQHandle* pQuery; }; +#define VND_TSDB(vnd) ((vnd)->pTsdb) +#define VND_RSMA0(vnd) ((vnd)->pTsdb) +#define VND_RSMA1(vnd) ((vnd)->pRSma1) +#define VND_RSMA2(vnd) ((vnd)->pRSma2) + struct STbUidStore { tb_uid_t suid; + tb_uid_t uid; // TODO: just for debugging, remove when uid provided in SSDataBlock SArray* tbUids; SHashObj* uidHash; }; #define TD_VID(PVNODE) (PVNODE)->config.vgId -// typedef struct STbDdlH STbDdlH; +static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) { + SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]); + return (pRetention->freq > 0 && pRetention->keep > 0); +} // sma void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index e65f5229af604c98daff563624b66abc94c14930..85a49ffabd81499bd39d86b836fb0850d62b2abe 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -47,66 +47,66 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { // open env ret = tdbEnvOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv); if (ret < 0) { - metaError("vgId: %d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTbDb ret = tdbDbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb); if (ret < 0) { - metaError("vgId: %d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pSkmDb ret = tdbDbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb); if (ret < 0) { - metaError("vgId: %d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pUidIdx ret = tdbDbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx); if (ret < 0) { - metaError("vgId: %d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pNameIdx ret = tdbDbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx); if (ret < 0) { - metaError("vgId: %d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pCtbIdx ret = tdbDbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); if (ret < 0) { - metaError("vgId: %d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTagIdx ret = tdbDbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); if (ret < 0) { - metaError("vgId: %d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTtlIdx ret = tdbDbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); if (ret < 0) { - metaError("vgId: %d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open index if (metaOpenIdx(pMeta) < 0) { - metaError("vgId: %d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } - metaDebug("vgId: %d meta is opened", TD_VID(pVnode)); + metaDebug("vgId:%d meta is opened", TD_VID(pVnode)); *ppMeta = pMeta; return 0; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 46a39e72f91b392f1a84fd13bedcccc799b05eda..60c844e394a6990d41d7eadbc5e6e15fa7c62f60 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -23,7 +23,7 @@ void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { void metaReaderClear(SMetaReader *pReader) { tCoderClear(&pReader->coder); - TDB_FREE(pReader->pBuf); + tdbFree(pReader->pBuf); } int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t uid) { @@ -96,15 +96,17 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { metaReaderInit(&pTbCur->mr, pMeta, 0); - tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc); + tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL); + + tdbDbcMoveToFirst(pTbCur->pDbc); return pTbCur; } void metaCloseTbCursor(SMTbCursor *pTbCur) { if (pTbCur) { - TDB_FREE(pTbCur->pKey); - TDB_FREE(pTbCur->pVal); + tdbFree(pTbCur->pKey); + tdbFree(pTbCur->pVal); metaReaderClear(&pTbCur->mr); if (pTbCur->pDbc) { tdbDbcClose(pTbCur->pDbc); @@ -119,7 +121,7 @@ int metaTbCursorNext(SMTbCursor *pTbCur) { STbCfg tbCfg; for (;;) { - ret = tdbDbNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); + ret = tdbDbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); if (ret < 0) { return -1; } @@ -159,7 +161,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo // decode pBuf = pVal; - pSW = taosMemoryMalloc(sizeof(pSW)); + pSW = taosMemoryMalloc(sizeof(SSchemaWrapper)); tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_DECODER); tDecodeSSchemaWrapper(&coder, pSW); @@ -169,7 +171,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo pSW->pSchema = pSchema; - TDB_FREE(pVal); + tdbFree(pVal); return pSW; } @@ -185,7 +187,9 @@ struct SMCtbCursor { SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { SMCtbCursor *pCtbCur = NULL; + SCtbIdxKey ctbIdxKey; int ret; + int c; pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur)); if (pCtbCur == NULL) { @@ -193,12 +197,20 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { } pCtbCur->suid = uid; - ret = tdbDbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur); + ret = tdbDbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL); if (ret < 0) { taosMemoryFree(pCtbCur); return NULL; } + // move to the suid + ctbIdxKey.suid = uid; + ctbIdxKey.uid = INT64_MIN; + tdbDbcMoveTo(pCtbCur->pCur, &ctbIdxKey, sizeof(ctbIdxKey), &c); + if (c > 0) { + tdbDbcMoveToNext(pCtbCur->pCur); + } + return pCtbCur; } @@ -207,8 +219,8 @@ void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { if (pCtbCur->pCur) { tdbDbcClose(pCtbCur->pCur); - TDB_FREE(pCtbCur->pKey); - TDB_FREE(pCtbCur->pVal); + tdbFree(pCtbCur->pKey); + tdbFree(pCtbCur->pVal); } taosMemoryFree(pCtbCur); @@ -219,12 +231,15 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { int ret; SCtbIdxKey *pCtbIdxKey; - ret = tdbDbNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); + ret = tdbDbcNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); if (ret < 0) { return 0; } pCtbIdxKey = pCtbCur->pKey; + if (pCtbIdxKey->suid > pCtbCur->suid) { + return 0; + } return pCtbIdxKey->uid; } @@ -283,7 +298,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { while (true) { // TODO: lock during iterate? - if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { + if (tdbDbcNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { pSmaIdxKey = pCur->pKey; ASSERT(pSmaIdxKey != NULL); @@ -297,7 +312,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { ++pSW->number; STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); if (tptr == NULL) { - TDB_FREE(pSmaVal); + tdbFree(pSmaVal); metaCloseSmaCursor(pCur); tdDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); @@ -306,13 +321,13 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { pSW->tSma = tptr; pBuf = pSmaVal; if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { - TDB_FREE(pSmaVal); + tdbFree(pSmaVal); metaCloseSmaCursor(pCur); tdDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); return NULL; } - TDB_FREE(pSmaVal); + tdbFree(pSmaVal); continue; } break; @@ -354,7 +369,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { tb_uid_t uid = 0; while (true) { // TODO: lock during iterate? - if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { + if (tdbDbcNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { ASSERT(pSmaIdxKey != NULL); pSmaIdxKey = pCur->pKey; @@ -425,15 +440,15 @@ void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) { if (tDecodeTSma(pBuf, pCfg) == NULL) { tdDestroyTSma(pCfg); taosMemoryFree(pCfg); - TDB_FREE(pVal); + tdbFree(pVal); return NULL; } - TDB_FREE(pVal); + tdbFree(pVal); return pCfg; #endif #endif return NULL; } -#endif \ No newline at end of file +#endif diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index fd5f679d0b4efbfcb6800dc05af0a61d9499f0ec..cb556e8630a3d01d6d98d8c990b48f5a43adf5e7 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -407,7 +407,7 @@ static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_ pSchemaWrapper = taosMemoryMalloc(sizeof(*pSchemaWrapper)); metaDecodeSchemaEx(pBuf, pSchemaWrapper, isGetEx); - TDB_FREE(pVal); + tdbFree(pVal); return pSchemaWrapper; } @@ -438,7 +438,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { while (true) { // TODO: lock during iterate? - if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { + if (tdbDbcNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { pSmaIdxKey = pCur->pKey; ASSERT(pSmaIdxKey != NULL); @@ -450,7 +450,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { } if ((pSW == NULL) && ((pSW = taosMemoryCalloc(1, sizeof(*pSW))) == NULL)) { - TDB_FREE(pSmaVal); + tdbFree(pSmaVal); metaCloseSmaCursor(pCur); return NULL; } @@ -458,7 +458,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { ++pSW->number; STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); if (tptr == NULL) { - TDB_FREE(pSmaVal); + tdbFree(pSmaVal); metaCloseSmaCursor(pCur); tdDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); @@ -467,13 +467,13 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { pSW->tSma = tptr; pBuf = pSmaVal; if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { - TDB_FREE(pSmaVal); + tdbFree(pSmaVal); metaCloseSmaCursor(pCur); tdDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); return NULL; } - TDB_FREE(pSmaVal); + tdbFree(pSmaVal); continue; } break; @@ -613,7 +613,7 @@ int64_t metaSmaCursorNext(SMSmaCursor *pCur) { void *pBuf; SSmaIdxKey *smaIdxKey; - ret = tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, &pCur->pVal, &pCur->vLen); + ret = tdbDbcNext(pCur->pCur, &pCur->pKey, &pCur->kLen, &pCur->pVal, &pCur->vLen); if (ret < 0) { return 0; } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 3e70bbb4797c459e2af51207092e5fdb3dc9b73e..397e0740616f55be8dced00570494a90bc4ef6ac 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -61,19 +61,75 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { if (metaHandleEntry(pMeta, &me) < 0) goto _err; - metaDebug("vgId: %d super table is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + metaDebug("vgId:%d super table is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); return 0; _err: - metaError("vgId: %d failed to create super table: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + metaError("vgId:%d failed to create super table: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, tstrerror(terrno)); return -1; } int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { - // TODO + TDBC *pNameIdxc = NULL; + TDBC *pUidIdxc = NULL; + TDBC *pCtbIdxc = NULL; + SCtbIdxKey *pCtbIdxKey; + const void *pKey = NULL; + int nKey; + const void *pData = NULL; + int nData; + int c, ret; + + // prepare uid idx cursor + tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); + if (ret < 0 || c != 0) { + terrno = TSDB_CODE_VND_TB_NOT_EXIST; + tdbDbcClose(pUidIdxc); + goto _err; + } + + // prepare name idx cursor + tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); + if (ret < 0 || c != 0) { + ASSERT(0); + } + + tdbDbcDelete(pUidIdxc); + tdbDbcDelete(pNameIdxc); + tdbDbcClose(pUidIdxc); + tdbDbcClose(pNameIdxc); + + // loop to drop each child table + tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (ret < 0 || (c < 0 && tdbDbcMoveToNext(pCtbIdxc) < 0)) { + tdbDbcClose(pCtbIdxc); + goto _exit; + } + + for (;;) { + tdbDbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL); + pCtbIdxKey = (SCtbIdxKey *)pKey; + + if (pCtbIdxKey->suid > pReq->suid) break; + + // drop the child table (TODO) + + if (tdbDbcMoveToNext(pCtbIdxc) < 0) break; + } + +_exit: + metaDebug("vgId:%d super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); return 0; + +_err: + metaError("vgId:%d failed to drop super table %s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, tstrerror(terrno)); + return -1; } int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { @@ -127,18 +183,122 @@ _err: return -1; } -int metaDropTable(SMeta *pMeta, tb_uid_t uid) { -#if 0 - if (metaRemoveTableFromIdx(pMeta, uid) < 0) { - // TODO: handle error +int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { + TDBC *pTbDbc = NULL; + TDBC *pUidIdxc = NULL; + TDBC *pNameIdxc = NULL; + const void *pData; + int nData; + tb_uid_t uid; + int64_t tver; + SMetaEntry me = {0}; + SCoder coder = {0}; + int8_t type; + int64_t ctime; + tb_uid_t suid; + int c, ret; + + // search & delete the name idx + tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); + if (ret < 0 || c) { + tdbDbcClose(pNameIdxc); + terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; } - if (metaRemoveTableFromIdx(pMeta, uid) < 0) { - // TODO + ret = tdbDbcGet(pNameIdxc, NULL, NULL, &pData, &nData); + if (ret < 0) { + ASSERT(0); return -1; } -#endif + + uid = *(tb_uid_t *)pData; + + tdbDbcDelete(pNameIdxc); + tdbDbcClose(pNameIdxc); + + // search & delete uid idx + tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + if (ret < 0 || c != 0) { + ASSERT(0); + return -1; + } + + ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + if (ret < 0) { + ASSERT(0); + return -1; + } + + tver = *(int64_t *)pData; + tdbDbcDelete(pUidIdxc); + tdbDbcClose(pUidIdxc); + + // search and get meta entry + tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + ret = tdbDbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c); + if (ret < 0 || c != 0) { + ASSERT(0); + return -1; + } + + ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // decode entry + void *pDataCopy = taosMemoryMalloc(nData); // remove the copy (todo) + memcpy(pDataCopy, pData, nData); + tCoderInit(&coder, TD_LITTLE_ENDIAN, pDataCopy, nData, TD_DECODER); + ret = metaDecodeEntry(&coder, &me); + if (ret < 0) { + ASSERT(0); + return -1; + } + + type = me.type; + if (type == TSDB_CHILD_TABLE) { + ctime = me.ctbEntry.ctime; + suid = me.ctbEntry.suid; + } else if (type == TSDB_NORMAL_TABLE) { + ctime = me.ntbEntry.ctime; + suid = 0; + } else { + ASSERT(0); + } + + taosMemoryFree(pDataCopy); + tCoderClear(&coder); + tdbDbcClose(pTbDbc); + + if (type == TSDB_CHILD_TABLE) { + // remove the pCtbIdx + TDBC *pCtbIdxc = NULL; + tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + + ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c); + if (ret < 0 || c != 0) { + ASSERT(0); + return -1; + } + + tdbDbcDelete(pCtbIdxc); + tdbDbcClose(pCtbIdxc); + + // remove tags from pTagIdx (todo) + } else if (type == TSDB_NORMAL_TABLE) { + // remove from pSkmDb + } else { + ASSERT(0); + } + + // remove from ttl (todo) + if (ctime > 0) { + } return 0; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7cee82f660d92ce0b64e3c6e491ae13da7030c78..261b4aa8ae79e07a577598c5e0f61eae9b7b6a71 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -58,6 +58,48 @@ void tqClose(STQ* pTq) { // TODO } +static void tdSRowDemo() { +#define DEMO_N_COLS 3 + + int16_t schemaVersion = 0; + int32_t numOfCols = DEMO_N_COLS; // ts + int + SRowBuilder rb = {0}; + + SSchema schema[DEMO_N_COLS] = { + {.type = TSDB_DATA_TYPE_TIMESTAMP, .colId = 1, .name = "ts", .bytes = 8, .flags = SCHEMA_SMA_ON}, + {.type = TSDB_DATA_TYPE_INT, .colId = 2, .name = "c1", .bytes = 4, .flags = SCHEMA_SMA_ON}, + {.type = TSDB_DATA_TYPE_INT, .colId = 3, .name = "c2", .bytes = 4, .flags = SCHEMA_SMA_ON}}; + + SSchema* pSchema = schema; + STSchema* pTSChema = tdGetSTSChemaFromSSChema(&pSchema, numOfCols); + + tdSRowInit(&rb, schemaVersion); + tdSRowSetTpInfo(&rb, numOfCols, pTSChema->flen); + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSChema); + void* row = taosMemoryCalloc(1, maxLen); // make sure the buffer is enough + + // set row buf + tdSRowResetBuf(&rb, row); + + for (int32_t idx = 0; idx < pTSChema->numOfCols; ++idx) { + STColumn* pColumn = pTSChema->columns + idx; + if (idx == 0) { + int64_t tsKey = 1651234567; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, &tsKey, true, pColumn->offset, idx); + } else if (idx == 1) { + int32_t val1 = 10; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, &val1, true, pColumn->offset, idx); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, true, pColumn->offset, idx); + } + } + + // print + tdSRowPrint(row, pTSChema, __func__); + + taosMemoryFree(pTSChema); +} + int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { if (msgType != TDMT_VND_SUBMIT) return 0; void* pIter = NULL; @@ -864,7 +906,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { } int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { - SStreamTask* pTask = taosMemoryMalloc(sizeof(SStreamTask)); + SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { return -1; } @@ -884,6 +926,12 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { pTask->ahandle = pTq->pVnode; if (pTask->sinkType == TASK_SINK__SMA) { pTask->smaSink.smaHandle = smaHandleRes; + } else if (pTask->sinkType == TASK_SINK__TABLE) { + ASSERT(pTask->tbSink.pSchemaWrapper); + ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); + pTask->tbSink.pTSchema = + tdGetSTSChemaFromSSChema(&pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols); + ASSERT(pTask->tbSink.pTSchema); } taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), pTask, sizeof(SStreamTask)); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index cf1154071dbc14ac48ebdae04427de083b9e1857..511d57ed58fcaf28dafeb30025998ca848976c41 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -37,9 +37,9 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t // pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); // iterate and convert - if (tInitSubmitMsgIterEx(pMsg, &pReadHandle->msgIter) < 0) return -1; + if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1; while (true) { - if (tGetSubmitMsgNextEx(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1; + if (tGetSubmitMsgNext(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1; if (pReadHandle->pBlock == NULL) break; // pReadHandle->pBlock->uid = htobe64(pReadHandle->pBlock->uid); @@ -50,7 +50,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t // pReadHandle->pBlock->numOfRows = htons(pReadHandle->pBlock->numOfRows); } - if (tInitSubmitMsgIterEx(pMsg, &pReadHandle->msgIter) < 0) return -1; + if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1; pReadHandle->ver = ver; memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter)); return 0; @@ -58,7 +58,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t bool tqNextDataBlock(STqReadHandle* pHandle) { while (1) { - if (tGetSubmitMsgNextEx(&pHandle->msgIter, &pHandle->pBlock) < 0) { + if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { return false; } if (pHandle->pBlock == NULL) return false; @@ -91,16 +91,8 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p int32_t sversion = 0; if (pHandle->sver != sversion) { pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); -#if 0 - tb_uid_t quid; - STbCfg* pTbCfg = metaGetTbInfoByUid(pHandle->pVnodeMeta, pHandle->msgIter.uid); - if (pTbCfg->type == META_CHILD_TABLE) { - quid = pTbCfg->ctbCfg.suid; - } else { - quid = pHandle->msgIter.uid; - } - pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, quid, sversion, true); -#endif + + // this interface use suid instead of uid pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.suid, sversion, true); pHandle->sver = sversion; } @@ -177,8 +169,8 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p tdSTSRowIterInit(&iter, pTschema); STSRow* row; int32_t curRow = 0; - tInitSubmitBlkIterEx(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); - while ((row = tGetSubmitBlkNextEx(&pHandle->blkIter)) != NULL) { + tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); + while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); // get all wanted col of that block for (int32_t i = 0; i < colActual; i++) { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 8bcb848516890e917e0c1b91293a2017df853217..97b796c6a50b9712d35a7b77a328d78c089e4792 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -55,7 +55,7 @@ typedef struct { #define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh)) #define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh)) #define TSDB_COMMIT_EXBUF(ch) TSDB_READ_EXBUF(&((ch)->readh)) -#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRows) +#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->pVnode->config.tsdbCfg.maxRows) #define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) static void tsdbStartCommit(STsdb *pRepo); @@ -69,10 +69,6 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith); static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); static void tsdbResetCommitFile(SCommitH *pCommith); static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); -// static int tsdbCommitMeta(STsdbRepo *pRepo); -// static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact); -// static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); -// static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile); static int tsdbCommitToTable(SCommitH *pCommith, int tid); static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable); static int tsdbComparKeyBlock(const void *arg1, const void *arg2); @@ -922,7 +918,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF STColumn *pColumn = pSchema->columns + ncol; SDataCol *pDataCol = pDataCols->cols + ncol; SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; - SAggrBlkCol *pAggrBlkCol = (SAggrBlkCol *)pAggrBlkData + nColsNotAllNull; + SAggrBlkCol *pAggrBlkCol = (SAggrBlkCol *)pAggrBlkData + nColsOfBlockSma; if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it continue; @@ -935,7 +931,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF pBlockCol->type = pDataCol->type; pAggrBlkCol->colId = pDataCol->colId; - if (isSuper && pColumn->sma && tDataTypes[pDataCol->type].statisFunc) { + if (isSuper && IS_BSMA_ON(pColumn) && tDataTypes[pDataCol->type].statisFunc) { #if 0 (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), @@ -951,6 +947,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF } else { TD_SET_COL_ROWS_MISC(pBlockCol); } + ++nColsOfBlockSma; } else if (tdIsBitmapBlkNorm(pDataCol->pBitmap, rowsToWrite, pDataCols->bitmapMode)) { // check if all rows normal TD_SET_COL_ROWS_NORM(pBlockCol); @@ -959,10 +956,6 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF } ++nColsNotAllNull; - - if (isSuper && pColumn->sma) { - ++nColsOfBlockSma; - } } ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 4ff3227e08890a9c8a50b558a95b948c3338f7ea..6eda476b651ce69d062e36f329ebf5108da97447 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -15,6 +15,8 @@ #include "tsdb.h" +extern const char *TSDB_LEVEL_DNAME[]; + typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; static const char *tsdbTxnFname[] = {"current.t", "current"}; #define TSDB_MAX_FSETS(keep, days) ((keep) / (days) + 3) @@ -35,12 +37,12 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired); // static int tsdbProcessExpiredFS(STsdb *pRepo); // static int tsdbCreateMeta(STsdb *pRepo); -static void tsdbGetRootDir(int repoid, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb", repoid); +static void tsdbGetRootDir(int repoid, int8_t level, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, TSDB_LEVEL_DNAME[level]); } -static void tsdbGetDataDir(int repoid, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data", repoid); +static void tsdbGetDataDir(int repoid, int8_t level, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, TSDB_LEVEL_DNAME[level]); } // For backward compatibility @@ -588,8 +590,8 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) { } static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { - snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo), - tsdbTxnFname[ftype]); + snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/%s/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo), + TSDB_LEVEL_DNAME[REPO_LEVEL(pRepo)], tsdbTxnFname[ftype]); } static int tsdbOpenFSFromCurrent(STsdb *pRepo) { @@ -719,7 +721,7 @@ static int tsdbScanRootDir(STsdb *pRepo) { STsdbFS *pfs = REPO_FS(pRepo); const STfsFile *pf; - tsdbGetRootDir(REPO_ID(pRepo), rootDir); + tsdbGetRootDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), rootDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), rootDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); @@ -753,7 +755,7 @@ static int tsdbScanDataDir(STsdb *pRepo) { STsdbFS *pfs = REPO_FS(pRepo); const STfsFile *pf; - tsdbGetDataDir(REPO_ID(pRepo), dataDir); + tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), dataDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); @@ -801,7 +803,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { regex_t regex; STsdbFS *pfs = REPO_FS(pRepo); - tsdbGetDataDir(REPO_ID(pRepo), dataDir); + tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir); // Resource allocation and init regcomp(®ex, pattern, REG_EXTENDED); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index b1af3e046161300b8d104dd2a4968ee418d7ea56..7f024786de55a01f65580a7df2bd76fbc99e1017 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -27,7 +27,13 @@ static const char *TSDB_FNAME_SUFFIX[] = { "rsma", // TSDB_FILE_RSMA }; -static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); +const char *TSDB_LEVEL_DNAME[] = { + "tsdb", + "rsma1", + "rsma2", +}; + +static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char* dname, char *fname); // static int tsdbRollBackMFile(SMFile *pMFile); static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo); static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo); @@ -45,7 +51,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t pDFile->info.magic = TSDB_FILE_INIT_MAGIC; pDFile->info.fver = tsdbGetDFSVersion(ftype); - tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, fname); + tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, TSDB_LEVEL_DNAME[pRepo->level], fname); tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname); } @@ -431,14 +437,15 @@ int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype return 0; } -static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname) { +static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char *dname, char *fname) { ASSERT(ftype != TSDB_FILE_MAX); if (ftype < TSDB_FILE_MAX) { if (ver == 0) { - snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s", vid, vid, fid, TSDB_FNAME_SUFFIX[ftype]); + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data/v%df%d.%s", vid, dname, vid, fid, + TSDB_FNAME_SUFFIX[ftype]); } else { - snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s-ver%" PRIu32, vid, vid, fid, + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data/v%df%d.%s-ver%" PRIu32, vid, dname, vid, fid, TSDB_FNAME_SUFFIX[ftype], ver); } } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 48e672d9bc61f6b7b8c09edaf198de1a854c073e..b52a34cfb2f9bf9044700ae4c6d3c102e44e5de0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -190,38 +190,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey return 0; } -int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg) { - ASSERT(pMsg != NULL); - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - STSRow *row = NULL; - - terrno = TSDB_CODE_SUCCESS; - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; - if (pBlock == NULL) break; - - pBlock->uid = htobe64(pBlock->uid); - pBlock->suid = htobe64(pBlock->suid); - pBlock->sversion = htonl(pBlock->sversion); - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->schemaLen = htonl(pBlock->schemaLen); - pBlock->numOfRows = htons(pBlock->numOfRows); - } - - if (terrno != TSDB_CODE_SUCCESS) return -1; - return 0; -} - -int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows) { - // STsdbMeta *pMeta = pRepo->tsdbMeta; - // int32_t points = 0; - // STable *pTable = NULL; +int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, int32_t *pAffectedRows) { SSubmitBlkIter blkIter = {0}; STsdbMemTable *pMemTable = pTsdb->mem; void *tptr; @@ -232,15 +201,15 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows SSubmitBlk *pBlkCopy; // create container is nedd - tptr = taosHashGet(pMemTable->pHashIdx, &(pBlock->uid), sizeof(pBlock->uid)); + tptr = taosHashGet(pMemTable->pHashIdx, &(pMsgIter->uid), sizeof(pMsgIter->uid)); if (tptr == NULL) { - pTbData = tsdbNewTbData(pBlock->uid); + pTbData = tsdbNewTbData(pMsgIter->uid); if (pTbData == NULL) { return -1; } // Put into hash - taosHashPut(pMemTable->pHashIdx, &(pBlock->uid), sizeof(pBlock->uid), &(pTbData), sizeof(pTbData)); + taosHashPut(pMemTable->pHashIdx, &(pMsgIter->uid), sizeof(pMsgIter->uid), &(pTbData), sizeof(pTbData)); // Put into skiplist tSkipListPut(pMemTable->pSlIdx, pTbData); @@ -249,10 +218,11 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows } // copy data to buffer pool - pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, pBlock->dataLen + sizeof(*pBlock)); - memcpy(pBlkCopy, pBlock, pBlock->dataLen + sizeof(*pBlock)); + int32_t tlen = pMsgIter->dataLen + pMsgIter->schemaLen + sizeof(*pBlock); + pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, tlen); + memcpy(pBlkCopy, pBlock, tlen); - tInitSubmitBlkIter(pBlkCopy, &blkIter); + tInitSubmitBlkIter(pMsgIter, pBlkCopy, &blkIter); if (blkIter.row == NULL) return 0; keyMin = TD_ROW_KEY(blkIter.row); @@ -261,15 +231,15 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows // Set statistics keyMax = TD_ROW_KEY(blkIter.row); - pTbData->nrows += pBlock->numOfRows; + pTbData->nrows += pMsgIter->numOfRows; if (pTbData->keyMin > keyMin) pTbData->keyMin = keyMin; if (pTbData->keyMax < keyMax) pTbData->keyMax = keyMax; - pMemTable->nRow += pBlock->numOfRows; + pMemTable->nRow += pMsgIter->numOfRows; if (pMemTable->keyMin > keyMin) pMemTable->keyMin = keyMin; if (pMemTable->keyMax < keyMax) pMemTable->keyMax = keyMax; - (*pAffectedRows) += pBlock->numOfRows; + (*pAffectedRows) = pMsgIter->numOfRows; return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c new file mode 100644 index 0000000000000000000000000000000000000000..ae0d4630d13b0103c602815973a5675b6c3f557a --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +typedef struct SMemTable SMemTable; +typedef struct SMemData SMemData; +typedef struct SMemSkipList SMemSkipList; +typedef struct SMemSkipListCfg SMemSkipListCfg; + +struct SMemTable { + STsdb *pTsdb; + TSKEY minKey; + TSKEY maxKey; + int64_t minVer; + int64_t maxVer; + int64_t nRows; + int32_t nHash; + int32_t nBucket; + SMemData **pBuckets; +}; + +struct SMemSkipListCfg { + int8_t maxLevel; + int32_t nKey; + int32_t nData; +}; + +struct SMemSkipList { + int8_t level; + uint32_t seed; +}; + +struct SMemData { + SMemData *pHashNext; + tb_uid_t suid; + tb_uid_t uid; + TSKEY minKey; + TSKEY maxKey; + int64_t minVer; + int64_t maxVer; + int64_t nRows; + SMemSkipList sl; +}; + +// SMemTable +int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { + SMemTable *pMemTb = NULL; + + pMemTb = taosMemoryCalloc(1, sizeof(*pMemTb)); + if (pMemTb == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMemTb->pTsdb = pTsdb; + pMemTb->minKey = TSKEY_MAX; + pMemTb->maxKey = TSKEY_MIN; + pMemTb->minVer = -1; + pMemTb->maxVer = -1; + pMemTb->nRows = 0; + pMemTb->nHash = 0; + pMemTb->nBucket = 1024; + pMemTb->pBuckets = taosMemoryCalloc(pMemTb->nBucket, sizeof(*pMemTb->pBuckets)); + if (pMemTb->pBuckets == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + *ppMemTb = pMemTb; + return 0; +} + +int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMT) { + // TODO + return 0; +} + +// SMemData + +// SMemSkipList \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index e5b25184154401897c6d16fe4eee9540e8d96483..e18c01dc016fc1d63d14ca49440f9cb1528d62fa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -15,12 +15,44 @@ #include "tsdb.h" -int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb) { +static int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level); + +int tsdbOpen(SVnode *pVnode, int8_t type) { + switch (type) { + case TSDB_TYPE_TSDB: + return tsdbOpenImpl(pVnode, type, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0); + case TSDB_TYPE_TSMA: + ASSERT(0); + break; + case TSDB_TYPE_RSMA_L0: + return tsdbOpenImpl(pVnode, type, &VND_RSMA0(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0); + case TSDB_TYPE_RSMA_L1: + return tsdbOpenImpl(pVnode, type, &VND_RSMA1(pVnode), VNODE_RSMA1_DIR, TSDB_RETENTION_L1); + case TSDB_TYPE_RSMA_L2: + return tsdbOpenImpl(pVnode, type, &VND_RSMA2(pVnode), VNODE_RSMA2_DIR, TSDB_RETENTION_L2); + default: + ASSERT(0); + break; + } + return 0; +} + +/** + * @brief + * + * @param pVnode + * @param type + * @param ppTsdb + * @param dir + * @param level retention level + * @return int + */ +int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level) { STsdb *pTsdb = NULL; int slen = 0; *ppTsdb = NULL; - slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(VNODE_TSDB_DIR) + 3; + slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + 3; // create handle pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); @@ -31,12 +63,12 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb) { pTsdb->path = (char *)&pTsdb[1]; sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, - VNODE_TSDB_DIR); + dir); pTsdb->pVnode = pVnode; + pTsdb->level = level; pTsdb->repoLocked = false; - tdbMutexInit(&pTsdb->mutex, NULL); - pTsdb->config = pVnode->config.tsdbCfg; - pTsdb->fs = tsdbNewFS(&pTsdb->config); + taosThreadMutexInit(&pTsdb->mutex, NULL); + pTsdb->fs = tsdbNewFS(REPO_CFG(pTsdb)); // create dir (TODO: use tfsMkdir) taosMkDir(pTsdb->path); @@ -46,7 +78,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb) { goto _err; } - tsdbDebug("vgId: %d tsdb is opened", TD_VID(pVnode)); + tsdbDebug("vgId:%d tsdb is opened for %s", TD_VID(pVnode), pTsdb->path); *ppTsdb = pTsdb; return 0; @@ -58,6 +90,7 @@ _err: int tsdbClose(STsdb *pTsdb) { if (pTsdb) { + // TODO: destroy mem/imem tsdbCloseFS(pTsdb); tsdbFreeFS(pTsdb->fs); taosMemoryFree(pTsdb); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 148f2630ca157866d128186c75d7062823b4fa5d..0da18ebdeb36630e59e8c0f4e0176c092b5987b2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -97,39 +97,47 @@ typedef struct SIOCostSummary { int64_t headFileLoadTime; } SIOCostSummary; +typedef struct SBlockLoadSuppInfo { + SColumnDataAgg* pstatis; + SColumnDataAgg** plist; + SArray* defaultLoadColumn; // default load column + int32_t* slotIds; // colId to slotId +} SBlockLoadSuppInfo; + typedef struct STsdbReadHandle { - STsdb* pTsdb; - SQueryFilePos cur; // current position - int16_t order; - STimeWindow window; // the primary query time window that applies to all queries - SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time - int32_t numOfBlocks; - SArray* pColumns; // column list, SColumnInfoData array list - bool locateStart; - int32_t outputCapacity; - int32_t realNumOfRows; - SArray* pTableCheckInfo; // SArray - int32_t activeIndex; - bool checkFiles; // check file stage - int8_t cachelastrow; // check if last row cached - bool loadExternalRow; // load time window external data rows - bool currentLoadExternalRows; // current load external rows - int32_t loadType; // block load type - char* idStr; // query info handle, for debug purpose + STsdb* pTsdb; + SQueryFilePos cur; // current position + int16_t order; + STimeWindow window; // the primary query time window that applies to all queries + // SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time + // SColumnDataAgg** pstatis;// the ptr array list to return to caller + int32_t numOfBlocks; + SArray* pColumns; // column list, SColumnInfoData array list + bool locateStart; + int32_t outputCapacity; + int32_t realNumOfRows; + SArray* pTableCheckInfo; // SArray + int32_t activeIndex; + bool checkFiles; // check file stage + int8_t cachelastrow; // check if last row cached + bool loadExternalRow; // load time window external data rows + bool currentLoadExternalRows; // current load external rows + int32_t loadType; // block load type + char* idStr; // query info handle, for debug purpose int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows SDFileSet* pFileGroup; SFSIter fileIter; SReadH rhelper; STableBlockInfo* pDataBlockInfo; - SDataCols* pDataCols; // in order to hold current file data block - int32_t allocSize; // allocated data block size - SArray* defaultLoadColumn; // default load column - SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ - SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */ - - SArray* prev; // previous row which is before than time window - SArray* next; // next row which is after the query time window - SIOCostSummary cost; + SDataCols* pDataCols; // in order to hold current file data block + int32_t allocSize; // allocated data block size + SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ + SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */ + SBlockLoadSuppInfo suppInfo; + SArray* prev; // previous row which is before than time window + SArray* next; // next row which is after the query time window + SIOCostSummary cost; + STSchema* pSchema; } STsdbReadHandle; typedef struct STableGroupSupporter { @@ -147,7 +155,6 @@ static int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle); static void changeQueryHandleForInterpQuery(tsdbReaderT pHandle); static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); -static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbReadHandle* pTsdbReadHandle); static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); @@ -156,6 +163,8 @@ static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); // static void* destroyTableCheckInfo(SArray* pTableCheckInfo); static bool tsdbGetExternalRow(tsdbReaderT pHandle); +static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions); + static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; pBlockLoadInfo->uid = 0; @@ -311,7 +320,7 @@ static bool emptyQueryTimewindow(STsdbReadHandle* pTsdbReadHandle) { // Update the query time window according to the data time to live(TTL) information, in order to avoid to return // the expired data to client, even it is queried already. static int64_t getEarliestValidTimestamp(STsdb* pTsdb) { - STsdbCfg* pCfg = &pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pTsdb); int64_t now = taosGetTimestamp(pCfg->precision); return now - (tsTickPerDay[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick @@ -342,15 +351,52 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); } } +#if 0 +int nQUERY = 0; +#endif +static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions) { + if (vnodeIsRollup(pVnode)) { + int level = 0; +#if 1 + int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); + for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { + SRetention* pRetention = retentions + i; + if (pRetention->keep <= 0 || (now - pRetention->keep) >= winSKey) { + break; + } + } +#endif +#if 0 + ++nQUERY; + if(nQUERY%3 == 0) { + level = 2; + } else if(nQUERY%2 == 0) { + level = 1; + } else { + level = 0; + } +#endif + if (level == TSDB_RETENTION_L0) { + return VND_RSMA0(pVnode); + } else if (level == TSDB_RETENTION_L1) { + return VND_RSMA1(pVnode); + } else { + return VND_RSMA2(pVnode); + } + } + return pVnode->pTsdb; +} -static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, SQueryTableDataCond* pCond, uint64_t qId, uint64_t taskId) { +static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* pCond, uint64_t qId, uint64_t taskId) { STsdbReadHandle* pReadHandle = taosMemoryCalloc(1, sizeof(STsdbReadHandle)); if (pReadHandle == NULL) { goto _end; } + STsdb* pTsdb = getTsdbByRetentions(pVnode, pCond->twindow.skey, pVnode->config.tsdbCfg.retentions); + pReadHandle->order = pCond->order; - pReadHandle->pTsdb = tsdb; + pReadHandle->pTsdb = pTsdb; pReadHandle->type = TSDB_QUERY_TYPE_ALL; pReadHandle->cur.fid = INT32_MIN; pReadHandle->cur.win = TSWINDOW_INITIALIZER; @@ -368,7 +414,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, SQueryTableDataCond* pC snprintf(buf, tListLen(buf), "TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, qId); pReadHandle->idStr = strdup(buf); - if (tsdbInitReadH(&pReadHandle->rhelper, (STsdb*)tsdb) != 0) { + if (tsdbInitReadH(&pReadHandle->rhelper, pReadHandle->pTsdb) != 0) { goto _end; } @@ -377,8 +423,8 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, SQueryTableDataCond* pC if (pCond->numOfCols > 0) { // allocate buffer in order to load data blocks from file - pReadHandle->statis = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnDataAgg)); - if (pReadHandle->statis == NULL) { + pReadHandle->suppInfo.pstatis = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnDataAgg)); + if (pReadHandle->suppInfo.pstatis == NULL) { goto _end; } @@ -398,13 +444,16 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, SQueryTableDataCond* pC } taosArrayPush(pReadHandle->pColumns, &colInfo); - pReadHandle->statis[i].colId = colInfo.info.colId; } - pReadHandle->defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); + pReadHandle->suppInfo.defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); + pReadHandle->suppInfo.slotIds = + taosMemoryMalloc(sizeof(int32_t) * taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn)); + pReadHandle->suppInfo.plist = + taosMemoryCalloc(taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn), POINTER_BYTES); } - pReadHandle->pDataCols = tdNewDataCols(1000, pReadHandle->pTsdb->config.maxRows); + pReadHandle->pDataCols = tdNewDataCols(1000, pVnode->config.tsdbCfg.maxRows); if (pReadHandle->pDataCols == NULL) { tsdbError("%p failed to malloc buf for pDataCols, %s", pReadHandle, pReadHandle->idStr); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -422,9 +471,9 @@ _end: return NULL; } -tsdbReaderT* tsdbQueryTables(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, +tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { - STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, taskId); + STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(pVnode, pCond, qId, taskId); if (pTsdbReadHandle == NULL) { return NULL; } @@ -441,6 +490,29 @@ tsdbReaderT* tsdbQueryTables(STsdb* tsdb, SQueryTableDataCond* pCond, STableGrou return NULL; } + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0); + + pTsdbReadHandle->pSchema = metaGetTbTSchema(pVnode->pMeta, pCheckInfo->tableId, 0); + int32_t numOfCols = taosArrayGetSize(pTsdbReadHandle->suppInfo.defaultLoadColumn); + int16_t* ids = pTsdbReadHandle->suppInfo.defaultLoadColumn->pData; + + STSchema* pSchema = pTsdbReadHandle->pSchema; + + int32_t i = 0, j = 0; + while (i < numOfCols && j < pSchema->numOfCols) { + if (ids[i] == pSchema->columns[j].colId) { + pTsdbReadHandle->suppInfo.slotIds[i] = j; + i++; + j++; + } else if (ids[i] > pSchema->columns[j].colId) { + j++; + } else { + // tsdbCleanupReadHandle(pTsdbReadHandle); + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + } + tsdbDebug("%p total numOfTable:%" PRIzu " in this query, group %" PRIzu " %s", pTsdbReadHandle, taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), taosArrayGetSize(groupList->pGroupList), pTsdbReadHandle->idStr); @@ -477,7 +549,8 @@ void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond* pCond) { } // allocate buffer in order to load data blocks from file - memset(pTsdbReadHandle->statis, 0, sizeof(SColumnDataAgg)); + memset(pTsdbReadHandle->suppInfo.pstatis, 0, sizeof(SColumnDataAgg)); + memset(pTsdbReadHandle->suppInfo.plist, 0, POINTER_BYTES); tsdbInitDataBlockLoadInfo(&pTsdbReadHandle->dataBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pTsdbReadHandle->compBlockLoadInfo); @@ -505,7 +578,8 @@ void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, SQueryTableDataCon } // allocate buffer in order to load data blocks from file - memset(pTsdbReadHandle->statis, 0, sizeof(SColumnDataAgg)); + memset(pTsdbReadHandle->suppInfo.pstatis, 0, sizeof(SColumnDataAgg)); + memset(pTsdbReadHandle->suppInfo.plist, 0, POINTER_BYTES); tsdbInitDataBlockLoadInfo(&pTsdbReadHandle->dataBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pTsdbReadHandle->compBlockLoadInfo); @@ -526,7 +600,7 @@ void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, SQueryTableDataCon // pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); } -tsdbReaderT tsdbQueryLastRow(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, +tsdbReaderT tsdbQueryLastRow(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { pCond->twindow = updateLastrowForEachGroup(groupList); @@ -535,7 +609,7 @@ tsdbReaderT tsdbQueryLastRow(STsdb* tsdb, SQueryTableDataCond* pCond, STableGrou return NULL; } - STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)tsdbQueryTables(tsdb, pCond, groupList, qId, taskId); + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)tsdbQueryTables(pVnode, pCond, groupList, qId, taskId); if (pTsdbReadHandle == NULL) { return NULL; } @@ -555,8 +629,8 @@ tsdbReaderT tsdbQueryLastRow(STsdb* tsdb, SQueryTableDataCond* pCond, STableGrou } #if 0 -tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { - STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); +tsdbReaderT tsdbQueryCacheLastT(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTablesT(tsdb, pCond, groupList, qId, pMemRef); if (pTsdbReadHandle == NULL) { return NULL; } @@ -618,7 +692,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr return pNew; } -tsdbReaderT tsdbQueryRowsInExternalWindow(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, +tsdbReaderT tsdbQueryRowsInExternalWindow(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); @@ -634,7 +708,7 @@ tsdbReaderT tsdbQueryRowsInExternalWindow(STsdb* tsdb, SQueryTableDataCond* pCon } } - STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)tsdbQueryTables(tsdb, pCond, pNew, qId, taskId); + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)tsdbQueryTables(pVnode, pCond, pNew, qId, taskId); pTsdbReadHandle->loadExternalRow = true; pTsdbReadHandle->currentLoadExternalRows = true; @@ -889,7 +963,7 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { } static bool hasMoreDataInCache(STsdbReadHandle* pHandle) { - STsdbCfg* pCfg = &pHandle->pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pHandle->pTsdb); size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); pHandle->cur.fid = INT32_MIN; @@ -986,7 +1060,7 @@ static int32_t loadBlockInfo(STsdbReadHandle* pTsdbReadHandle, int32_t index, in pCheckInfo->numOfBlocks = 0; STable table = {.uid = pCheckInfo->tableId, .tid = pCheckInfo->tableId}; - table.pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0); + table.pSchema = pTsdbReadHandle->pSchema; if (tsdbSetReadTable(&pTsdbReadHandle->rhelper, &table) != TSDB_CODE_SUCCESS) { code = terrno; @@ -1091,29 +1165,28 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl int32_t slotIndex) { int64_t st = taosGetTimestampUs(); - STSchema* pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0); - int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pSchema); + int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pTsdbReadHandle->pSchema); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p failed to malloc buf for pDataCols, %s", pTsdbReadHandle, pTsdbReadHandle->idStr); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[0], pSchema); + code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[0], pTsdbReadHandle->pSchema); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], %s", pTsdbReadHandle, pTsdbReadHandle->idStr); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[1], pSchema); + code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[1], pTsdbReadHandle->pSchema); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], %s", pTsdbReadHandle, pTsdbReadHandle->idStr); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - int16_t* colIds = pTsdbReadHandle->defaultLoadColumn->pData; + int16_t* colIds = pTsdbReadHandle->suppInfo.defaultLoadColumn->pData; int32_t ret = tsdbLoadBlockDataCols(&(pTsdbReadHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)), true); @@ -1169,7 +1242,7 @@ static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo) { SQueryFilePos* cur = &pTsdbReadHandle->cur; - STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); TSKEY key; int32_t code = TSDB_CODE_SUCCESS; @@ -1263,6 +1336,8 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* return code; } +static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); + static int32_t loadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { SQueryFilePos* cur = &pTsdbReadHandle->cur; @@ -1756,7 +1831,7 @@ int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* p static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { SQueryFilePos* cur = &pTsdbReadHandle->cur; SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); - STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); initTableMemIterator(pTsdbReadHandle, pCheckInfo); @@ -2200,7 +2275,7 @@ static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exi int32_t numOfBlocks = 0; int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); - STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); STimeWindow win = TSWINDOW_INITIALIZER; while (true) { @@ -2306,7 +2381,7 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* // find the start data block in file pTsdbReadHandle->locateStart = true; - STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->days, pCfg->precision); tsdbRLockFS(pFileHandle); @@ -2407,7 +2482,7 @@ static int32_t getDataBlocksInFiles(STsdbReadHandle* pTsdbReadHandle, bool* exis // find the start data block in file if (!pTsdbReadHandle->locateStart) { pTsdbReadHandle->locateStart = true; - STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->days, pCfg->precision); tsdbRLockFS(pFileHandle); @@ -2498,7 +2573,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STsdbReadHandle* pTsdbReadHandle) { int numOfRows = 0; int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); - STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); win->skey = TSKEY_INITIAL_VAL; int64_t st = taosGetTimestampUs(); @@ -3233,14 +3308,15 @@ void tsdbRetrieveDataBlockInfo(tsdbReaderT* pTsdbReadHandle, SDataBlockInfo* pDa pDataBlockInfo->rows = cur->rows; pDataBlockInfo->window = cur->win; - pDataBlockInfo->numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pHandle)); +// ASSERT(pDataBlockInfo->numOfCols >= (int32_t)(QH_GET_NUM_OF_COLS(pHandle)); } /* * return null for mixed data block, if not a complete file data block, the statistics value will always return NULL */ -int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SColumnDataAgg** pBlockStatis) { +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SColumnDataAgg*** pBlockStatis, bool* allHave) { STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; + *allHave = false; SQueryFilePos* c = &pHandle->cur; if (c->mixBlock) { @@ -3269,35 +3345,47 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SColumnDat tsdbDebug("vgId:%d succeed to load block statis part for uid %" PRIu64, REPO_ID(pHandle->pTsdb), TSDB_READ_TABLE_UID(&pHandle->rhelper)); - int16_t* colIds = pHandle->defaultLoadColumn->pData; + int16_t* colIds = pHandle->suppInfo.defaultLoadColumn->pData; size_t numOfCols = QH_GET_NUM_OF_COLS(pHandle); - memset(pHandle->statis, 0, numOfCols * sizeof(SColumnDataAgg)); + memset(pHandle->suppInfo.plist, 0, numOfCols * POINTER_BYTES); + memset(pHandle->suppInfo.pstatis, 0, numOfCols * sizeof(SColumnDataAgg)); + for (int32_t i = 0; i < numOfCols; ++i) { - pHandle->statis[i].colId = colIds[i]; + pHandle->suppInfo.pstatis[i].colId = colIds[i]; } - tsdbGetBlockStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols, pBlockInfo->compBlock); + *allHave = true; + tsdbGetBlockStatis(&pHandle->rhelper, pHandle->suppInfo.pstatis, (int)numOfCols, pBlockInfo->compBlock); // always load the first primary timestamp column data - SColumnDataAgg* pPrimaryColStatis = &pHandle->statis[0]; + SColumnDataAgg* pPrimaryColStatis = &pHandle->suppInfo.pstatis[0]; assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_ID); pPrimaryColStatis->numOfNull = 0; pPrimaryColStatis->min = pBlockInfo->compBlock->keyFirst; pPrimaryColStatis->max = pBlockInfo->compBlock->keyLast; + pHandle->suppInfo.plist[0] = &pHandle->suppInfo.pstatis[0]; // update the number of NULL data rows + int32_t* slotIds = pHandle->suppInfo.slotIds; for (int32_t i = 1; i < numOfCols; ++i) { - if (pHandle->statis[i].numOfNull == -1) { // set the column data are all NULL - pHandle->statis[i].numOfNull = pBlockInfo->compBlock->numOfRows; + ASSERT(colIds[i] == pHandle->pSchema->columns[slotIds[i]].colId); + if (IS_BSMA_ON(&(pHandle->pSchema->columns[slotIds[i]]))) { + if (pHandle->suppInfo.pstatis[i].numOfNull == -1) { // set the column data are all NULL + pHandle->suppInfo.pstatis[i].numOfNull = pBlockInfo->compBlock->numOfRows; + } else { + pHandle->suppInfo.plist[i] = &pHandle->suppInfo.pstatis[i]; + } + } else { + *allHave = false; } } int64_t elapsed = taosGetTimestampUs() - stime; pHandle->cost.statisInfoLoadTime += elapsed; - *pBlockStatis = pHandle->statis; + *pBlockStatis = pHandle->suppInfo.plist; return TSDB_CODE_SUCCESS; } @@ -3717,7 +3805,7 @@ _error: } #if 0 -int32_t tsdbGetTableGroupFromIdList(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetTableGroupFromIdListT(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) { return terrno; } @@ -3799,9 +3887,10 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle) { pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns); - taosArrayDestroy(pTsdbReadHandle->defaultLoadColumn); + taosArrayDestroy(pTsdbReadHandle->suppInfo.defaultLoadColumn); taosMemoryFreeClear(pTsdbReadHandle->pDataBlockInfo); - taosMemoryFreeClear(pTsdbReadHandle->statis); + taosMemoryFreeClear(pTsdbReadHandle->suppInfo.pstatis); + taosMemoryFreeClear(pTsdbReadHandle->suppInfo.plist); if (!emptyQueryTimewindow(pTsdbReadHandle)) { // tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index c39780372b98a83b60285d4530f397be16726fd6..d7c9a70c183786cce37129cb714d36186eacdf29 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -248,7 +248,8 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { ASSERT(pBlock->numOfSubBlocks > 0); - int8_t update = pReadh->pRepo->config.update; + STsdbCfg *pCfg = REPO_CFG(pReadh->pRepo); + int8_t update = pCfg->update; SBlock *iBlock = pBlock; if (pBlock->numOfSubBlocks > 1) { @@ -279,7 +280,7 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds, bool mergeBitmap) { ASSERT(pBlock->numOfSubBlocks > 0); - int8_t update = pReadh->pRepo->config.update; + int8_t update = pReadh->pRepo->pVnode->config.tsdbCfg.update; SBlock *iBlock = pBlock; if (pBlock->numOfSubBlocks > 1) { @@ -471,7 +472,7 @@ void tsdbGetBlockStatis(SReadH *pReadh, SColumnDataAgg *pStatis, int numOfCols, SAggrBlkData *pAggrBlkData = pReadh->pAggrBlkData; for (int i = 0, j = 0; i < numOfCols;) { - if (j >= pBlock->numOfCols) { + if (j >= pBlock->numOfBSma) { pStatis[i].numOfNull = -1; ++i; continue; diff --git a/source/dnode/vnode/src/tsdb/tsdbScan.c b/source/dnode/vnode/src/tsdb/tsdbScan.c deleted file mode 100644 index cc6fb473d00c299475c7140e57d6471649915328..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/tsdb/tsdbScan.c +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#if 0 -#include "tsdb.h" -#ifndef _TSDB_PLUGINS - -int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } - -STsdbScanHandle* tsdbNewScanHandle() { return NULL; } - -void tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream) {} - -int tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } - -int tsdbScanSBlockIdx(STsdbScanHandle* pScanHandle) { return 0; } - -int tsdbScanSBlock(STsdbScanHandle* pScanHandle, int idx) { return 0; } - -int tsdbCloseScanFile(STsdbScanHandle* pScanHandle) { return 0; } - -void tsdbFreeScanHandle(STsdbScanHandle* pScanHandle) {} - -#endif -#endif \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 1abca21d34f9235511b477ee3e484d32e68558d3..480b1319146c7ff6f69608374f9092a6477b8605 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -103,10 +103,9 @@ typedef struct { STSma *pSma; // cache schema } SSmaStatItem; -#define RSMA_MAX_LEVEL 2 #define RSMA_TASK_INFO_HASH_SLOT 8 struct SRSmaInfo { - void *taskInfo[RSMA_MAX_LEVEL]; // qTaskInfo_t + void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t }; struct SSmaStat { @@ -128,7 +127,7 @@ static FORCE_INLINE void tsdbFreeTaskHandle(qTaskInfo_t *taskHandle) { } static FORCE_INLINE void *tsdbFreeRSmaInfo(SRSmaInfo *pInfo) { - for (int32_t i = 0; i < RSMA_MAX_LEVEL; ++i) { + for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) { if (pInfo->taskInfo[i]) { tsdbFreeTaskHandle(pInfo->taskInfo[i]); } @@ -175,6 +174,9 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg); static FORCE_INLINE int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids); +static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, + qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, + int8_t level); // mgmt interface static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); @@ -224,7 +226,7 @@ static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) { } static SPoolMem *openPool() { - SPoolMem *pPool = (SPoolMem *)tdbOsMalloc(sizeof(*pPool)); + SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool)); pPool->prev = pPool->next = pPool; pPool->size = 0; @@ -246,7 +248,7 @@ static void clearPool(SPoolMem *pPool) { pMem->prev->next = pMem->next; pPool->size -= pMem->size; - tdbOsFree(pMem); + taosMemoryFree(pMem); } while (1); assert(pPool->size == 0); @@ -255,7 +257,7 @@ static void clearPool(SPoolMem *pPool) { static void closePool(SPoolMem *pPool) { if (pPool) { clearPool(pPool); - tdbOsFree(pPool); + taosMemoryFree(pPool); } } @@ -264,7 +266,7 @@ static void *poolMalloc(void *arg, size_t size) { SPoolMem *pPool = (SPoolMem *)arg; SPoolMem *pMem; - pMem = (SPoolMem *)tdbOsMalloc(sizeof(*pMem) + size); + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); if (!pMem) { assert(0); } @@ -291,7 +293,7 @@ static void poolFree(void *arg, void *ptr) { pMem->prev->next = pMem->next; pPool->size -= pMem->size; - tdbOsFree(pMem); + taosMemoryFree(pMem); } int32_t tsdbInitSma(STsdb *pTsdb) { @@ -703,25 +705,25 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers SInterval interval = {0}; TSKEY lastWinSKey = INT64_MIN; - if (tInitSubmitMsgIterEx(pMsg, &msgIter) != TSDB_CODE_SUCCESS) { + if (tInitSubmitMsgIter(pMsg, &msgIter) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; } while (true) { - tGetSubmitMsgNextEx(&msgIter, &pBlock); + tGetSubmitMsgNext(&msgIter, &pBlock); if (!pBlock) break; STSmaWrapper *pSW = NULL; STSma *pTSma = NULL; SSubmitBlkIter blkIter = {0}; - if (tInitSubmitBlkIterEx(&msgIter, pBlock, &blkIter) != TSDB_CODE_SUCCESS) { + if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) != TSDB_CODE_SUCCESS) { pSW = tdFreeTSmaWrapper(pSW); break; } while (true) { - STSRow *row = tGetSubmitBlkNextEx(&blkIter); + STSRow *row = tGetSubmitBlkNext(&blkIter); if (!row) { tdFreeTSmaWrapper(pSW); break; @@ -1882,8 +1884,10 @@ int32_t tsdbFetchTbUidList(STsdb *pTsdb, STbUidStore **ppStore, tb_uid_t suid, t return TSDB_CODE_SUCCESS; } + ASSERT(ppStore != NULL); + if (!(*ppStore)) { - if (tsdbUidStoreInit((STbUidStore **)ppStore) != 0) { + if (tsdbUidStoreInit(ppStore) != 0) { return TSDB_CODE_FAILED; } } @@ -1958,6 +1962,21 @@ int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pStore) { return TSDB_CODE_SUCCESS; } +static int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { + if (!pReq) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + + if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { ASSERT(pMsg != NULL); SSubmitMsgIter msgIter = {0}; @@ -1967,87 +1986,101 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { terrno = TSDB_CODE_SUCCESS; - if (tInitSubmitMsgIterEx(pMsg, &msgIter) < 0) return -1; + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; while (true) { - if (tGetSubmitMsgNextEx(&msgIter, &pBlock) < 0) return -1; + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; if (!pBlock) break; tsdbUidStorePut(pStore, msgIter.suid, NULL); + pStore->uid = msgIter.uid; // TODO: remove, just for debugging } if (terrno != TSDB_CODE_SUCCESS) return -1; return 0; } -int32_t tsdbExecuteRSma(STsdb *pTsdb, SMeta *pMeta, const void *pMsg, int32_t inputType, tb_uid_t *suid) { +static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, + qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, + int8_t level) { + SArray *pResult = NULL; + tsdbDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), level, taskInfo, + suid); + + qSetStreamInput(taskInfo, pMsg, inputType); + while (1) { + SSDataBlock *output = NULL; + uint64_t ts; + if (qExecTask(taskInfo, &output, &ts) < 0) { + ASSERT(false); + } + if (!output) { + break; + } + if (!pResult) { + pResult = taosArrayInit(0, sizeof(SSDataBlock)); + if (!pResult) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + } + + taosArrayPush(pResult, output); + } + + if (taosArrayGetSize(pResult) > 0) { + blockDebugShowData(pResult); + STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pTsdb->pVnode->pRSma1 : pTsdb->pVnode->pRSma2); + SSubmitReq *pReq = NULL; + if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, TD_VID(pTsdb->pVnode), uid, suid) != 0) { + taosArrayDestroy(pResult); + return TSDB_CODE_FAILED; + } + if (tsdbProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) != 0) { + taosArrayDestroy(pResult); + taosMemoryFreeClear(pReq); + return TSDB_CODE_FAILED; + } + taosMemoryFreeClear(pReq); + } else { + tsdbWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", REPO_ID(pTsdb), level, tstrerror(terrno)); + } + + taosArrayDestroy(pResult); + + return TSDB_CODE_SUCCESS; +} + +static int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType, tb_uid_t suid, tb_uid_t uid) { SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); if (!pEnv) { // only applicable when rsma env exists return TSDB_CODE_SUCCESS; } + ASSERT(uid != 0); // TODO: remove later + SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { - tsdbDebug("vgId:%d no rsma info for suid:%" PRIu64, REPO_ID(pTsdb), *suid); + tsdbDebug("vgId:%d no rsma info for suid:%" PRIu64, REPO_ID(pTsdb), suid); return TSDB_CODE_SUCCESS; } - SArray *pResult = NULL; - - pResult = taosArrayInit(0, sizeof(SSDataBlock)); - if (!pResult) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - if (pRSmaInfo->taskInfo[0]) { - tsdbDebug("vgId:%d execute rsma task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), pRSmaInfo->taskInfo[0], - *suid); - qSetStreamInput(pRSmaInfo->taskInfo[0], pMsg, inputType); - while (1) { - SSDataBlock *output; - uint64_t ts; - if (qExecTask(pRSmaInfo->taskInfo[0], &output, &ts) < 0) { - ASSERT(false); - } - if (!output) { - break; - } - taosArrayPush(pResult, output); - } - if (taosArrayGetSize(pResult) > 0) { - blockDebugShowData(pResult); - } else { - tsdbWarn("vgId:%d no sma data generated since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - } - - // if (pRSmaInfo->taskInfo[1]) { - // qSetStreamInput(pRSmaInfo->taskInfo[1], pMsg, inputType); - // while (1) { - // SSDataBlock *output; - // uint64_t ts; - // if (qExecTask(pRSmaInfo->taskInfo[1], &output, &ts) < 0) { - // ASSERT(false); - // } - // if (!output) { - // break; - // } - // taosArrayPush(pResult, output); - // } - // blockDebugShowData(pResult); - // } + // TODO: use the proper schema instead of 0, and cache STSchema in cache + STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 0); + tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[0], pTSchema, suid, uid, TSDB_RETENTION_L1); + tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[1], pTSchema, suid, uid, TSDB_RETENTION_L2); + taosMemoryFree(pTSchema); } return TSDB_CODE_SUCCESS; } -int32_t tsdbTriggerRSma(STsdb *pTsdb, SMeta *pMeta, void *pMsg, int32_t inputType) { +int32_t tsdbTriggerRSma(STsdb *pTsdb, void *pMsg, int32_t inputType) { SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); if (!pEnv) { // only applicable when rsma env exists @@ -2059,12 +2092,12 @@ int32_t tsdbTriggerRSma(STsdb *pTsdb, SMeta *pMeta, void *pMsg, int32_t inputTyp tsdbFetchSubmitReqSuids(pMsg, &uidStore); if (uidStore.suid != 0) { - tsdbExecuteRSma(pTsdb, pMeta, pMsg, inputType, &uidStore.suid); + tsdbExecuteRSma(pTsdb, pMsg, inputType, uidStore.suid, uidStore.uid); void *pIter = taosHashIterate(uidStore.uidHash, NULL); while (pIter) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - tsdbExecuteRSma(pTsdb, pMeta, pMsg, inputType, pTbSuid); + tsdbExecuteRSma(pTsdb, pMsg, inputType, *pTbSuid, 0); pIter = taosHashIterate(uidStore.uidHash, pIter); } diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c index ac3baf8c3af5eed608206ba187f0e5a795419a0c..8a553e94fb8b9b4287ca8e4d5cf4db5318c2d94f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c @@ -55,12 +55,12 @@ static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, i } static int32_t tsdbOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { - int ret; - FKeyComparator compFunc; + int ret; + tdb_cmpr_fn_t compFunc; // Create a database compFunc = tsdbSmaKeyCmpr; - ret = tdbDbOpen(pFName, TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, ppDB); + ret = tdbDbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 82fa6b9ae5772276ad5623c6b37fc69b541ed600..88b637bc246f3d73c2a7f0acc147606be2e537d0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -38,11 +38,11 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * while (true) { tGetSubmitMsgNext(&msgIter, &pBlock); if (pBlock == NULL) break; - if (tsdbInsertTableData(pTsdb, pBlock, &affectedrows) < 0) { + if (tsdbInsertTableData(pTsdb, &msgIter, pBlock, &affectedrows) < 0) { return -1; } - numOfRows += pBlock->numOfRows; + numOfRows += msgIter.numOfRows; } if (pRsp != NULL) { @@ -60,25 +60,26 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { SSubmitBlk *pBlock = NULL; SSubmitBlkIter blkIter = {0}; STSRow *row = NULL; - TSKEY now = taosGetTimestamp(pTsdb->config.precision); - TSKEY minKey = now - tsTickPerDay[pTsdb->config.precision] * pTsdb->config.keep2; - TSKEY maxKey = now + tsTickPerDay[pTsdb->config.precision] * pTsdb->config.days; + STsdbCfg *pCfg = REPO_CFG(pTsdb); + TSKEY now = taosGetTimestamp(pCfg->precision); + TSKEY minKey = now - tsTickPerDay[pCfg->precision] * pCfg->keep2; + TSKEY maxKey = now + tsTickPerDay[pCfg->precision] * pCfg->days; terrno = TSDB_CODE_SUCCESS; - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + // pMsg->length = htonl(pMsg->length); + // pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; while (true) { if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; if (pBlock == NULL) break; - pBlock->uid = htobe64(pBlock->uid); - pBlock->suid = htobe64(pBlock->suid); - pBlock->sversion = htonl(pBlock->sversion); - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->schemaLen = htonl(pBlock->schemaLen); - pBlock->numOfRows = htons(pBlock->numOfRows); + // pBlock->uid = htobe64(pBlock->uid); + // pBlock->suid = htobe64(pBlock->suid); + // pBlock->sversion = htonl(pBlock->sversion); + // pBlock->dataLen = htonl(pBlock->dataLen); + // pBlock->schemaLen = htonl(pBlock->schemaLen); + // pBlock->numOfRows = htons(pBlock->numOfRows); #if 0 if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) { diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 0e55113dc3f1600ac3ed6a041b43f9d934d5d440..7eac0389d22858f94df2666acf5c7daa46cfee35 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -23,7 +23,6 @@ const SVnodeCfg vnodeCfgDefault = { .szCache = 256, .szBuf = 96 * 1024 * 1024, .isHeap = false, - .streamMode = 0, .isWeak = 0, .tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI, .update = 0, @@ -56,7 +55,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "streamMode", pCfg->streamMode) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1; @@ -68,6 +66,26 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; + if (pCfg->tsdbCfg.retentions[0].freq > 0) { + int32_t nRetention = 1; + if (pCfg->tsdbCfg.retentions[1].freq > 0) { + ++nRetention; + if (pCfg->tsdbCfg.retentions[2].freq > 0) { + ++nRetention; + } + } + SJson *pNodeRetentions = tjsonCreateArray(); + tjsonAddItemToObject(pJson, "retentions", pNodeRetentions); + for (int32_t i = 0; i < nRetention; ++i) { + SJson *pNodeRetention = tjsonCreateObject(); + const SRetention *pRetention = pCfg->tsdbCfg.retentions + i; + tjsonAddIntegerToObject(pNodeRetention, "freq", pRetention->freq); + tjsonAddIntegerToObject(pNodeRetention, "freqUnit", pRetention->freqUnit); + tjsonAddIntegerToObject(pNodeRetention, "keep", pRetention->keep); + tjsonAddIntegerToObject(pNodeRetention, "keepUnit", pRetention->keepUnit); + tjsonAddItemToArray(pNodeRetentions, pNodeRetention); + } + } if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; @@ -104,7 +122,6 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tjsonGetNumberValue(pJson, "szCache", pCfg->szCache) < 0) return -1; if (tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf) < 0) return -1; if (tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap) < 0) return -1; - if (tjsonGetNumberValue(pJson, "streamMode", pCfg->streamMode) < 0) return -1; if (tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak) < 0) return -1; if (tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; if (tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1; @@ -116,6 +133,19 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; if (tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; if (tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; + SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions"); + int32_t nRetention = tjsonGetArraySize(pNodeRetentions); + if (nRetention > TSDB_RETENTION_MAX) { + nRetention = TSDB_RETENTION_MAX; + } + for (int32_t i = 0; i < nRetention; ++i) { + SJson *pNodeRetention = tjsonGetArrayItem(pNodeRetentions, i); + ASSERT(pNodeRetention != NULL); + tjsonGetNumberValue(pNodeRetention, "freq", (pCfg->tsdbCfg.retentions)[i].freq); + tjsonGetNumberValue(pNodeRetention, "freqUnit", (pCfg->tsdbCfg.retentions)[i].freqUnit); + tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep); + tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit); + } if (tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; if (tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; if (tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 1c0e65aa95a8a0aa543b101ef92f21444b2fef52..6d8bcb35c84a683b627b4a85dfc5e216c08b76e9 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -42,14 +42,31 @@ int vnodeBegin(SVnode *pVnode) { // begin meta if (metaBegin(pVnode->pMeta) < 0) { - vError("vgId: %d failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } // begin tsdb - if (tsdbBegin(pVnode->pTsdb) < 0) { - vError("vgId: %d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; + if (vnodeIsRollup(pVnode)) { + if (tsdbBegin(VND_RSMA0(pVnode)) < 0) { + vError("vgId:%d failed to begin rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + + if (tsdbBegin(VND_RSMA1(pVnode)) < 0) { + vError("vgId:%d failed to begin rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + + if (tsdbBegin(VND_RSMA2(pVnode)) < 0) { + vError("vgId:%d failed to begin rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + } else { + if (tsdbBegin(pVnode->pTsdb) < 0) { + vError("vgId:%d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } } return 0; @@ -93,7 +110,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { // free info binary taosMemoryFree(data); - vInfo("vgId: %d vnode info is saved, fname: %s", pInfo->config.vgId, fname); + vInfo("vgId:%d vnode info is saved, fname: %s", pInfo->config.vgId, fname); return 0; @@ -115,7 +132,7 @@ int vnodeCommitInfo(const char *dir, const SVnodeInfo *pInfo) { return -1; } - vInfo("vgId: %d vnode info is committed", pInfo->config.vgId); + vInfo("vgId:%d vnode info is committed", pInfo->config.vgId); return 0; } @@ -190,7 +207,7 @@ int vnodeSyncCommit(SVnode *pVnode) { } int vnodeCommit(SVnode *pVnode) { - SVnodeInfo info; + SVnodeInfo info = {0}; char dir[TSDB_FILENAME_LEN]; vInfo("vgId:%d start to commit, version: %" PRId64, TD_VID(pVnode), pVnode->state.applied); @@ -212,10 +229,28 @@ int vnodeCommit(SVnode *pVnode) { ASSERT(0); return -1; } - if (tsdbCommit(pVnode->pTsdb) < 0) { - ASSERT(0); - return -1; + + if(vnodeIsRollup(pVnode)) { + if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { + ASSERT(0); + return -1; + } + if (tsdbCommit(VND_RSMA1(pVnode)) < 0) { + ASSERT(0); + return -1; + } + if (tsdbCommit(VND_RSMA2(pVnode)) < 0) { + ASSERT(0); + return -1; + } + } else { + if (tsdbCommit(pVnode->pTsdb) < 0) { + ASSERT(0); + return -1; + } } + + if (tqCommit(pVnode->pTq) < 0) { ASSERT(0); return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index ee64976df972e65a9af2865cf6fb2dc7d780acc0..ae134e64964ab60b0e1c462013f53b4eb822bddc 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -23,13 +23,13 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { // check config if (vnodeCheckCfg(pCfg) < 0) { - vError("vgId: %d failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); return -1; } // create vnode env if (tfsMkdir(pTfs, path) < 0) { - vError("vgId: %d failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); return -1; } @@ -39,11 +39,11 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { info.state.applied = -1; if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) { - vError("vgId: %d failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno)); return -1; } - vInfo("vgId: %d vnode is created", pCfg->vgId); + vInfo("vgId:%d vnode is created", pCfg->vgId); return 0; } @@ -70,7 +70,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { pVnode = (SVnode *)taosMemoryCalloc(1, sizeof(*pVnode) + strlen(path) + 1); if (pVnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - vError("vgId: %d failed to open vnode since %s", info.config.vgId, tstrerror(terrno)); + vError("vgId:%d failed to open vnode since %s", info.config.vgId, tstrerror(terrno)); return NULL; } @@ -85,27 +85,44 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // open buffer pool if (vnodeOpenBufPool(pVnode, pVnode->config.isHeap ? 0 : pVnode->config.szBuf / 3) < 0) { - vError("vgId: %d failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open meta if (metaOpen(pVnode, &pVnode->pMeta) < 0) { - vError("vgId: %d failed to open vnode meta since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode meta since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open tsdb - if (tsdbOpen(pVnode, &pVnode->pTsdb) < 0) { - vError("vgId: %d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; + if (vnodeIsRollup(pVnode)) { + if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L0) < 0) { + vError("vgId:%d failed to open vnode rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + + if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L1) < 0) { + vError("vgId:%d failed to open vnode rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + + if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L2) < 0) { + vError("vgId:%d failed to open vnode rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + } else { + if (tsdbOpen(pVnode, TSDB_TYPE_TSDB) < 0) { + vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } } // open wal sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg)); if (pVnode->pWal == NULL) { - vError("vgId: %d failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } @@ -113,25 +130,25 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); pVnode->pTq = tqOpen(tdir, pVnode, pVnode->pWal); if (pVnode->pTq == NULL) { - vError("vgId: %d failed to open vnode tq since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode tq since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open query if (vnodeQueryOpen(pVnode)) { - vError("vgId: %d failed to open vnode query since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode query since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // vnode begin if (vnodeBegin(pVnode) < 0) { - vError("vgId: %d failed to begin since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to begin since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open sync if (vnodeSyncOpen(pVnode, dir)) { - vError("vgId: %d failed to open sync since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open sync since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } @@ -143,6 +160,8 @@ _err: if (pVnode->pWal) walClose(pVnode->pWal); if (pVnode->pTsdb) tsdbClose(pVnode->pTsdb); if (pVnode->pMeta) metaClose(pVnode->pMeta); + tsdbClose(VND_RSMA1(pVnode)); + tsdbClose(VND_RSMA2(pVnode)); tsem_destroy(&(pVnode->canCommit)); taosMemoryFree(pVnode); return NULL; @@ -155,7 +174,9 @@ void vnodeClose(SVnode *pVnode) { vnodeQueryClose(pVnode); walClose(pVnode->pWal); tqClose(pVnode->pTq); - tsdbClose(pVnode->pTsdb); + tsdbClose(VND_TSDB(pVnode)); + tsdbClose(VND_RSMA1(pVnode)); + tsdbClose(VND_RSMA2(pVnode)); metaClose(pVnode->pMeta); vnodeCloseBufPool(pVnode); // destroy handle diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 605185c9d940f7eb1d0a42f142433c9ff12103ba..7e80eacf8f8026f594998bda2a079c59d21f7145 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -147,4 +147,19 @@ void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) { if (vgId) { *vgId = TD_VID(pVnode); } +} + +// wrapper of tsdb read interface +tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableGroupInfo *groupList, uint64_t qId, + void *pMemRef) { +#if 0 + return tsdbQueryCacheLastT(pVnode->pTsdb, pCond, groupList, qId, pMemRef); +#endif + return 0; +} +int32_t tsdbGetTableGroupFromIdList(SVnode *pVnode, SArray *pTableIdList, STableGroupInfo *pGroupInfo) { +#if 0 + return tsdbGetTableGroupFromIdListT(pVnode->pTsdb, pTableIdList, pGroupInfo); +#endif + return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 90d93bd3f5bbb7cac774dd576a6bb08282a89584..8460400b5971dd6ef613cd71af826cd996d221c8 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -17,10 +17,10 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessDropStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); +static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessDropTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); +static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { @@ -52,7 +52,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg int len; int ret; - vTrace("vgId: %d start to process write request %s, version %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + vTrace("vgId:%d start to process write request %s, version %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); pVnode->state.applied = version; @@ -62,7 +62,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg len = pMsg->contLen - sizeof(SMsgHead); if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { - vError("vgId: %d failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } @@ -75,7 +75,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg if (vnodeProcessAlterStbReq(pVnode, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_STB: - if (vnodeProcessDropStbReq(pVnode, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDropStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_CREATE_TABLE: if (vnodeProcessCreateTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; @@ -84,7 +84,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg if (vnodeProcessAlterTbReq(pVnode, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_TABLE: - if (vnodeProcessDropTbReq(pVnode, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_CREATE_SMA: { // timeRangeSMA if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { @@ -119,7 +119,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg break; } - vDebug("vgId: %d process %s request success, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); + vDebug("vgId:%d process %s request success, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); // commit if need if (vnodeShouldCommit(pVnode)) { @@ -134,15 +134,17 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg return 0; _err: - vDebug("vgId: %d process %s request failed since %s, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + vDebug("vgId:%d process %s request failed since %s, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), tstrerror(terrno), version); return -1; } int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in vnode query queue is processing"); +#if 0 SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode}; - +#endif + SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode}; switch (pMsg->msgType) { case TDMT_VND_QUERY: return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg); @@ -337,7 +339,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, goto _exit; } - rsp.pArray = taosArrayInit(sizeof(cRsp), req.nReqs); + rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp)); if (rsp.pArray == NULL) { rcode = -1; terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -407,9 +409,32 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpc return 0; } -static int vnodeProcessDropStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) { - // TODO - // ASSERT(0); +static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SVDropStbReq req = {0}; + int rcode = TSDB_CODE_SUCCESS; + SCoder coder = {0}; + + pRsp->msgType = TDMT_VND_CREATE_STB_RSP; + pRsp->pCont = NULL; + pRsp->contLen = 0; + + // decode request + tCoderInit(&coder, TD_LITTLE_ENDIAN, pReq, len, TD_DECODER); + if (tDecodeSVDropStbReq(&coder, &req) < 0) { + rcode = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + // process request + // if (metaDropSTable(pVnode->pMeta, version, &req) < 0) { + // rcode = terrno; + // goto _exit; + // } + + // return rsp +_exit: + pRsp->code = rcode; + tCoderClear(&coder); return 0; } @@ -419,32 +444,109 @@ static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcM return 0; } -static int vnodeProcessDropTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) { - // TODO - ASSERT(0); +static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SVDropTbBatchReq req = {0}; + SVDropTbBatchRsp rsp = {0}; + SCoder coder = {0}; + int ret; + + pRsp->msgType = TDMT_VND_DROP_TABLE_RSP; + pRsp->pCont = NULL; + pRsp->contLen = 0; + pRsp->code = TSDB_CODE_SUCCESS; + + // decode req + tCoderInit(&coder, TD_LITTLE_ENDIAN, pReq, len, TD_DECODER); + ret = tDecodeSVDropTbBatchReq(&coder, &req); + if (ret < 0) { + terrno = TSDB_CODE_INVALID_MSG; + pRsp->code = terrno; + goto _exit; + } + + // process req + rsp.pArray = taosArrayInit(sizeof(SVDropTbRsp), req.nReqs); + for (int iReq = 0; iReq < req.nReqs; iReq++) { + SVDropTbReq *pDropTbReq = req.pReqs + iReq; + SVDropTbRsp dropTbRsp = {0}; + + /* code */ + ret = metaDropTable(pVnode->pMeta, version, pDropTbReq); + if (ret < 0) { + if (pDropTbReq->igNotExists && terrno == TSDB_CODE_VND_TABLE_NOT_EXIST) { + dropTbRsp.code = TSDB_CODE_SUCCESS; + } else { + dropTbRsp.code = terrno; + } + } else { + dropTbRsp.code = TSDB_CODE_SUCCESS; + } + + taosArrayPush(rsp.pArray, &dropTbRsp); + } + +_exit: + tCoderClear(&coder); + // encode rsp (TODO) return 0; } static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - SSubmitRsp rsp = {0}; + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock; + SSubmitRsp rsp = {0}; + SVCreateTbReq createTbReq = {0}; + SCoder coder = {0}; + int32_t nRows; pRsp->code = 0; - tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + // handle the request - if (tsdbInsertData(pVnode->pTsdb, version, pSubmitReq, &rsp) < 0) { - pRsp->code = terrno; - return -1; + if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { + pRsp->code = TSDB_CODE_INVALID_MSG; + goto _exit; } - // pRsp->msgType = TDMT_VND_SUBMIT_RSP; - // vnodeProcessSubmitReq(pVnode, ptr, pRsp); - // tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + for (;;) { + tGetSubmitMsgNext(&msgIter, &pBlock); + if (pBlock == NULL) break; + + // create table for auto create table mode + if (msgIter.schemaLen > 0) { + tCoderInit(&coder, TD_LITTLE_ENDIAN, pBlock->data, msgIter.schemaLen, TD_DECODER); + if (tDecodeSVCreateTbReq(&coder, &createTbReq) < 0) { + pRsp->code = TSDB_CODE_INVALID_MSG; + tCoderClear(&coder); + goto _exit; + } + + if (metaCreateTable(pVnode->pMeta, version, &createTbReq) < 0) { + if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + pRsp->code = terrno; + tCoderClear(&coder); + goto _exit; + } + } + + tCoderClear(&coder); + } + if (tsdbInsertTableData(pVnode->pTsdb, &msgIter, pBlock, &nRows) < 0) { + pRsp->code = terrno; + goto _exit; + } + + rsp.numOfRows += nRows; + } + +_exit: // encode the response (TODO) pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp)); memcpy(pRsp->pCont, &rsp, sizeof(rsp)); pRsp->contLen = sizeof(SSubmitRsp); + tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + return 0; } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 918892b78627cdaa0c6658fba9349abb0f511635..3e8528e3d9d670ca6f323b27d30c07967f1d12c9 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -54,6 +54,7 @@ enum { CTG_ACT_REMOVE_DB, CTG_ACT_REMOVE_STB, CTG_ACT_REMOVE_TBL, + CTG_ACT_UPDATE_USER, CTG_ACT_MAX }; @@ -95,8 +96,18 @@ typedef struct SCtgRentMgmt { SCtgRentSlot *slots; } SCtgRentMgmt; +typedef struct SCtgUserAuth { + int32_t version; + SRWLatch lock; + bool superUser; + SHashObj *createdDbs; + SHashObj *readDbs; + SHashObj *writeDbs; +} SCtgUserAuth; + typedef struct SCatalog { uint64_t clusterId; + SHashObj *userCache; //key:user, value:SCtgUserAuth SHashObj *dbCache; //key:dbname, value:SCtgDBCache SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; @@ -124,6 +135,8 @@ typedef struct SCtgCacheStat { uint64_t vgMissNum; uint64_t tblHitNum; uint64_t tblMissNum; + uint64_t userHitNum; + uint64_t userMissNum; } SCtgCacheStat; typedef struct SCatalogStat { @@ -169,6 +182,11 @@ typedef struct SCtgRemoveTblMsg { uint64_t dbId; } SCtgRemoveTblMsg; +typedef struct SCtgUpdateUserMsg { + SCatalog* pCtg; + SGetUserAuthRsp userAuth; +} SCtgUpdateUserMsg; + typedef struct SCtgMetaAction { int32_t act; @@ -234,6 +252,8 @@ typedef struct SCtgAction { #define CTG_FLAG_SYS_DB 0x8 #define CTG_FLAG_FORCE_UPDATE 0x10 +#define CTG_FLAG_SET(_flag, _v) ((_flag) |= (_v)) + #define CTG_FLAG_IS_STB(_flag) ((_flag) & CTG_FLAG_STB) #define CTG_FLAG_IS_NOT_STB(_flag) ((_flag) & CTG_FLAG_NOT_STB) #define CTG_FLAG_IS_UNKNOWN_STB(_flag) ((_flag) & CTG_FLAG_UNKNOWN_STB) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 272c370ca3943c65d6163e9d1a169d19c9b80a82..f485f85809318acdef81b7ec1e6f6c7fd34fd66d 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -17,12 +17,14 @@ #include "query.h" #include "tname.h" #include "catalogInt.h" +#include "systable.h" int32_t ctgActUpdateVg(SCtgMetaAction *action); int32_t ctgActUpdateTbl(SCtgMetaAction *action); int32_t ctgActRemoveDB(SCtgMetaAction *action); int32_t ctgActRemoveStb(SCtgMetaAction *action); int32_t ctgActRemoveTbl(SCtgMetaAction *action); +int32_t ctgActUpdateUser(SCtgMetaAction *action); extern SCtgDebug gCTGDebug; SCatalogMgmt gCtgMgmt = {0}; @@ -50,6 +52,11 @@ SCtgAction gCtgAction[CTG_ACT_MAX] = {{ CTG_ACT_REMOVE_TBL, "remove tbMeta", ctgActRemoveTbl + }, + { + CTG_ACT_UPDATE_USER, + "update user", + ctgActUpdateUser } }; @@ -356,6 +363,31 @@ _return: CTG_RET(code); } +int32_t ctgPushUpdateUserMsgInQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq}; + SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + msg->userAuth = *pAuth; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + tFreeSGetUserAuthRsp(pAuth); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { CTG_LOCK(CTG_READ, &dbCache->vgLock); @@ -686,6 +718,43 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEp return TSDB_CODE_SUCCESS; } +int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *user, SGetUserAuthRsp *authRsp) { + char *msg = NULL; + int32_t msgLen = 0; + + ctgDebug("try to get user auth from mnode, user:%s", user); + + int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)]((void *)user, &msg, 0, &msgLen); + if (code) { + ctgError("Build get user auth msg failed, code:%x, db:%s", code, user); + CTG_ERR_RET(code); + } + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_GET_USER_AUTH, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + if (TSDB_CODE_SUCCESS != rpcRsp.code) { + ctgError("error rsp for get user auth, error:%s, user:%s", tstrerror(rpcRsp.code), user); + CTG_ERR_RET(rpcRsp.code); + } + + code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)](authRsp, rpcRsp.pCont, rpcRsp.contLen); + if (code) { + ctgError("Process get user auth rsp failed, code:%x, user:%s", code, user); + CTG_ERR_RET(code); + } + + ctgDebug("Got user auth from mnode, user:%s", user); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { if (NULL == pCtg->dbCache) { @@ -858,6 +927,55 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const char* dbFName, const char return TSDB_CODE_SUCCESS; } +int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { + if (NULL == pCtg->userCache) { + ctgDebug("empty user auth cache, user:%s", user); + goto _return; + } + + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user)); + if (NULL == pUser) { + ctgDebug("user not in cache, user:%s", user); + goto _return; + } + + *inCache = true; + + ctgDebug("Got user from cache, user:%s", user); + CTG_CACHE_STAT_ADD(userHitNum, 1); + + if (pUser->superUser) { + *pass = true; + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &pUser->lock); + if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) { + *pass = true; + CTG_UNLOCK(CTG_READ, &pUser->lock); + return TSDB_CODE_SUCCESS; + } + + if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { + *pass = true; + } + + if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { + *pass = true; + } + + CTG_UNLOCK(CTG_READ, &pUser->lock); + + return TSDB_CODE_SUCCESS; + +_return: + + *inCache = false; + CTG_CACHE_STAT_ADD(userMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; @@ -1951,6 +2069,45 @@ _return: CTG_RET(code); } +int32_t ctgChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { + bool inCache = false; + int32_t code = 0; + + *pass = false; + + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); + + if (inCache) { + return TSDB_CODE_SUCCESS; + } + + SGetUserAuthRsp authRsp = {0}; + CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pRpc, pMgmtEps, user, &authRsp)); + + if (authRsp.superAuth) { + *pass = true; + goto _return; + } + + if (authRsp.createdDbs && taosHashGet(authRsp.createdDbs, dbFName, strlen(dbFName))) { + *pass = true; + goto _return; + } + + if (authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { + *pass = true; + } + + if (authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { + *pass = true; + } + +_return: + + ctgPushUpdateUserMsgInQueue(pCtg, &authRsp, false); + + return TSDB_CODE_SUCCESS; +} int32_t ctgActUpdateVg(SCtgMetaAction *action) { @@ -2120,6 +2277,67 @@ _return: CTG_RET(code); } +int32_t ctgActUpdateUser(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateUserMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + if (NULL == pCtg->userCache) { + pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == pCtg->userCache) { + ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user)); + if (NULL == pUser) { + SCtgUserAuth userAuth = {0}; + + userAuth.version = msg->userAuth.version; + userAuth.superUser = msg->userAuth.superAuth; + userAuth.createdDbs = msg->userAuth.createdDbs; + userAuth.readDbs = msg->userAuth.readDbs; + userAuth.writeDbs = msg->userAuth.writeDbs; + + if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { + ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; + } + + pUser->version = msg->userAuth.version; + + CTG_LOCK(CTG_WRITE, &pUser->lock); + + taosHashCleanup(pUser->createdDbs); + pUser->createdDbs = msg->userAuth.createdDbs; + msg->userAuth.createdDbs = NULL; + + taosHashCleanup(pUser->readDbs); + pUser->readDbs = msg->userAuth.readDbs; + msg->userAuth.readDbs = NULL; + + taosHashCleanup(pUser->writeDbs); + pUser->writeDbs = msg->userAuth.writeDbs; + msg->userAuth.writeDbs = NULL; + + CTG_UNLOCK(CTG_WRITE, &pUser->lock); + +_return: + + + taosHashCleanup(msg->userAuth.createdDbs); + taosHashCleanup(msg->userAuth.readDbs); + taosHashCleanup(msg->userAuth.writeDbs); + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + void* ctgUpdateThreadFunc(void* param) { setThreadName("catalog"); @@ -2835,6 +3053,35 @@ int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); } +int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == users || NULL == num) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + *num = taosHashGetSize(pCtg->userCache); + if (*num > 0) { + *users = taosMemoryCalloc(*num, sizeof(SUserAuthVersion)); + if (NULL == *users) { + ctgError("calloc %d userAuthVersion failed", *num); + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } + } + + uint32_t i = 0; + SCtgUserAuth *pAuth = taosHashIterate(pCtg->userCache, NULL); + while (pAuth != NULL) { + void *key = taosHashGetKey(pAuth, NULL); + strncpy((*users)[i].user, key, sizeof((*users)[i].user)); + (*users)[i].version = pAuth->version; + pAuth = taosHashIterate(pCtg->userCache, pAuth); + } + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); +} + + int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { CTG_API_ENTER(); @@ -2879,6 +3126,31 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == pass) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgChkAuth(pCtg, pRpc, pMgmtEps, user, dbFName, type, pass)); + +_return: + + CTG_API_LEAVE(code); +} + +int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pAuth) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_API_LEAVE(ctgPushUpdateUserMsgInQueue(pCtg, pAuth, false)); +} + void catalogDestroy(void) { qInfo("start to destroy catalog"); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 1045acbe939417ed94a1a13f15c561797702146c..cff0087d6ccc7dbab5f690ffe1cecc27ec4813b7 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -97,24 +97,22 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { SCreateDbReq createReq = {0}; strcpy(createReq.db, "1.db1"); createReq.numOfVgroups = 2; - createReq.cacheBlockSize = 16; - createReq.totalBlocks = 10; + createReq.buffer = -1; + createReq.pageSize = -1; + createReq.pages = -1; createReq.daysPerFile = 10; createReq.daysToKeep0 = 3650; createReq.daysToKeep1 = 3650; createReq.daysToKeep2 = 3650; createReq.minRows = 100; createReq.maxRows = 4096; - createReq.commitTime = 3600; createReq.fsyncPeriod = 3000; createReq.walLevel = 1; createReq.precision = 0; createReq.compression = 2; createReq.replications = 1; createReq.strict = 1; - createReq.update = 0; createReq.cacheLastRow = 0; - createReq.ttl = 1; createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); @@ -254,7 +252,6 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { rspMsg->numOfColumns = ctgTestColNum; rspMsg->precision = 1 + 1; rspMsg->tableType = TSDB_SUPER_TABLE; - rspMsg->update = 1 + 1; rspMsg->sversion = ctgTestSVersion + 1; rspMsg->tversion = ctgTestTVersion + 1; rspMsg->suid = ctgTestSuid + 1; @@ -333,7 +330,6 @@ void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * metaRsp.numOfColumns = ctgTestColNum; metaRsp.precision = 1; metaRsp.tableType = TSDB_NORMAL_TABLE; - metaRsp.update = 1; metaRsp.sversion = ctgTestSVersion; metaRsp.tversion = ctgTestTVersion; metaRsp.suid = 0; @@ -379,7 +375,6 @@ void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg metaRsp.numOfColumns = ctgTestColNum; metaRsp.precision = 1; metaRsp.tableType = TSDB_CHILD_TABLE; - metaRsp.update = 1; metaRsp.sversion = ctgTestSVersion; metaRsp.tversion = ctgTestTVersion; metaRsp.suid = 0x0000000000000002; @@ -426,7 +421,6 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg metaRsp.numOfColumns = ctgTestColNum; metaRsp.precision = 1; metaRsp.tableType = TSDB_SUPER_TABLE; - metaRsp.update = 1; metaRsp.sversion = ctgTestSVersion; metaRsp.tversion = ctgTestTVersion; metaRsp.suid = ctgTestSuid; @@ -475,7 +469,6 @@ void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRp metaRsp.numOfColumns = ctgTestColNum; metaRsp.precision = 1; metaRsp.tableType = TSDB_SUPER_TABLE; - metaRsp.update = 1; metaRsp.sversion = ctgTestSVersion; metaRsp.tversion = ctgTestTVersion; metaRsp.suid = ctgTestSuid + idx; diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index 452804ac91c8a92527b3fc596d8aa631ca184c33..775dee28a4e2528cbf8509cb5955d567b658a5dd 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -56,10 +56,9 @@ extern "C" { #define EXPLAIN_ROWS_FORMAT "rows=%" PRIu64 #define EXPLAIN_COLUMNS_FORMAT "columns=%d" #define EXPLAIN_WIDTH_FORMAT "width=%d" +#define EXPLAIN_TABLE_SCAN_FORMAT "order=[asc|%d desc|%d]" #define EXPLAIN_GROUPS_FORMAT "groups=%d" #define EXPLAIN_WIDTH_FORMAT "width=%d" -#define EXPLAIN_LOOPS_FORMAT "loops=%d" -#define EXPLAIN_REVERSE_FORMAT "reverse=%d" #define EXPLAIN_FUNCTIONS_FORMAT "functions=%d" #define EXPLAIN_EXECINFO_FORMAT "cost=%" PRIu64 "..%" PRIu64 " rows=%" PRIu64 diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index d17a5a59a444776c8465dac0487c6d73a2a29506..2e94ec8d0c3a79f7068580e95f30373aeff6ac5f 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -13,14 +13,13 @@ * along with this program. If not, see . */ -#include "query.h" -#include "plannodes.h" #include "commandInt.h" +#include "plannodes.h" +#include "query.h" int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes); int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level); - void qExplainFreeResNode(SExplainResNode *resNode) { if (NULL == resNode) { return; @@ -28,12 +27,10 @@ void qExplainFreeResNode(SExplainResNode *resNode) { taosMemoryFreeClear(resNode->pExecInfo); - SNode* node = NULL; - FOREACH(node, resNode->pChildren) { - qExplainFreeResNode((SExplainResNode *)node); - } + SNode *node = NULL; + FOREACH(node, resNode->pChildren) { qExplainFreeResNode((SExplainResNode *)node); } nodesClearList(resNode->pChildren); - + taosMemoryFreeClear(resNode); } @@ -59,24 +56,24 @@ void qExplainFreeCtx(SExplainCtx *pCtx) { taosMemoryFreeClear(rsp->subplanInfo); } } - + pIter = taosHashIterate(pCtx->groupHash, pIter); } } - + taosHashCleanup(pCtx->groupHash); taosArrayDestroy(pCtx->rows); taosMemoryFree(pCtx); } int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, double ratio, EExplainMode mode) { - int32_t code = 0; + int32_t code = 0; SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx)); if (NULL == ctx) { qError("calloc SExplainCtx failed"); QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); if (NULL == rows) { qError("taosArrayInit SQueryExplainRowInfo failed"); @@ -95,7 +92,7 @@ int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, d ctx->tbuf = tbuf; ctx->rows = rows; ctx->groupHash = groupHash; - + *pCtx = ctx; return TSDB_CODE_SUCCESS; @@ -110,9 +107,9 @@ _return: } int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNodeList **pChildren) { - int32_t tlen = 0; + int32_t tlen = 0; SNodeList *pPhysiChildren = NULL; - + switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; @@ -120,47 +117,47 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo break; } case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: { STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; pPhysiChildren = pTblScanNode->scan.node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: { SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; pPhysiChildren = pSTblScanNode->scan.node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; pPhysiChildren = pPrjNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ + case QUERY_NODE_PHYSICAL_PLAN_JOIN: { SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; pPhysiChildren = pJoinNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_AGG:{ + case QUERY_NODE_PHYSICAL_PLAN_AGG: { SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; pPhysiChildren = pAggNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: { SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; pPhysiChildren = pExchNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_SORT:{ + case QUERY_NODE_PHYSICAL_PLAN_SORT: { SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; pPhysiChildren = pSortNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; pPhysiChildren = pIntNode->window.node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: { SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode; pPhysiChildren = pSessNode->window.node.pChildren; break; @@ -178,7 +175,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo } } - SNode* node = NULL; + SNode *node = NULL; SExplainResNode *pResNode = NULL; FOREACH(node, pPhysiChildren) { QRY_ERR_RET(qExplainGenerateResNode((SPhysiNode *)node, group, &pResNode)); @@ -195,14 +192,14 @@ int32_t qExplainGenerateResNodeExecInfo(SArray **pExecInfo, SExplainGroup *group return TSDB_CODE_QRY_OUT_OF_MEMORY; } - SExplainRsp *rsp = NULL; + SExplainRsp *rsp = NULL; for (int32_t i = 0; i < group->nodeNum; ++i) { rsp = taosArrayGet(group->nodeExecInfo, i); if (group->physiPlanExecIdx >= rsp->numOfPlans) { qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); return TSDB_CODE_QRY_APP_ERROR; } - + taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); } @@ -217,7 +214,7 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai qError("physical node is NULL"); return TSDB_CODE_QRY_APP_ERROR; } - + SExplainResNode *resNode = taosMemoryCalloc(1, sizeof(SExplainResNode)); if (NULL == resNode) { qError("calloc SPhysiNodeExplainRes failed"); @@ -226,15 +223,15 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai int32_t code = 0; resNode->pNode = pNode; - + if (group->nodeExecInfo) { QRY_ERR_JRET(qExplainGenerateResNodeExecInfo(&resNode->pExecInfo, group)); } - + QRY_ERR_JRET(qExplainGenerateResChildren(pNode, group, &resNode->pChildren)); ++group->physiPlanNum; - + *pResNode = resNode; return TSDB_CODE_SUCCESS; @@ -242,15 +239,15 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai _return: qExplainFreeResNode(resNode); - + QRY_RET(code); } int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { - int32_t tlen = *len; - int32_t nodeNum = taosArrayGetSize(pExecInfo); + int32_t tlen = *len; + int32_t nodeNum = taosArrayGetSize(pExecInfo); SExplainExecInfo maxExecInfo = {0}; - + for (int32_t i = 0; i < nodeNum; ++i) { SExplainExecInfo *execInfo = taosArrayGet(pExecInfo, i); if (execInfo->startupCost > maxExecInfo.startupCost) { @@ -263,20 +260,20 @@ int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { maxExecInfo.numOfRows = execInfo->numOfRows; } } - + EXPLAIN_ROW_APPEND(EXPLAIN_EXECINFO_FORMAT, maxExecInfo.startupCost, maxExecInfo.totalCost, maxExecInfo.numOfRows); *len = tlen; - + return TSDB_CODE_SUCCESS; } int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { - int32_t tlen = 0; - bool gotVerbose = false; - int32_t nodeNum = taosArrayGetSize(pExecInfo); + int32_t tlen = 0; + bool gotVerbose = false; + int32_t nodeNum = taosArrayGetSize(pExecInfo); SExplainExecInfo maxExecInfo = {0}; - + for (int32_t i = 0; i < nodeNum; ++i) { SExplainExecInfo *execInfo = taosArrayGet(pExecInfo, i); if (execInfo->verboseInfo) { @@ -289,11 +286,10 @@ int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t } *len = tlen; - + return TSDB_CODE_SUCCESS; } - int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) { SQueryExplainRowInfo row = {0}; row.buf = taosMemoryMalloc(len); @@ -304,7 +300,7 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t memcpy(row.buf, tbuf, len); row.level = level; - row.len = len; + row.len = len; ctx->dataSize += row.len; if (NULL == taosArrayPush(ctx->rows, &row)) { @@ -316,21 +312,21 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t return TSDB_CODE_SUCCESS; } -static uint8_t getIntervalPrecision(SIntervalPhysiNode* pIntNode) { - return ((SColumnNode*)pIntNode->window.pTspk)->node.resType.precision; +static uint8_t getIntervalPrecision(SIntervalPhysiNode *pIntNode) { + return ((SColumnNode *)pIntNode->window.pTspk)->node.resType.precision; } int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { - int32_t tlen = 0; - bool isVerboseLine = false; - char *tbuf = ctx->tbuf; - bool verbose = ctx->verbose; - SPhysiNode* pNode = pResNode->pNode; + int32_t tlen = 0; + bool isVerboseLine = false; + char *tbuf = ctx->tbuf; + bool verbose = ctx->verbose; + SPhysiNode *pNode = pResNode->pNode; if (NULL == pNode) { qError("pyhsical node in explain res node is NULL"); return TSDB_CODE_QRY_APP_ERROR; } - + switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; @@ -339,7 +335,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pTagScanNode->pScanCols->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->totalRowSize); @@ -350,79 +346,85 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pTagScanNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pTagScanNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - + if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendVerboseExecInfo(pResNode->pExecInfo, tbuf, &tlen)); if (tlen) { EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - } + } } break; } case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: { STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pTblScanNode->scan.pScanCols->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_TABLE_SCAN_FORMAT, pTblScanNode->scanSeq[0], pTblScanNode->scanSeq[1]); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (verbose) { + if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pTblScanNode->scan.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pTblScanNode->scan.node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, + pTblScanNode->scanRange.ekey); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: { SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pSTblScanNode->scan.pScanCols->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (verbose) { + if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pSTblScanNode->scan.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pSTblScanNode->scan.node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); @@ -430,91 +432,96 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pSTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - } break; } - case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pPrjNode->pProjections->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pPrjNode->node.pOutputDataBlockDesc->totalRowSize); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pPrjNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pPrjNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pPrjNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - + if (pPrjNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ + case QUERY_NODE_PHYSICAL_PLAN_JOIN: { SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pJoinNode->pTargets->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pJoinNode->node.pOutputDataBlockDesc->totalRowSize); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pJoinNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pJoinNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pJoinNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pJoinNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); + QRY_ERR_RET( + nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } - case QUERY_NODE_PHYSICAL_PLAN_AGG:{ + case QUERY_NODE_PHYSICAL_PLAN_AGG: { SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pAggNode->pAggFuncs->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->totalRowSize); @@ -522,57 +529,61 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_GROUPS_FORMAT, pAggNode->pGroupKeys->length); } - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pAggNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pAggNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pAggNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: { SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; - SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcGroupId, sizeof(pExchNode->srcGroupId)); + SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcGroupId, sizeof(pExchNode->srcGroupId)); if (NULL == group) { qError("exchange src group %d not in groupHash", pExchNode->srcGroupId); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - + EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, group->nodeNum); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pExchNode->node.pOutputDataBlockDesc->totalRowSize); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pExchNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pExchNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pExchNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pExchNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -581,14 +592,14 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(qExplainAppendGroupResRows(ctx, pExchNode->srcGroupId, level + 1)); break; } - case QUERY_NODE_PHYSICAL_PLAN_SORT:{ + case QUERY_NODE_PHYSICAL_PLAN_SORT: { SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pSortNode->pSortKeys->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSortNode->node.pOutputDataBlockDesc->totalRowSize); @@ -598,29 +609,31 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pSortNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pSortNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSortNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pSortNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pIntNode->window.pFuncs->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->totalRowSize); @@ -629,41 +642,39 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pIntNode->window.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pIntNode->window.node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); uint8_t precision = getIntervalPrecision(pIntNode); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), - pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), - INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), pIntNode->slidingUnit); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), + pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), + pIntNode->slidingUnit); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - - if (pIntNode->pFill) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, getFillModeString(pIntNode->pFill->mode)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } if (pIntNode->window.node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: { SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pSessNode->window.pFuncs->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSessNode->window.node.pOutputDataBlockDesc->totalRowSize); @@ -671,10 +682,10 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pSessNode->window.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pSessNode->window.node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSessNode->window.node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); @@ -685,8 +696,9 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pSessNode->window.node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSessNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pSessNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -701,7 +713,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i return TSDB_CODE_SUCCESS; } - int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); @@ -711,29 +722,28 @@ int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32 int32_t code = 0; QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, ctx, level)); - SNode* pNode = NULL; - FOREACH(pNode, pResNode->pChildren) { - QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, level + 1)); - } + SNode *pNode = NULL; + FOREACH(pNode, pResNode->pChildren) { QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, level + 1)); } return TSDB_CODE_SUCCESS; } int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level) { SExplainResNode *node = NULL; - int32_t code = 0; - SExplainCtx *ctx = (SExplainCtx *)pCtx; + int32_t code = 0; + SExplainCtx *ctx = (SExplainCtx *)pCtx; SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId)); if (NULL == group) { qError("group %d not in groupHash", groupId); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - + QRY_ERR_RET(qExplainGenerateResNode(group->plan->pNode, group, &node)); if ((EXPLAIN_MODE_ANALYZE == ctx->mode) && (group->physiPlanNum != group->physiPlanExecNum)) { - qError("physiPlanNum %d mismatch with physiExecNum %d in group %d", group->physiPlanNum, group->physiPlanExecNum, groupId); + qError("physiPlanNum %d mismatch with physiExecNum %d in group %d", group->physiPlanNum, group->physiPlanExecNum, + groupId); QRY_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -742,21 +752,21 @@ int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level) { _return: qExplainFreeResNode(node); - + QRY_RET(code); } - int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { SExplainCtx *pCtx = (SExplainCtx *)ctx; - int32_t rowNum = taosArrayGetSize(pCtx->rows); + int32_t rowNum = taosArrayGetSize(pCtx->rows); if (rowNum <= 0) { qError("empty explain res rows"); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - + int32_t colNum = 1; - int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; + int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + + sizeof(int32_t) * rowNum + pCtx->dataSize; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); if (NULL == rsp) { qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); @@ -767,13 +777,14 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->numOfRows = htonl(rowNum); // payload length - *(int32_t *)rsp->data = sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; + *(int32_t *)rsp->data = + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; // group id - *(uint64_t*)(rsp->data + sizeof(int32_t)) = 0; + *(uint64_t *)(rsp->data + sizeof(int32_t)) = 0; // column length - int32_t* colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t)); + int32_t *colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t)); // varchar column offset segment int32_t *offset = (int32_t *)((char *)colLength + sizeof(int32_t)); @@ -781,7 +792,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { // varchar data real payload char *data = (char *)(offset + rowNum); - char* start = data; + char *start = data; for (int32_t i = 0; i < rowNum; ++i) { SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); offset[i] = data - start; @@ -799,11 +810,11 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { } int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { - int32_t code = 0; + int32_t code = 0; SNodeListNode *plans = NULL; int32_t taskNum = 0; SExplainGroup *pGroup = NULL; - SExplainCtx *ctx = NULL; + SExplainCtx *ctx = NULL; if (pDag->numOfSubplans <= 0) { qError("invalid subplan num:%d", pDag->numOfSubplans); @@ -816,13 +827,15 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SHashObj *groupHash = taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + SHashObj *groupHash = + taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (NULL == groupHash) { qError("groupHash %d failed", EXPLAIN_MAX_GROUP_NUM); QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - QRY_ERR_JRET(qExplainInitCtx(&ctx, groupHash, pDag->explainInfo.verbose, pDag->explainInfo.ratio, pDag->explainInfo.mode)); + QRY_ERR_JRET( + qExplainInitCtx(&ctx, groupHash, pDag->explainInfo.verbose, pDag->explainInfo.ratio, pDag->explainInfo.mode)); for (int32_t i = 0; i < levelNum; ++i) { plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); @@ -849,7 +862,7 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { SExplainGroup group = {0}; group.nodeNum = 1; group.plan = plan; - + if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { qError("taosHashPut to explainGroupHash failed, taskIdx:%d", n); QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -871,7 +884,7 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { *pCtx = ctx; return TSDB_CODE_SUCCESS; - + _return: qExplainFreeCtx(ctx); @@ -885,7 +898,7 @@ int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) { } int32_t tlen = 0; - char *tbuf = pCtx->tbuf; + char *tbuf = pCtx->tbuf; EXPLAIN_SUM_ROW_NEW(EXPLAIN_RATIO_TIME_FORMAT, pCtx->ratio); EXPLAIN_SUM_ROW_END(); @@ -910,11 +923,11 @@ int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { return TSDB_CODE_SUCCESS; } -int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp) { +int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp) { SExplainResNode *node = NULL; - int32_t code = 0; - bool groupDone = false; - SExplainCtx *ctx = (SExplainCtx *)pCtx; + int32_t code = 0; + bool groupDone = false; + SExplainCtx *ctx = (SExplainCtx *)pCtx; SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId)); if (NULL == group) { @@ -930,30 +943,32 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, i qError("taosArrayInit %d explainExecInfo failed", group->nodeNum); taosMemoryFreeClear(pRspMsg->subplanInfo); taosWUnLockLatch(&group->lock); - + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } group->physiPlanExecNum = pRspMsg->numOfPlans; } else if (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum) { - qError("group execInfo already full, size:%d, nodeNum:%d", (int32_t)taosArrayGetSize(group->nodeExecInfo), group->nodeNum); + qError("group execInfo already full, size:%d, nodeNum:%d", (int32_t)taosArrayGetSize(group->nodeExecInfo), + group->nodeNum); taosMemoryFreeClear(pRspMsg->subplanInfo); taosWUnLockLatch(&group->lock); - + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } if (group->physiPlanExecNum != pRspMsg->numOfPlans) { - qError("physiPlanExecNum %d mismatch with others %d in group %d", pRspMsg->numOfPlans, group->physiPlanExecNum, groupId); + qError("physiPlanExecNum %d mismatch with others %d in group %d", pRspMsg->numOfPlans, group->physiPlanExecNum, + groupId); taosMemoryFreeClear(pRspMsg->subplanInfo); taosWUnLockLatch(&group->lock); - + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } taosArrayPush(group->nodeExecInfo, pRspMsg); groupDone = (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum); - + taosWUnLockLatch(&group->lock); if (groupDone && (taosHashGetSize(pCtx->groupHash) == atomic_add_fetch_32(&pCtx->groupDoneNum, 1))) { @@ -968,14 +983,13 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, i return TSDB_CODE_SUCCESS; } - int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { - int32_t code = 0; + int32_t code = 0; SExplainCtx *pCtx = NULL; QRY_ERR_RET(qExplainPrepareCtx(pDag, &pCtx)); QRY_ERR_JRET(qExplainGenerateRsp(pCtx, pRsp)); - + _return: qExplainFreeCtx(pCtx); QRY_RET(code); @@ -983,7 +997,7 @@ _return: int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs) { QRY_ERR_RET(qExplainPrepareCtx(pDag, pCtx)); - + (*pCtx)->reqStartTs = startTs; (*pCtx)->jobStartTs = taosGetTimestampUs(); @@ -993,7 +1007,7 @@ int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs) int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { int32_t code = 0; pCtx->jobDoneTs = taosGetTimestampUs(); - + atomic_store_8((int8_t *)&pCtx->execDone, true); if (taosHashGetSize(pCtx->groupHash) == atomic_load_32(&pCtx->groupDoneNum)) { @@ -1005,6 +1019,3 @@ int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { return TSDB_CODE_SUCCESS; } - - - diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f0cb1c1107e8f473764472b44b25b0299a57dbf8..9f834fd659000e66eaa97208e9a0f30fc861ba9d 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -73,7 +73,7 @@ typedef struct SResKeyPos { } SResKeyPos; typedef struct SResultRowInfo { - SResultRowPosition *pPosition; + SResultRowPosition *pPosition; // todo remove this int32_t size; // number of result set int32_t capacity; // max capacity SResultRowPosition cur; diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 59db66beccf6d911616fe5c5c853490bcbb86c34..f2f0bc20551336b8bfa14414697b9b5299ea6a49 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -20,6 +20,12 @@ extern "C" { #endif +typedef struct { + char* pData; + bool isNull; + int16_t type; + int32_t bytes; +} SGroupKeys, SStateKeys; #ifdef __cplusplus } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 7a6432a06cadf7195448f730c3b7dd66e285a0f5..3b5d0c209f718313b46eac49b9660a8132f83daa 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -40,6 +40,7 @@ extern "C" { #include "tpagedbuf.h" #include "vnode.h" +#include "executorInt.h" typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); @@ -86,9 +87,7 @@ typedef struct SResultInfo { // TODO refactor typedef struct STableQueryInfo { TSKEY lastKey; // last check ts, todo remove it later SResultRowPosition pos; // current active time window -// int32_t groupIndex; // group id in table list // SVariant tag; -// SResultRowInfo resInfo; // result info } STableQueryInfo; typedef enum { @@ -118,6 +117,17 @@ typedef struct SLimit { int64_t offset; } SLimit; +typedef struct SFileBlockLoadRecorder { + uint64_t totalRows; + uint64_t totalCheckedRows; + uint32_t totalBlocks; + uint32_t loadBlocks; + uint32_t loadBlockStatis; + uint32_t skipBlocks; + uint32_t filterOutBlocks; + uint64_t elapsedTime; +} SFileBlockLoadRecorder; + typedef struct STaskCostInfo { int64_t created; int64_t start; @@ -131,14 +141,10 @@ typedef struct STaskCostInfo { uint64_t loadDataInCacheSize; uint64_t loadDataTime; - uint64_t totalRows; - uint64_t totalCheckedRows; - uint32_t totalBlocks; - uint32_t loadBlocks; - uint32_t loadBlockStatis; - uint32_t skipBlocks; - uint32_t filterOutBlocks; + + SFileBlockLoadRecorder* pRecoder; uint64_t elapsedTime; + uint64_t firstStageMergeTime; uint64_t winInfoSize; uint64_t tableInfoSize; @@ -196,7 +202,7 @@ typedef bool (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, struct SAggS struct SOptrBasicInfo* pInfo, char* result, int32_t length); typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* pOptr); -typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr, bool* newgroup); +typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr); typedef void (*__optr_close_fn_t)(void* param, int32_t num); typedef int32_t (*__optr_get_explain_fn_t)(struct SOperatorInfo* pOptr, void** pOptrExplain); @@ -267,9 +273,9 @@ typedef struct SOperatorFpSet { typedef struct SOperatorInfo { uint8_t operatorType; - bool blockingOptr; // block operator or not + bool blocking; // block operator or not uint8_t status; // denote if current operator is completed - int32_t numOfOutput; // number of columns of the current operator results + int32_t numOfExprs; // number of columns of the current operator results char* name; // name, used to show the query execution plan void* info; // extension attribution SExprInfo* pExpr; @@ -332,17 +338,14 @@ typedef struct SScanInfo { typedef struct STableScanInfo { void* dataReader; - - int32_t numOfBlocks; // extract basic running information. - int32_t numOfSkipped; - int32_t numOfBlockStatis; + SFileBlockLoadRecorder readRecorder; int64_t numOfRows; int64_t elapsedTime; - int32_t prevGroupId; // previous table group id +// int32_t prevGroupId; // previous table group id SScanInfo scanInfo; - int32_t current; - SNode* pFilterNode; // filter operator info - SqlFunctionCtx* pCtx; // next operator query context + int32_t scanTimes; + SNode* pFilterNode; // filter info, which is push down by optimizer + SqlFunctionCtx* pCtx; // which belongs to the direct upstream operator operator query context SResultRowInfo* pResultRowInfo; int32_t* rowCellInfoOffset; SExprInfo* pExpr; @@ -358,11 +361,12 @@ typedef struct STableScanInfo { } STableScanInfo; typedef struct STagScanInfo { - SColumnInfo *pCols; - SSDataBlock *pRes; - int32_t totalTables; - int32_t curPos; - void *pReader; + SColumnInfo *pCols; + SSDataBlock *pRes; + SArray *pColMatchInfo; + int32_t curPos; + SReadHandle readHandle; + STableGroupInfo *pTableGroups; } STagScanInfo; typedef struct SStreamBlockScanInfo { @@ -375,6 +379,7 @@ typedef struct SStreamBlockScanInfo { uint64_t numOfExec; // execution times void* readerHandle; // stream block reader handle SArray* pColMatchInfo; // + SNode* pCondition; } SStreamBlockScanInfo; typedef struct SSysTableScanInfo { @@ -395,7 +400,6 @@ typedef struct SSysTableScanInfo { SArray* scanCols; // SArray scan column id list SName name; SSDataBlock* pRes; - int32_t capacity; int64_t numOfBlocks; // extract basic running information. SLoadRemoteDataInfo loadInfo; } SSysTableScanInfo; @@ -410,8 +414,6 @@ typedef struct SOptrBasicInfo { // TODO move the resultrowsiz together with SOptrBasicInfo:rowCellInfoOffset typedef struct SAggSupporter { SHashObj* pResultRowHashTable; // quick locate the window object for each result -// SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not -// SArray* pResultRowArrayList; // The array list that contains the Result rows char* keyBuf; // window key buffer SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row @@ -423,7 +425,7 @@ typedef struct STimeWindowSupp { SColumnInfoData timeWindowData; // query time window info for scalar function execution. } STimeWindowAggSupp; -typedef struct STableIntervalOperatorInfo { +typedef struct SIntervalAggOperatorInfo { SOptrBasicInfo binfo; // basic info SGroupResInfo groupResInfo; // multiple results build supporter SInterval interval; // interval info @@ -438,7 +440,7 @@ typedef struct STableIntervalOperatorInfo { SArray* pUpdatedWindow; // updated time window due to the input data block from the downstream operator. STimeWindowAggSupp twAggSup; struct SFillInfo* pFillInfo; // fill info -} STableIntervalOperatorInfo; +} SIntervalAggOperatorInfo; typedef struct SAggOperatorInfo { SOptrBasicInfo binfo; @@ -477,16 +479,8 @@ typedef struct SFillOperatorInfo { void** p; SSDataBlock* existNewGroupBlock; bool multigroupResult; - SInterval intervalInfo; } SFillOperatorInfo; -typedef struct { - char* pData; - bool isNull; - int16_t type; - int32_t bytes; -} SGroupKeys, SStateKeys; - typedef struct SGroupbyOperatorInfo { SOptrBasicInfo binfo; SArray* pGroupCols; // group by columns, SArray @@ -539,6 +533,7 @@ typedef struct SSessionAggOperatorInfo { SWindowRowsSup winSup; bool reptScan; // next round scan int64_t gap; // session window gap + int32_t tsSlotId; // primary timestamp slot id STimeWindowAggSupp twAggSup; } SSessionAggOperatorInfo; @@ -556,6 +551,7 @@ typedef struct SStateWindowOperatorInfo { int32_t colIndex; // start row index bool hasKey; SStateKeys stateKey; + int32_t tsSlotId; // primary timestamp column slot id STimeWindowAggSupp twAggSup; // bool reptScan; } SStateWindowOperatorInfo; @@ -578,13 +574,12 @@ typedef struct SSortedMergeOperatorInfo { } SSortedMergeOperatorInfo; typedef struct SSortOperatorInfo { + SOptrBasicInfo binfo; uint32_t sortBufSize; // max buffer size for in-memory sort - SSDataBlock* pDataBlock; SArray* pSortInfo; SSortHandle* pSortHandle; - SArray* inputSlotMap; // for index map from table scan output + SArray* pColMatchInfo; // for index map from table scan output int32_t bufPageSize; - int32_t numOfRowsInRes; // TODO extact struct int64_t startTs; // sort start time @@ -612,6 +607,9 @@ typedef struct SJoinOperatorInfo { SNode *pOnCondition; } SJoinOperatorInfo; +#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) +#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) + SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, __optr_decode_fn_t decode, __optr_get_explain_fn_t explain); @@ -622,8 +620,8 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, size_t keyBufSize, const char* pkey); void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); -void doBuildResultDatablock(SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, - SDiskbasedBuf* pBuf, int32_t* rowCellOffset, SqlFunctionCtx* pCtx); +void doBuildResultDatablock(SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf); + void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, @@ -640,6 +638,20 @@ int32_t getTableScanOrder(SOperatorInfo* pOperator); void doSetOperatorCompleted(SOperatorInfo* pOperator); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset); +void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols); +void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); +void cleanupAggSup(SAggSupporter* pAggSup); +void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); +void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); + +SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo); +SSDataBlock* loadNextDataBlock(void* param); + +void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); + +SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, + char* pData, int16_t bytes, bool masterscan, uint64_t groupId, + SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup); SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); @@ -650,7 +662,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SLimit* pLimit, SLimit* pSlimit, SExecTaskInfo* pTaskInfo); -SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pIndexMap, SExecTaskInfo* pTaskInfo); +SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols, + SArray* pIndexMap, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); @@ -661,32 +674,35 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); + SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, - SArray* pTableIdList, SExecTaskInfo* pTaskInfo); + SArray* pTableIdList, SExecTaskInfo* pTaskInfo, SNode* pConditions); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SInterval* pInterval, SSDataBlock* pResBlock, int32_t fillType, char* fillVal, + SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); + SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, int32_t tsSlotId, SExecTaskInfo* pTaskInfo); SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); + SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, SSDataBlock* pResBlock, SArray* pColMatchInfo, STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); #if 0 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); @@ -694,15 +710,14 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim SExprInfo* pExpr, int32_t numOfOutput); #endif -void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, +int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t numOfOutput, SArray* pPseudoList); void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol); -void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput); void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); -STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win); +STableQueryInfo* createTableQueryInfo(void* buf, STimeWindow win); bool isTaskKilled(SExecTaskInfo* pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index 26d066d9a9f9fbe484b51d5802341b55527211bd..a1f45fd66527bb3c904c1eb8bbeef258162bfc1f 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -27,13 +27,12 @@ extern "C" { struct SSDataBlock; typedef struct SFillColInfo { -// STColumn col; // column info - SResSchema col; - int16_t functionId; // sql function id - int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN - int16_t tagIndex; // index of current tag in SFillTagColInfo array list - int32_t offset; - union {int64_t i; double d;} val; + SExprInfo *pExpr; +// SResSchema schema; +// int16_t functionId; // sql function id + int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN + int16_t tagIndex; // index of current tag in SFillTagColInfo array list + SVariant fillVal; } SFillColInfo; typedef struct { @@ -56,9 +55,10 @@ typedef struct SFillInfo { int32_t numOfCols; // number of columns, including the tags columns int32_t rowSize; // size of each row SInterval interval; - char * prevValues; // previous row of data, to generate the interpolation results - char * nextValues; // next row of data - char** pData; // original result data block involved in filling data + + SArray *prev; + SArray *next; + SSDataBlock *pSrcBlock; int32_t alloc; // data buffer size in rows SFillColInfo* pFillCol; // column info for fill operations @@ -72,7 +72,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); -struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SValueNode* val); +struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SNodeListNode* val); bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, @@ -80,7 +80,7 @@ SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int3 struct SFillColInfo* pCol, const char* id); void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); -int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity); +int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity); int64_t getFillInfoStart(struct SFillInfo *pFillInfo); diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index c584df05dd36c98d354149172bd35ef439334407..d74628a72fb4723d1837a0547574da414253bef6 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -89,7 +89,7 @@ int32_t tsortClose(SSortHandle* pHandle); * * @return */ -int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fp); +int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fetchFp, void (*fp)(SSDataBlock*, void*), void* param); /** * @@ -117,18 +117,25 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle); /** * * @param pHandle - * @param colIndex + * @param colId * @return */ -bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex); +bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId); /** * * @param pHandle - * @param colIndex + * @param colId * @return */ -void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex); +void* tsortGetValue(STupleHandle* pVHandle, int32_t colId); + +/** + * + * @param pSortHandle + * @return + */ +SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle); #ifdef __cplusplus } diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index a5ce01dba28c07b0354d6e69c595048da511cd99..a217701471c07b1b2b86b25de7bc89ac9f400a45 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -31,6 +31,7 @@ typedef struct SDataDispatchBuf { typedef struct SDataCacheEntry { int32_t dataLen; int32_t numOfRows; + int32_t numOfCols; int8_t compressed; char data[]; } SDataCacheEntry; @@ -76,6 +77,7 @@ static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputDat SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; pEntry->compressed = (int8_t)needCompress(pInput->pData, numOfCols); pEntry->numOfRows = pInput->pData->info.rows; + pEntry->numOfCols = pInput->pData->info.numOfCols; pEntry->dataLen = 0; pBuf->useSize = sizeof(SRetrieveTableRsp); @@ -169,6 +171,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); pOutput->numOfRows = pEntry->numOfRows; + pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; taosMemoryFreeClear(pDispatcher->nextOutput.pData); // todo persistent pOutput->bufStatus = updateStatus(pDispatcher); diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 516afe5553e3a4d2827d162f80a7f18118147472..77057446946fa6cc99c6d75cd658642562bf4a07 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -154,14 +154,12 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); - bool newgroup = false; publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); - int64_t st = 0; - - st = taosGetTimestampUs(); - *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot, &newgroup); + int64_t st = taosGetTimestampUs(); + *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot); uint64_t el = (taosGetTimestampUs() - st); + pTaskInfo->cost.elapsedTime += el; publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 943a016c2741c885d8c7b4bdb40a7967dcda67b1..943d4b278396ccf2490fddd779864be4a3e026f4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -41,8 +41,6 @@ #define SET_MAIN_SCAN_FLAG(runtime) ((runtime)->scanFlag = MAIN_SCAN) #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) -#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) - #define SDATA_BLOCK_INITIALIZER \ (SDataBlockInfo) { {0}, 0 } @@ -54,11 +52,6 @@ enum { TS_JOIN_TAG_NOT_EQUALS = 2, }; -typedef enum SResultTsInterpType { - RESULT_ROW_START_INTERP = 1, - RESULT_ROW_END_INTERP = 2, -} SResultTsInterpType; - #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = taosRand(); @@ -97,48 +90,6 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { #define GET_NUM_OF_TABLEGROUP(q) taosArrayGetSize((q)->tableqinfoGroupInfo.pGroupList) #define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) -#define TSKEY_MAX_ADD(a, b) \ - do { \ - if (a < 0) { \ - a = a + b; \ - break; \ - } \ - if (sizeof(a) == sizeof(int32_t)) { \ - if ((b) > 0 && ((b) >= INT32_MAX - (a))) { \ - a = INT32_MAX; \ - } else { \ - a = a + b; \ - } \ - } else { \ - if ((b) > 0 && ((b) >= INT64_MAX - (a))) { \ - a = INT64_MAX; \ - } else { \ - a = a + b; \ - } \ - } \ - } while (0) - -#define TSKEY_MIN_SUB(a, b) \ - do { \ - if (a >= 0) { \ - a = a + b; \ - break; \ - } \ - if (sizeof(a) == sizeof(int32_t)) { \ - if ((b) < 0 && ((b) <= INT32_MIN - (a))) { \ - a = INT32_MIN; \ - } else { \ - a = a + b; \ - } \ - } else { \ - if ((b) < 0 && ((b) <= INT64_MIN - (a))) { \ - a = INT64_MIN; \ - } else { \ - a = a + b; \ - } \ - } \ - } while (0) - int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; } static int32_t getExprFunctionId(SExprInfo* pExprInfo) { @@ -146,39 +97,6 @@ static int32_t getExprFunctionId(SExprInfo* pExprInfo) { return 0; } -static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); - - tw->ekey -= 1; -} - static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); static bool functionNeedToExecute(SqlFunctionCtx* pCtx); @@ -190,19 +108,13 @@ static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutpu static int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo* pTableQueryInfo); static void releaseQueryBuf(size_t numOfTables); -static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -// static SQueryTableDataCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win); -static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo); static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr); -static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput); static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); -static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); -static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); static void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput); @@ -218,17 +130,15 @@ void doSetOperatorCompleted(SOperatorInfo* pOperator) { } } -#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) -#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) - int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { OPTR_SET_OPENED(pOperator); + pOperator->cost.openCost = 0; return TSDB_CODE_SUCCESS; } SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, - __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, - __optr_decode_fn_t decode, __optr_get_explain_fn_t explain) { + __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, + __optr_decode_fn_t decode, __optr_get_explain_fn_t explain) { SOperatorFpSet fpSet = { ._openFn = openFn, .getNextFn = nextFn, @@ -292,9 +202,9 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData idata = {{0}}; SSlotDescNode* pDescNode = nodesListGetNode(pNode->pSlots, i); - if (!pDescNode->output) { - continue; - } +// if (!pDescNode->output) { // todo disable it temporarily +// continue; +// } idata.info.type = pDescNode->dataType.type; idata.info.bytes = pDescNode->dataType.bytes; @@ -440,9 +350,9 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, * | 8 bytes | actual length | * +----------+---------------+ */ -static SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, int64_t uid, - char* pData, int16_t bytes, bool masterscan, uint64_t groupId, - SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) { +SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, + int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, + bool isIntervalQuery, SAggSupporter* pSup) { SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = @@ -463,9 +373,9 @@ static SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowI } } - // 1. close current opened time window + // 1. close current opened time window if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId && - pResult->offset != pResultRowInfo->cur.offset))) { + pResult->offset != pResultRowInfo->cur.offset))) { // todo extract function SResultRowPosition pos = pResultRowInfo->cur; SFilePage* pPage = getBufPage(pResultBuf, pos.pageId); @@ -482,7 +392,8 @@ static SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowI // add a new result set for a new group SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; - taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, sizeof(SResultRowPosition)); + taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, + sizeof(SResultRowPosition)); } // 2. set the new time window to be the new active time window @@ -498,61 +409,6 @@ static SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowI return pResult; } -static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, - bool ascQuery) { - if (ascQuery) { - getAlignQueryTimeWindow(pInterval, precision, ts, w); - } else { - // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - getAlignQueryTimeWindow(pInterval, precision, ts, w); - - int64_t key = w->skey; - while (key < ts) { // moving towards end - key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision); - if (key >= ts) { - break; - } - - w->skey = key; - } - } -} - -// get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, - SInterval* pInterval, int32_t precision, STimeWindow* win) { - STimeWindow w = {0}; - - if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value - getInitialStartTimeWindow(pInterval, precision, ts, &w, true); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win; - } - - if (w.skey > ts || w.ekey < ts) { - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - w.skey = taosTimeTruncate(ts, pInterval, precision); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - int64_t st = w.skey; - - if (st > ts) { - st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - int64_t et = st + pInterval->interval - 1; - if (et < ts) { - st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - w.skey = st; - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } - } - return w; -} - // get the correct time window according to the handled timestamp static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t ts, STaskAttr* pQueryAttr) { STimeWindow w = {0}; @@ -637,75 +493,6 @@ static bool chkWindowOutputBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE, masterscan, groupId); } -static void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, - int32_t* rowCellInfoOffset); - -static int32_t setResultOutputBufByKey_rv(SResultRowInfo* pResultRowInfo, int64_t id, STimeWindow* win, bool masterscan, - SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, - int32_t numOfOutput, int32_t* rowCellInfoOffset, SAggSupporter* pAggSup, - SExecTaskInfo* pTaskInfo) { - assert(win->skey <= win->ekey); - SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, id, (char*)&win->skey, - TSDB_KEYSIZE, masterscan, tableGroupId, pTaskInfo, true, pAggSup); - - if (pResultRow == NULL) { - *pResult = NULL; - return TSDB_CODE_SUCCESS; - } - - // set time window for current result - pResultRow->win = (*win); - *pResult = pResultRow; - setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); - return TSDB_CODE_SUCCESS; -} - -static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { - assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); - if (type == RESULT_ROW_START_INTERP) { - pResult->startInterp = true; - } else { - pResult->endInterp = true; - } -} - -static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { - assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); - if (type == RESULT_ROW_START_INTERP) { - return pResult->startInterp == true; - } else { - return pResult->endInterp == true; - } -} - -static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, - int16_t pos, int16_t order, int64_t* pData) { - int32_t forwardStep = 0; - - if (order == TSDB_ORDER_ASC) { - int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order); - if (end >= 0) { - forwardStep = end; - - if (pData[end + pos] == ekey) { - forwardStep += 1; - } - } - } else { - int32_t end = searchFn((char*)pData, pos + 1, ekey, order); - if (end >= 0) { - forwardStep = pos - end; - - if (pData[end] == ekey) { - forwardStep += 1; - } - } - } - - assert(forwardStep >= 0); - return forwardStep; -} - static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) { int64_t skey = TSKEY_INITIAL_VAL; @@ -775,46 +562,8 @@ static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey // } //} -static int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, - TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, - int32_t order) { - assert(startPos >= 0 && startPos < pDataBlockInfo->rows); - - int32_t num = -1; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - - if (order == TSDB_ORDER_ASC) { - if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) { - num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); - if (item != NULL) { - item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; - } - } else { - num = pDataBlockInfo->rows - startPos; - if (item != NULL) { - item->lastKey = pDataBlockInfo->window.ekey + step; - } - } - } else { // desc - if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) { - num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); - if (item != NULL) { - item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; - } - } else { - num = startPos + 1; - if (item != NULL) { - item->lastKey = pDataBlockInfo->window.skey + step; - } - } - } - - assert(num >= 0); - return num; -} - // query_range_start, query_range_end, window_duration, window_start, window_end -static void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow) { +void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow) { pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP; pColData->info.bytes = sizeof(int64_t); @@ -828,16 +577,6 @@ static void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQuer colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } -static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) { - int64_t* ts = (int64_t*)pColData->pData; - int32_t delta = includeEndpoint ? 1 : 0; - - int64_t duration = pWin->ekey - pWin->skey + delta; - ts[2] = duration; // set the duration - ts[3] = pWin->skey; // window start key - ts[4] = pWin->ekey + delta; // window end key -} - void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { @@ -889,77 +628,6 @@ void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* } } -static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, - TSKEY* primaryKeys, int32_t prevPosition, STableIntervalOperatorInfo* pInfo) { - int32_t order = pInfo->order; - bool ascQuery = (order == TSDB_ORDER_ASC); - - int32_t precision = pInterval->precision; - getNextTimeWindow(pInterval, precision, order, pNext); - - // next time window is not in current block - if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) || - (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) { - return -1; - } - - TSKEY startKey = ascQuery ? pNext->skey : pNext->ekey; - int32_t startPos = 0; - - // tumbling time window query, a special case of sliding time window query - if (pInterval->sliding == pInterval->interval && prevPosition != -1) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - startPos = prevPosition + factor; - } else { - if (startKey <= pDataBlockInfo->window.skey && ascQuery) { - startPos = 0; - } else if (startKey >= pDataBlockInfo->window.ekey && !ascQuery) { - startPos = pDataBlockInfo->rows - 1; - } else { - startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, startKey, order); - } - } - - /* interp query with fill should not skip time window */ - // if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) { - // return startPos; - // } - - /* - * This time window does not cover any data, try next time window, - * this case may happen when the time window is too small - */ - if (primaryKeys == NULL) { - if (ascQuery) { - assert(pDataBlockInfo->window.skey <= pNext->ekey); - } else { - assert(pDataBlockInfo->window.ekey >= pNext->skey); - } - } else { - if (ascQuery && primaryKeys[startPos] > pNext->ekey) { - TSKEY next = primaryKeys[startPos]; - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); - pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - pNext->skey = pNext->ekey - pInterval->interval + 1; - } - } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) { - TSKEY next = primaryKeys[startPos]; - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); - pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - pNext->ekey = pNext->skey + pInterval->interval - 1; - } - } - } - - return startPos; -} - static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr* pQueryAttr, STimeWindow* pWindow) { TSKEY ekey = -1; int32_t order = TSDB_ORDER_ASC; @@ -978,47 +646,12 @@ static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr* pQueryAttr, STimeWindow* p return ekey; } -static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { - if (type == RESULT_ROW_START_INTERP) { - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].start.key = INT64_MIN; - } - } else { - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].end.key = INT64_MIN; - } - } -} - -static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowIndex, int32_t numOfCols) { - if (pDataBlock == NULL) { - return; - } - - for (int32_t k = 0; k < numOfCols; ++k) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, k); - memcpy(pRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); - } -} - -static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows, bool ascQuery) { - TSKEY ts = TSKEY_INITIAL_VAL; - if (tsCols == NULL) { - ts = ascQuery ? win->skey : win->ekey; - } else { - int32_t offset = ascQuery ? 0 : rows - 1; - ts = tsCols[offset]; - } - - return ts; -} - static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol); static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { - for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { + for (int32_t i = 0; i < pOperator->numOfExprs; ++i) { pCtx[i].order = order; pCtx[i].size = pBlock->info.rows; setBlockStatisInfo(&pCtx[i], &pOperator->pExpr[i], pBlock); @@ -1034,8 +667,8 @@ void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlo } } -static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, - int32_t paramIndex, int32_t numOfRows) { +static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t paramIndex, + int32_t numOfRows) { SColumnInfoData* pColInfo = NULL; if (pInput->pData[paramIndex] == NULL) { pColInfo = taosMemoryCalloc(1, sizeof(SColumnInfoData)); @@ -1066,9 +699,9 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc colDataAppendDouble(pColInfo, i, &v); } } else if (type == TSDB_DATA_TYPE_VARCHAR) { - char *tmp = taosMemoryMalloc(pFuncParam->param.nLen + VARSTR_HEADER_SIZE); + char* tmp = taosMemoryMalloc(pFuncParam->param.nLen + VARSTR_HEADER_SIZE); STR_WITH_SIZE_TO_VARSTR(tmp, pFuncParam->param.pz, pFuncParam->param.nLen); - for(int32_t i = 0; i < numOfRows; ++i) { + for (int32_t i = 0; i < numOfRows; ++i) { colDataAppend(pColInfo, i, tmp, false); } } @@ -1080,10 +713,10 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt bool createDummyCol) { int32_t code = TSDB_CODE_SUCCESS; - for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { - pCtx[i].order = order; - pCtx[i].size = pBlock->info.rows; - pCtx[i].pSrcBlock = pBlock; + for (int32_t i = 0; i < pOperator->numOfExprs; ++i) { + pCtx[i].order = order; + pCtx[i].size = pBlock->info.rows; + pCtx[i].pSrcBlock = pBlock; pCtx[i].currentStage = MAIN_SCAN; SInputColumnInfoData* pInput = &pCtx[i].input; @@ -1100,7 +733,10 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pInput->numOfRows = pBlock->info.rows; pInput->startRowIndex = 0; - pInput->pPTS = taosArrayGet(pBlock->pDataBlock, 0); // todo set the correct timestamp column + // the last parameter is the timestamp column + if (fmIsTimelineFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) { + pInput->pPTS = pInput->pData[j]; + } ASSERT(pInput->pData[j] != NULL); } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { // todo avoid case: top(k, 12), 12 is the value parameter. @@ -1162,7 +798,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt } static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunctionCtx* pCtx) { - for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { + for (int32_t k = 0; k < pOperator->numOfExprs; ++k) { if (functionNeedToExecute(&pCtx[k])) { pCtx[k].startTs = startTs; // this can be set during create the struct pCtx[k].fpSet.process(&pCtx[k]); @@ -1177,8 +813,8 @@ static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, S } } -void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, - int32_t numOfOutput, SArray* pPseudoList) { +int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, + int32_t numOfOutput, SArray* pPseudoList) { setPseudoOutputColInfo(pResult, pCtx, pPseudoList); pResult->info.groupId = pSrcBlock->info.groupId; @@ -1195,7 +831,7 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) { // it is a project query SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId); if (pResult->info.rows > 0 && !createNewColModel) { - colDataMergeCol(pColInfoData, pResult->info.rows, pfCtx->input.pData[0], pfCtx->input.numOfRows); + colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0], pfCtx->input.numOfRows); } else { colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows); } @@ -1217,13 +853,13 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* taosArrayPush(pBlockList, &pSrcBlock); SColumnInfoData* pResColData = taosArrayGet(pResult->pDataBlock, outputSlotId); - SColumnInfoData idata = {.info = pResColData->info}; + SColumnInfoData idata = {.info = pResColData->info, .hasNull = true}; SScalarParam dest = {.columnData = &idata}; scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; - colDataMergeCol(pResColData, startOffset, &idata, dest.numOfRows); + colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); numOfRows = dest.numOfRows; taosArrayDestroy(pBlockList); @@ -1233,9 +869,6 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* if (fmIsPseudoColumnFunc(pfCtx->functionId)) { // do nothing } else if (fmIsNonstandardSQLFunc(pfCtx->functionId)) { - // todo set the correct timestamp column - pfCtx->input.pPTS = taosArrayGet(pSrcBlock->pDataBlock, 1); - SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[k]); pfCtx->fpSet.init(&pCtx[k], pResInfo); @@ -1254,13 +887,17 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* taosArrayPush(pBlockList, &pSrcBlock); SColumnInfoData* pResColData = taosArrayGet(pResult->pDataBlock, outputSlotId); - SColumnInfoData idata = {.info = pResColData->info}; + SColumnInfoData idata = {.info = pResColData->info, .hasNull = true}; SScalarParam dest = {.columnData = &idata}; - scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest); + int32_t code = scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pBlockList); + return code; + } int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; - colDataMergeCol(pResColData, startOffset, &idata, dest.numOfRows); + colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); numOfRows = dest.numOfRows; taosArrayDestroy(pBlockList); @@ -1273,510 +910,106 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* if (!createNewColModel) { pResult->info.rows += numOfRows; } -} - -void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, - int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { - SExprInfo* pExpr = pOperator->pExpr; - - SqlFunctionCtx* pCtx = pInfo->pCtx; - - for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { - int32_t functionId = pCtx[k].functionId; - if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { - pCtx[k].start.key = INT64_MIN; - continue; - } - - SColIndex* pColIndex = NULL /*&pExpr[k].base.colInfo*/; - int16_t index = pColIndex->colIndex; - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, index); - - // assert(pColInfo->info.colId == pColIndex->info.colId && curTs != windowKey); - double v1 = 0, v2 = 0, v = 0; - - if (prevRowIndex == -1) { - // GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pRuntimeEnv->prevRow[index]); - } else { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); - } - - GET_TYPED_DATA(v2, double, pColInfo->info.type, (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes); - - if (functionId == FUNCTION_INTERP) { - if (type == RESULT_ROW_START_INTERP) { - pCtx[k].start.key = prevTs; - pCtx[k].start.val = v1; - - pCtx[k].end.key = curTs; - pCtx[k].end.val = v2; - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - if (prevRowIndex == -1) { - // pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index]; - } else { - pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes; - } - pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes; - } - } - } else if (functionId == FUNCTION_TWA) { - SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; - SPoint point2 = (SPoint){.key = curTs, .val = &v2}; - SPoint point = (SPoint){.key = windowKey, .val = &v}; - - taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); - - if (type == RESULT_ROW_START_INTERP) { - pCtx[k].start.key = point.key; - pCtx[k].start.val = v; - } else { - pCtx[k].end.key = point.key; - pCtx[k].end.val = v; - } - } - } + return TSDB_CODE_SUCCESS; } -static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos, - int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, - STimeWindow* win) { - bool ascQuery = true; - TSKEY curTs = tsCols[pos]; - TSKEY lastTs = 0; //*(TSKEY*)pRuntimeEnv->prevRow[0]; +static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { + if (IS_VAR_DATA_TYPE(type)) { + // todo disable this + // if (pResultRow->key == NULL) { + // pResultRow->key = taosMemoryMalloc(varDataTLen(pData)); + // varDataCopy(pResultRow->key, pData); + // } else { + // ASSERT(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0); + // } + } else { + int64_t v = -1; + GET_TYPED_DATA(v, int64_t, type, pData); - // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. - // start exactly from this point, no need to do interpolation - TSKEY key = ascQuery ? win->skey : win->ekey; - if (key == curTs) { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - return true; + pResultRow->win.skey = v; + pResultRow->win.ekey = v; } +} - if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - return true; - } +int32_t setGroupResultOutputBuf(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, int16_t bytes, + int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, + SAggSupporter* pAggSup) { + SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo; + SqlFunctionCtx* pCtx = binfo->pCtx; - int32_t step = 1; // GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery)) ? lastTs : tsCols[pos - step]; + SResultRow* pResultRow = + doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo, false, pAggSup); + assert(pResultRow != NULL); - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, - RESULT_ROW_START_INTERP); - return true; + setResultRowKey(pResultRow, pData, type); + setResultRowInitCtx(pResultRow, pCtx, numOfCols, binfo->rowCellInfoOffset); + return TSDB_CODE_SUCCESS; } -static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex, - SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, - STimeWindow* win) { - int32_t order = TSDB_ORDER_ASC; - int32_t numOfOutput = pOperatorInfo->numOfOutput; +static bool functionNeedToExecute(SqlFunctionCtx* pCtx) { + struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - TSKEY actualEndKey = tsCols[endRowIndex]; - TSKEY key = order ? win->ekey : win->skey; + // in case of timestamp column, always generated results. + int32_t functionId = pCtx->functionId; + if (functionId == -1) { + return false; + } - // not ended in current data block, do not invoke interpolation - if ((key > blockEkey /*&& QUERY_IS_ASC_QUERY(pQueryAttr)*/) || - (key < blockEkey /*&& !QUERY_IS_ASC_QUERY(pQueryAttr)*/)) { - setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + if (isRowEntryCompleted(pResInfo)) { return false; } - // there is actual end point of current time window, no interpolation need - if (key == actualEndKey) { - setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); - return true; + if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) { + // return QUERY_IS_ASC_QUERY(pQueryAttr); + } + + // denote the order type + if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) { + // return pCtx->param[0].i == pQueryAttr->order.order; } - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - int32_t nextRowIndex = endRowIndex + step; - assert(nextRowIndex >= 0); + // in the reverse table scan, only the following functions need to be executed + // if (IS_REVERSE_SCAN(pRuntimeEnv) || + // (pRuntimeEnv->scanFlag == REPEAT_SCAN && functionId != FUNCTION_STDDEV && functionId != FUNCTION_PERCT)) { + // return false; + // } - TSKEY nextKey = tsCols[nextRowIndex]; - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey, - nextRowIndex, key, RESULT_ROW_END_INTERP); return true; } -static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SqlFunctionCtx* pCtx, - SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep, - int32_t order, bool timeWindowInterpo) { - if (!timeWindowInterpo) { - return; - } - - assert(pBlock != NULL); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); +static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, + int32_t paramIndex, int32_t numOfRows) { + if (pInput->pData[paramIndex] == NULL) { + pInput->pData[paramIndex] = taosMemoryCalloc(1, sizeof(SColumnInfoData)); + if (pInput->pData[paramIndex] == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } - if (pBlock->pDataBlock == NULL) { - // tscError("pBlock->pDataBlock == NULL"); - return; + // Set the correct column info (data type and bytes) + pInput->pData[paramIndex]->info.type = type; + pInput->pData[paramIndex]->info.bytes = tDataTypes[type].bytes; } - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); - - TSKEY* tsCols = (TSKEY*)(pColInfo->pData); - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { // it is not interpolated, now start to generated the interpolated value - int32_t startRowIndex = startPos; - bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, - pBlock->pDataBlock, tsCols, win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + SColumnDataAgg* da = NULL; + if (pInput->pColumnDataAgg[paramIndex] == NULL) { + da = taosMemoryCalloc(1, sizeof(SColumnDataAgg)); + pInput->pColumnDataAgg[paramIndex] = da; + if (da == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; } } else { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + da = pInput->pColumnDataAgg[paramIndex]; } - // point interpolation does not require the end key time window interpolation. - // if (pointInterpQuery) { - // return; - // } - - // interpolation query does not generate the time window end interpolation - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); - if (!done) { - int32_t endRowIndex = startPos + (forwardStep - 1) * step; - - TSKEY endKey = (order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey; - bool interp = - setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } else { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_END_INTERP); - } -} + ASSERT(!IS_VAR_DATA_TYPE(type)); -static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, - int32_t tableGroupId) { - STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*)pOperatorInfo->info; - - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - int32_t numOfOutput = pOperatorInfo->numOfOutput; - - SArray* pUpdated = NULL; - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - pUpdated = taosArrayInit(4, POINTER_BYTES); - } - - int32_t step = 1; - bool ascScan = true; - - // int32_t prevIndex = pResultRowInfo->curPos; - - TSKEY* tsCols = NULL; - if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - tsCols = (int64_t*)pColDataInfo->pData; - } - - int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan); - - STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, - pInfo->interval.precision, &pInfo->win); - bool masterScan = true; - - SResultRow* pResult = NULL; - int32_t ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, - tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, - &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - pos->groupId = tableGroupId; - pos->pos = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset}; - *(int64_t*) pos->key = pResult->win.skey; - - taosArrayPush(pUpdated, &pos); - } - - int32_t forwardStep = 0; - TSKEY ekey = win.ekey; - forwardStep = - getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - - // prev time window not interpolation yet. - // int32_t curIndex = pResultRowInfo->curPos; - -#if 0 - if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) { - for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. - SResultRow* pRes = getResultRow(pResultRowInfo, j); - if (pRes->closed) { - assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); - continue; - } - - STimeWindow w = pRes->win; - ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult, tableGroupId, - pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, - pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - doTimeWindowInterpolation(pOperatorInfo, &pInfo->binfo, pSDataBlock->pDataBlock, *(TSKEY*)pInfo->pRow[0], -1, - tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); - - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pInfo->binfo.pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - - doApplyFunctions(pInfo->binfo.pCtx, &w, &pInfo->timeWindowData, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - } - - // restore current time window - ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, - pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, - pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } -#endif - - // window start key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, - pInfo->order, false); - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); - doApplyFunctions(pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, - pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - - STimeWindow nextWin = win; - while (1) { - int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo); - if (startPos < 0) { - break; - } - - // null data, failed to allocate more memory buffer - int32_t code = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &nextWin, masterScan, &pResult, - tableGroupId, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - pos->groupId = tableGroupId; - pos->pos = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset}; - *(int64_t*) pos->key = pResult->win.skey; - - taosArrayPush(pUpdated, &pos); - } - - ekey = nextWin.ekey; // reviseWindowEkey(pQueryAttr, &nextWin); - forwardStep = - getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - - // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, - pInfo->order, false); - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, - pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - } - - if (pInfo->timeWindowInterpo) { - int32_t rowIndex = ascScan ? (pSDataBlock->info.rows - 1) : 0; - saveDataBlockLastRow(pInfo->pRow, pSDataBlock->pDataBlock, rowIndex, pSDataBlock->info.numOfCols); - } - - return pUpdated; - // updateResultRowInfoActiveIndex(pResultRowInfo, &pInfo->win, pRuntimeEnv->current->lastKey, true, false); -} - -static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts) { - pRowSup->win.ekey = ts; - pRowSup->prevTs = ts; - pRowSup->numOfRows += 1; -} - -static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex) { - pRowSup->startRowIndex = rowIndex; - pRowSup->numOfRows = 0; - pRowSup->win.skey = tsList[rowIndex]; -} - -// todo handle multiple tables cases. -static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - // todo find the correct time stamp column slot - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - - bool masterScan = true; - int32_t numOfOutput = pOperator->numOfOutput; - int64_t gid = pBlock->info.groupId; - - int64_t gap = pInfo->gap; - - if (!pInfo->reptScan) { - pInfo->reptScan = true; - pInfo->winSup.prevTs = INT64_MIN; - } - - SWindowRowsSup* pRowSup = &pInfo->winSup; - pRowSup->numOfRows = 0; - - // In case of ascending or descending order scan data, only one time window needs to be kepted for each table. - TSKEY* tsList = (TSKEY*)pColInfoData->pData; - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - if (pInfo->winSup.prevTs == INT64_MIN) { - doKeepNewWindowStartInfo(pRowSup, tsList, j); - doKeepTuple(pRowSup, tsList[j]); - } else if (tsList[j] - pRowSup->prevTs <= gap && (tsList[j] - pRowSup->prevTs) >= 0) { - // The gap is less than the threshold, so it belongs to current session window that has been opened already. - doKeepTuple(pRowSup, tsList[j]); - if (j == 0 && pRowSup->startRowIndex != 0) { - pRowSup->startRowIndex = 0; - } - } else { // start a new session window - SResultRow* pResult = NULL; - - // keep the time window for the closed time window. - STimeWindow window = pRowSup->win; - - pRowSup->win.ekey = pRowSup->win.skey; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &window, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); - } - - // pInfo->numOfRows data belong to the current session window - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); - doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, - pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - - // here we start a new session window - doKeepNewWindowStartInfo(pRowSup, tsList, j); - doKeepTuple(pRowSup, tsList[j]); - } - } - - SResultRow* pResult = NULL; - pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &pRowSup->win, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); - } - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); - doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, - pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); -} - -static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { - if (IS_VAR_DATA_TYPE(type)) { - // todo disable this - // if (pResultRow->key == NULL) { - // pResultRow->key = taosMemoryMalloc(varDataTLen(pData)); - // varDataCopy(pResultRow->key, pData); - // } else { - // ASSERT(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0); - // } - } else { - int64_t v = -1; - GET_TYPED_DATA(v, int64_t, type, pData); - - pResultRow->win.skey = v; - pResultRow->win.ekey = v; - } -} - -int32_t setGroupResultOutputBuf(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, int16_t bytes, - int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, - SAggSupporter* pAggSup) { - SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo; - SqlFunctionCtx* pCtx = binfo->pCtx; - - SResultRow* pResultRow = doSetResultOutBufByKey(pBuf, pResultRowInfo, groupId, (char*)pData, bytes, true, groupId, - pTaskInfo, false, pAggSup); - assert(pResultRow != NULL); - - setResultRowKey(pResultRow, pData, type); - setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfCols, binfo->rowCellInfoOffset); - return TSDB_CODE_SUCCESS; -} - -static bool functionNeedToExecute(SqlFunctionCtx* pCtx) { - struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - - // in case of timestamp column, always generated results. - int32_t functionId = pCtx->functionId; - if (functionId == -1) { - return false; - } - - if (isRowEntryCompleted(pResInfo)) { - return false; - } - - if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) { - // return QUERY_IS_ASC_QUERY(pQueryAttr); - } - - // denote the order type - if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) { - // return pCtx->param[0].i == pQueryAttr->order.order; - } - - // in the reverse table scan, only the following functions need to be executed - // if (IS_REVERSE_SCAN(pRuntimeEnv) || - // (pRuntimeEnv->scanFlag == REPEAT_SCAN && functionId != FUNCTION_STDDEV && functionId != FUNCTION_PERCT)) { - // return false; - // } - - return true; -} - -static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, - int32_t paramIndex, int32_t numOfRows) { - if (pInput->pData[paramIndex] == NULL) { - pInput->pData[paramIndex] = taosMemoryCalloc(1, sizeof(SColumnInfoData)); - if (pInput->pData[paramIndex] == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - // Set the correct column info (data type and bytes) - pInput->pData[paramIndex]->info.type = type; - pInput->pData[paramIndex]->info.bytes = tDataTypes[type].bytes; - } - - SColumnDataAgg* da = NULL; - if (pInput->pColumnDataAgg[paramIndex] == NULL) { - da = taosMemoryCalloc(1, sizeof(SColumnDataAgg)); - pInput->pColumnDataAgg[paramIndex] = da; - if (da == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } else { - da = pInput->pColumnDataAgg[paramIndex]; - } - - ASSERT(!IS_VAR_DATA_TYPE(type)); - - if (type == TSDB_DATA_TYPE_BIGINT) { - int64_t v = pFuncParam->param.i; - *da = (SColumnDataAgg){.numOfNull = 0, .min = v, .max = v, .maxIndex = 0, .minIndex = 0, .sum = v * numOfRows}; - } else if (type == TSDB_DATA_TYPE_DOUBLE) { - double v = pFuncParam->param.d; - *da = (SColumnDataAgg){.numOfNull = 0, .maxIndex = 0, .minIndex = 0}; + if (type == TSDB_DATA_TYPE_BIGINT) { + int64_t v = pFuncParam->param.i; + *da = (SColumnDataAgg){.numOfNull = 0, .min = v, .max = v, .maxIndex = 0, .minIndex = 0, .sum = v * numOfRows}; + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + double v = pFuncParam->param.d; + *da = (SColumnDataAgg){.numOfNull = 0, .maxIndex = 0, .minIndex = 0}; *(double*)&da->min = v; *(double*)&da->max = v; @@ -1812,7 +1045,10 @@ void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) { int32_t slotId = pFuncParam->pCol->slotId; - pInput->pColumnDataAgg[j] = &pBlock->pBlockAgg[slotId]; + pInput->pColumnDataAgg[j] = pBlock->pBlockAgg[slotId]; + if (pInput->pColumnDataAgg[j] == NULL) { + pInput->colDataAggIsSet = false; + } // Here we set the column info data since the data type for each column data is required, but // the data in the corresponding SColumnInfoData will not be used. @@ -1890,7 +1126,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, pCtx->functionId = -1; pCtx->curBufPage = -1; - pCtx->pExpr = pExpr; + pCtx->pExpr = pExpr; if (pExpr->pExpr->nodeType == QUERY_NODE_FUNCTION) { SFuncExecEnv env = {0}; @@ -1901,7 +1137,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, if (!isUdaf) { fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); } else { - char *udfName = pExpr->pExpr->_function.pFunctNode->functionName; + char* udfName = pExpr->pExpr->_function.pFunctNode->functionName; strncpy(pCtx->udfName, udfName, strlen(udfName)); fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet); } @@ -1926,9 +1162,9 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, pCtx->pTsOutput = NULL; pCtx->resDataInfo.bytes = pFunct->resSchema.bytes; pCtx->resDataInfo.type = pFunct->resSchema.type; - pCtx->order = TSDB_ORDER_ASC; + pCtx->order = TSDB_ORDER_ASC; pCtx->start.key = INT64_MIN; - pCtx->end.key = INT64_MIN; + pCtx->end.key = INT64_MIN; pCtx->numOfParams = pExpr->base.numOfParams; pCtx->param = pFunct->pParam; @@ -2357,8 +1593,8 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc STaskCostInfo* pCost = &pTaskInfo->cost; - pCost->totalBlocks += 1; - pCost->totalRows += pBlock->info.rows; +// pCost->totalBlocks += 1; +// pCost->totalRows += pBlock->info.rows; #if 0 // Calculate all time windows that are overlapping or contain current data block. // If current data block is contained by all possible time window, do not load current data block. @@ -2479,7 +1715,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc // if (pQueryAttr->pFilters != NULL) { // filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock); // } - + // if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { // filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery); // } @@ -2488,69 +1724,6 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc return TSDB_CODE_SUCCESS; } -int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { - int32_t midPos = -1; - int32_t numOfRows; - - if (num <= 0) { - return -1; - } - - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - - TSKEY* keyList = (TSKEY*)pValue; - int32_t firstPos = 0; - int32_t lastPos = num - 1; - - if (order == TSDB_ORDER_DESC) { - // find the first position which is smaller than the key - while (1) { - if (key >= keyList[lastPos]) return lastPos; - if (key == keyList[firstPos]) return firstPos; - if (key < keyList[firstPos]) return firstPos - 1; - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1) + firstPos; - - if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { - firstPos = midPos + 1; - } else { - break; - } - } - - } else { - // find the first position which is bigger than the key - while (1) { - if (key <= keyList[firstPos]) return firstPos; - if (key == keyList[lastPos]) return lastPos; - - if (key > keyList[lastPos]) { - lastPos = lastPos + 1; - if (lastPos >= num) - return -1; - else - return lastPos; - } - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1u) + firstPos; - - if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { - firstPos = midPos + 1; - } else { - break; - } - } - } - - return midPos; -} - /* * set tag value in SqlFunctionCtx * e.g.,tag information into input buffer @@ -2718,8 +1891,8 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t int64_t tid = 0; int64_t groupId = 0; - SResultRow* pRow = doSetResultOutBufByKey(pSup->pResultBuf, pResultRowInfo, tid, (char*)&tid, sizeof(tid), true, - groupId, pTaskInfo, false, pSup); + SResultRow* pRow = doSetResultOutBufByKey(pSup->pResultBuf, pResultRowInfo, (char*)&tid, sizeof(tid), true, groupId, + pTaskInfo, false, pSup); for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { struct SResultRowEntryInfo* pEntry = getResultCell(pRow, i, rowCellInfoOffset); @@ -2860,53 +2033,9 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD } } -// todo merged with the build group result. -void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList, - int32_t* rowCellInfoOffset) { - size_t num = taosArrayGetSize(pUpdateList); - - for (int32_t i = 0; i < num; ++i) { - SResKeyPos * pPos = taosArrayGetP(pUpdateList, i); - - SFilePage* bufPage = getBufPage(pBuf, pPos->pos.pageId); - SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset); -// - for (int32_t j = 0; j < numOfOutput; ++j) { - pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset); -// - struct SResultRowEntryInfo* pResInfo = pCtx[j].resultInfo; -// if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { -// continue; -// } -// -// if (pCtx[j].fpSet.process) { // TODO set the dummy function. -//// pCtx[j].fpSet.finalize(&pCtx[j]); -// pResInfo->initialized = true; -// } -// - if (pRow->numOfRows < pResInfo->numOfRes) { - pRow->numOfRows = pResInfo->numOfRes; - } - } - - releaseBufPage(pBuf, bufPage); - } -} - -STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win) { +STableQueryInfo* createTableQueryInfo(void* buf, STimeWindow win) { STableQueryInfo* pTableQueryInfo = buf; pTableQueryInfo->lastKey = win.skey; - - // set more initial size of interval/groupby query - // if (/*QUERY_IS_INTERVAL_QUERY(pQueryAttr) || */groupbyColumn) { - int32_t initialSize = 128; - // int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize); - // if (code != TSDB_CODE_SUCCESS) { - // return NULL; - // } - // } else { // in other aggregate query, do not initialize the windowResInfo - // } - return pTableQueryInfo; } @@ -2919,8 +2048,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo) { // cleanupResultRowInfo(&pTableQueryInfo->resInfo); } -void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, - int32_t* rowCellInfoOffset) { +void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); @@ -3005,9 +2133,8 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, u SqlFunctionCtx* pCtx = pAggInfo->binfo.pCtx; int32_t* rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; - SResultRow* pResultRow = - doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, uid, (char*)&groupId, sizeof(groupId), - true, groupId, pTaskInfo, false, &pAggInfo->aggSup); + SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, (char*)&groupId, + sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup); assert(pResultRow != NULL); /* @@ -3022,7 +2149,7 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, u } } - setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); + setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); } void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* pTaskInfo, SAggOperatorInfo* pAggInfo) { @@ -3036,47 +2163,6 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p pAggInfo->groupId = groupId; } -/* - * There are two cases to handle: - * - * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including - * pQueryAttr->lastKey, pQueryAttr->window.skey, and pQueryAttr->eKey. - * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be - * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there - * is a previous result generated or not. - */ -void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) { - // SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; - // if (pResultRowInfo->curPos != -1) { - // return; - // } - - // pTableQueryInfo->win.skey = key; - // STimeWindow win = {.skey = key, .ekey = pQRange->ekey}; - - /** - * In handling the both ascending and descending order super table query, we need to find the first qualified - * timestamp of this table, and then set the first qualified start timestamp. - * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional - * operations involve. - */ - // STimeWindow w = TSWINDOW_INITIALIZER; - // - // TSKEY sk = TMIN(win.skey, win.ekey); - // TSKEY ek = TMAX(win.skey, win.ekey); - // getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); - - // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { - // if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { - // assert(win.ekey == pQueryAttr->window.ekey); - // } - // - // pResultRowInfo->prevSKey = w.skey; - // } - - // pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; -} - /** * For interval query of both super table and table, copy the data in ascending order, since the output results are * ordered in SWindowResutl already. While handling the group by query for both table and super table, @@ -3105,8 +2191,8 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased } for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { - SResKeyPos *pPos = taosArrayGetP(pGroupResInfo->pRows, i); - SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); + SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); + SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); if (pRow->numOfRows == 0) { @@ -3149,10 +2235,13 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased return 0; } -void doBuildResultDatablock(SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, - SDiskbasedBuf* pBuf, int32_t* rowCellOffset, SqlFunctionCtx* pCtx) { +void doBuildResultDatablock(SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); + int32_t* rowCellOffset = pbInfo->rowCellInfoOffset; + SSDataBlock* pBlock = pbInfo->pRes; + SqlFunctionCtx* pCtx = pbInfo->pCtx; + blockDataCleanup(pBlock); if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { return; @@ -3194,16 +2283,16 @@ static int32_t compressQueryColData(SColumnInfoData* pColRes, int32_t numOfRows, colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); } -int32_t doFillTimeIntervalGapsInResults(struct SFillInfo* pFillInfo, SSDataBlock* pOutput, int32_t capacity, void** p) { +int32_t doFillTimeIntervalGapsInResults(struct SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t capacity) { // for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { // SColumnInfoData* pColInfoData = taosArrayGet(pOutput->pDataBlock, i); // p[i] = pColInfoData->pData + (pColInfoData->info.bytes * pOutput->info.rows); // } - int32_t numOfRows = (int32_t)taosFillResultDataBlock(pFillInfo, p, capacity - pOutput->info.rows); - pOutput->info.rows += numOfRows; + int32_t numOfRows = (int32_t)taosFillResultDataBlock(pFillInfo, pBlock, capacity - pBlock->info.rows); + pBlock->info.rows += numOfRows; - return pOutput->info.rows; + return pBlock->info.rows; } void publishOperatorProfEvent(SOperatorInfo* pOperator, EQueryProfEventType eventType) { @@ -3323,12 +2412,13 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo) { // // calculateOperatorProfResults(pQInfo); - qDebug("%s :cost summary: elapsed time:%" PRId64 " us, first merge:%" PRId64 - " us, total blocks:%d, " - "load block statis:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, - GET_TASKID(pTaskInfo), pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, - pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - // + SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder; + if (pSummary->pRecoder != NULL) { + qDebug("%s :cost summary: elapsed time:%" PRId64 " us, first merge:%" PRId64 " us, total blocks:%d, " + "load block statis:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, + GET_TASKID(pTaskInfo), pSummary->elapsedTime, pSummary->firstStageMergeTime, pRecorder->totalBlocks, + pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows, pRecorder->totalCheckedRows); + } // qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, // hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, // pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); @@ -3617,30 +2707,6 @@ static void doTableQueryInfoTimeWindowCheck(SExecTaskInfo* pTaskInfo, STableQuer #endif } -// SQueryTableDataCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win) { -// SQueryTableDataCond cond = { -// .colList = pQueryAttr->tableCols, -// .order = pQueryAttr->order.order, -// .numOfCols = pQueryAttr->numOfCols, -// .type = BLOCK_LOAD_OFFSET_SEQ_ORDER, -// .loadExternalRows = false, -// }; -// -// TIME_WINDOW_COPY(cond.twindow, *win); -// return cond; -// } - -static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo) { - STableIdInfo tidInfo; - // STableId* id = TSDB_TABLEID(pTableQueryInfo->pTable); - // - // tidInfo.uid = id->uid; - // tidInfo.tid = id->tid; - // tidInfo.key = pTableQueryInfo->lastKey; - - return tidInfo; -} - // static void updateTableIdInfo(STableQueryInfo* pTableQueryInfo, SSDataBlock* pBlock, SHashObj* pTableIdInfo, int32_t // order) { // int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); @@ -3668,6 +2734,7 @@ int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; pRsp->numOfRows = htonl(pRsp->numOfRows); pRsp->compLen = htonl(pRsp->compLen); + pRsp->numOfCols = htonl(pRsp->numOfCols); pRsp->useconds = htobe64(pRsp->useconds); } else { pSourceDataInfo->code = code; @@ -3746,12 +2813,11 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf } // NOTE: sources columns are more than the destination SSDatablock columns. -static void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols) { +void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols) { size_t numOfSrcCols = taosArrayGetSize(pCols); - ASSERT(numOfSrcCols >= pBlock->info.numOfCols); int32_t i = 0, j = 0; - while(i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) { + while (i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) { SColumnInfoData* p = taosArrayGet(pCols, i); SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, j); if (!pmInfo->output) { @@ -3777,10 +2843,10 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI blockDataEnsureCapacity(pRes, numOfRows); if (pColList == NULL) { // data from other sources - int32_t dataLen = *(int32_t*) pData; + int32_t dataLen = *(int32_t*)pData; pData += sizeof(int32_t); - pRes->info.groupId = *(uint64_t*) pData; + pRes->info.groupId = *(uint64_t*)pData; pData += sizeof(uint64_t); int32_t* colLen = (int32_t*)pData; @@ -3810,8 +2876,8 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI memcpy(pColInfoData->pData, pStart, colLen[i]); } - //TODO setting this flag to true temporarily so aggregate function on stable will - //examine NULL value for non-primary key column + // TODO setting this flag to true temporarily so aggregate function on stable will + // examine NULL value for non-primary key column pColInfoData->hasNull = true; pStart += colLen[i]; } @@ -3847,11 +2913,11 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI blockDataEnsureCapacity(&block, numOfRows); - int32_t dataLen = *(int32_t*) pStart; - uint64_t groupId = *(uint64_t*) (pStart + sizeof(int32_t)); + int32_t dataLen = *(int32_t*)pStart; + uint64_t groupId = *(uint64_t*)(pStart + sizeof(int32_t)); pStart += sizeof(int32_t) + sizeof(uint64_t); - int32_t* colLen = (int32_t*) (pStart); + int32_t* colLen = (int32_t*)(pStart); pStart += sizeof(int32_t) * numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { @@ -3877,7 +2943,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI } // data from mnode - relocateColumnData(pRes, pColList, block.pDataBlock); + relocateColumnData(pRes, pColList, block.pDataBlock); } pRes->info.rows = numOfRows; @@ -3949,9 +3015,8 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx } SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; - code = - setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pOperator->numOfOutput, startTs, &pDataInfo->totalRows, NULL); + code = setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, + pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); if (code != 0) { goto _error; } @@ -4086,7 +3151,7 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; int32_t code = setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pOperator->numOfOutput, startTs, &pDataInfo->totalRows, NULL); + pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64 @@ -4126,7 +3191,7 @@ static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -4145,8 +3210,6 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator, bool* newgroup) { return NULL; } - *newgroup = false; - if (pExchangeInfo->seqLoadData) { return seqLoadRemoteData(pOperator); } else { @@ -4220,14 +3283,14 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; - pOperator->blockingOptr = false; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->numOfOutput = pBlock->info.numOfCols; + pOperator->numOfExprs = pBlock->info.numOfCols; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo, - NULL, NULL, NULL); + pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, + destroyExchangeOperatorInfo, NULL, NULL, NULL); #if 1 { // todo refactor @@ -4267,7 +3330,6 @@ _error: static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, const char* pKey); -static void cleanupAggSup(SAggSupporter* pAggSup); static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { SSortedMergeOperatorInfo* pInfo = (SSortedMergeOperatorInfo*)param; @@ -4282,65 +3344,6 @@ static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { cleanupAggSup(&pInfo->aggSup); } -static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { - assert(dst != NULL && src != NULL); - - *dst = *src; - - dst->pExpr = exprdup(src->pExpr); - dst->base.pParam = taosMemoryCalloc(src->base.numOfParams, sizeof(SColumn)); - memcpy(dst->base.pParam, src->base.pParam, sizeof(SColumn) * src->base.numOfParams); - - // memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param)); - // for (int32_t j = 0; j < src->base.numOfParams; ++j) { - // taosVariantAssign(&dst->base.param[j], &src->base.param[j]); - // } -} - -// TODO merge aggregate super table -static void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - - bool isNull = tsortIsNullVal(pTupleHandle, i); - if (isNull) { - colDataAppend(pColInfo, pBlock->info.rows, NULL, true); - } else { - char* pData = tsortGetValue(pTupleHandle, i); - colDataAppend(pColInfo, pBlock->info.rows, pData, false); - } - } - - pBlock->info.rows += 1; -} - -SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity) { - blockDataCleanup(pDataBlock); - blockDataEnsureCapacity(pDataBlock, capacity); - - blockDataEnsureCapacity(pDataBlock, capacity); - - while (1) { - STupleHandle* pTupleHandle = tsortNextTuple(pHandle); - if (pTupleHandle == NULL) { - break; - } - - appendOneRowToDataBlock(pDataBlock, pTupleHandle); - if (pDataBlock->info.rows >= capacity) { - return pDataBlock; - } - } - - return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; -} - -SSDataBlock* loadNextDataBlock(void* param) { - SOperatorInfo* pOperator = (SOperatorInfo*)param; - bool newgroup = false; - return pOperator->fpSet.getNextFn(pOperator, &newgroup); -} - static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int32_t rowIndex) { size_t size = taosArrayGetSize(groupInfo); if (size == 0) { @@ -4443,8 +3446,8 @@ static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock doMergeResultImpl(pInfo, pCtx, numOfExpr, i); } else { doFinalizeResultImpl(pCtx, numOfExpr); - int32_t numOfRows = getNumOfResult(pInfo->binfo.pCtx, pOperator->numOfOutput, NULL); - // setTagValueForMultipleRows(pCtx, pOperator->numOfOutput, numOfRows); + int32_t numOfRows = getNumOfResult(pInfo->binfo.pCtx, pOperator->numOfExprs, NULL); + // setTagValueForMultipleRows(pCtx, pOperator->numOfExprs, numOfRows); // TODO check for available buffer; @@ -4494,13 +3497,13 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) { setInputDataBlock(pOperator, pInfo->binfo.pCtx, pDataBlock, TSDB_ORDER_ASC, true); // updateOutputBuf(&pInfo->binfo, &pAggInfo->bufCapacity, pBlock->info.rows * pAggInfo->resultRowFactor, // pOperator->pRuntimeEnv, true); - doMergeImpl(pOperator, pOperator->numOfOutput, pDataBlock); + doMergeImpl(pOperator, pOperator->numOfExprs, pDataBlock); // flush to tuple store, and after all data have been handled, return to upstream node or sink node } - doFinalizeResultImpl(pInfo->binfo.pCtx, pOperator->numOfOutput); - int32_t numOfRows = getNumOfResult(pInfo->binfo.pCtx, pOperator->numOfOutput, NULL); - // setTagValueForMultipleRows(pCtx, pOperator->numOfOutput, numOfRows); + doFinalizeResultImpl(pInfo->binfo.pCtx, pOperator->numOfExprs); + int32_t numOfRows = getNumOfResult(pInfo->binfo.pCtx, pOperator->numOfExprs, NULL); + // setTagValueForMultipleRows(pCtx, pOperator->numOfExprs, numOfRows); // TODO check for available buffer; @@ -4509,7 +3512,7 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) { return (pInfo->binfo.pRes->info.rows > 0) ? pInfo->binfo.pRes : NULL; } -static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -4517,14 +3520,14 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator, bool* newgroup) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSortedMergeOperatorInfo* pInfo = pOperator->info; if (pOperator->status == OP_RES_TO_RETURN) { - return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity); + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL); } int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, NULL, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->binfo.pRes, "GET_TASKID(pTaskInfo)"); - tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); + tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL); for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); @@ -4628,10 +3631,10 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t pOperator->name = "SortedMerge"; // pOperator->operatorType = OP_SortedMerge; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->numOfOutput = num; + pOperator->numOfExprs = num; pOperator->pExpr = pExprInfo; pOperator->pTaskInfo = pTaskInfo; @@ -4656,79 +3659,6 @@ _error: return NULL; } -static SSDataBlock* doSort(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSortOperatorInfo* pInfo = pOperator->info; - - if (pOperator->status == OP_RES_TO_RETURN) { - return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->numOfRowsInRes); - } - - int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->inputSlotMap, SORT_SINGLESOURCE_SORT, - pInfo->bufPageSize, numOfBufPage, pInfo->pDataBlock, pTaskInfo->id.str); - - tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); - - SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); - ps->param = pOperator->pDownstream[0]; - tsortAddSource(pInfo->pSortHandle, ps); - - int32_t code = tsortOpen(pInfo->pSortHandle); - taosMemoryFreeClear(ps); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, terrno); - } - - pOperator->status = OP_RES_TO_RETURN; - return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->numOfRowsInRes); -} - -SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, - SArray* pIndexMap, SExecTaskInfo* pTaskInfo) { - SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - int32_t rowSize = pResBlock->info.rowSize; - - if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + header - - pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer - pInfo->numOfRowsInRes = 1024; - pInfo->pDataBlock = pResBlock; - pInfo->pSortInfo = pSortInfo; - pInfo->inputSlotMap = pIndexMap; - - pOperator->name = "SortOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - - pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSort, NULL, NULL, destroyOrderOperatorInfo, - NULL, NULL, NULL); - - int32_t code = appendDownstream(pOperator, &downstream, 1); - return pOperator; - -_error: - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pInfo); - taosMemoryFree(pOperator); - return NULL; -} - int32_t getTableScanOrder(SOperatorInfo* pOperator) { if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { @@ -4759,24 +3689,28 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { bool newgroup = true; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, &newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; } // if (pAggInfo->current != NULL) { - // setTagValue(pOperator, pAggInfo->current->pTable, pInfo->pCtx, pOperator->numOfOutput); + // setTagValue(pOperator, pAggInfo->current->pTable, pInfo->pCtx, pOperator->numOfExprs); // } // there is an scalar expression that needs to be calculated before apply the group aggregation. if (pAggInfo->pScalarExprInfo != NULL) { - projectApplyFunctions(pAggInfo->pScalarExprInfo, pBlock, pBlock, pAggInfo->pScalarCtx, pAggInfo->numOfScalarExpr, - NULL); + int32_t code = projectApplyFunctions(pAggInfo->pScalarExprInfo, pBlock, pBlock, pAggInfo->pScalarCtx, + pAggInfo->numOfScalarExpr, NULL); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + longjmp(pTaskInfo->env, pTaskInfo->code); + } } // the pDataBlock are always the same one, no need to call this again - setExecutionContext(pOperator->numOfOutput, pBlock->info.groupId, pTaskInfo, pAggInfo); + setExecutionContext(pOperator->numOfExprs, pBlock->info.groupId, pTaskInfo, pAggInfo); setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, true); doAggregateImpl(pOperator, 0, pInfo->pCtx); @@ -4797,7 +3731,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { } closeAllResultRows(&pAggInfo->binfo.resultRowInfo); - finalizeMultiTupleQueryResult(pAggInfo->binfo.pCtx, pOperator->numOfOutput, pAggInfo->aggSup.pResultBuf, + finalizeMultiTupleQueryResult(pAggInfo->binfo.pCtx, pOperator->numOfExprs, pAggInfo->aggSup.pResultBuf, &pAggInfo->binfo.resultRowInfo, pAggInfo->binfo.rowCellInfoOffset); initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, false); @@ -4805,7 +3739,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { SAggOperatorInfo* pAggInfo = pOperator->info; SOptrBasicInfo* pInfo = &pAggInfo->binfo; @@ -4820,8 +3754,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator, bool* newgroup) } blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->pRes, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf, - pInfo->rowCellInfoOffset, pInfo->pCtx); + doBuildResultDatablock(pInfo, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf); if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -4933,8 +3866,8 @@ bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasi initResultRow(resultRow); prepareResultListBuffer(&pInfo->resultRowInfo, pOperator->pTaskInfo->env); // pInfo->resultRowInfo.cur = pInfo->resultRowInfo.size; -// pInfo->resultRowInfo.pPosition[pInfo->resultRowInfo.size++] = -// (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset}; + // pInfo->resultRowInfo.pPosition[pInfo->resultRowInfo.size++] = + // (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset}; pInfo->resultRowInfo.cur = (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset}; } @@ -4946,13 +3879,13 @@ bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasi enum { PROJECT_RETRIEVE_CONTINUE = 0x1, - PROJECT_RETRIEVE_DONE = 0x2, + PROJECT_RETRIEVE_DONE = 0x2, }; static int32_t handleLimitOffset(SOperatorInfo* pOperator, SSDataBlock* pBlock) { SProjectOperatorInfo* pProjectInfo = pOperator->info; SOptrBasicInfo* pInfo = &pProjectInfo->binfo; - SSDataBlock* pRes = pInfo->pRes; + SSDataBlock* pRes = pInfo->pRes; if (pProjectInfo->curSOffset > 0) { if (pProjectInfo->groupId == 0) { // it is the first group @@ -5015,14 +3948,15 @@ static int32_t handleLimitOffset(SOperatorInfo* pOperator, SSDataBlock* pBlock) // todo optimize performance // If there are slimit/soffset value exists, multi-round result can not be packed into one group, since the // they may not belong to the same group the limit/offset value is not valid in this case. - if (pRes->info.rows >= pOperator->resultInfo.threshold || pProjectInfo->slimit.offset != -1 || pProjectInfo->slimit.limit != -1) { + if (pRes->info.rows >= pOperator->resultInfo.threshold || pProjectInfo->slimit.offset != -1 || + pProjectInfo->slimit.limit != -1) { return PROJECT_RETRIEVE_DONE; - } else { // not full enough, continue to accumulate the output data in the buffer. + } else { // not full enough, continue to accumulate the output data in the buffer. return PROJECT_RETRIEVE_CONTINUE; } } -static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { SProjectOperatorInfo* pProjectInfo = pOperator->info; SOptrBasicInfo* pInfo = &pProjectInfo->binfo; @@ -5041,17 +3975,17 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) // todo dynamic set tags // if (pTableQueryInfo != NULL) { - // setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); + // setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfExprs); // } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC); blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows); - projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfOutput); + projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs); if (pRes->info.rows >= pProjectInfo->binfo.capacity * 0.8) { - copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); - resetResultRowEntryResult(pInfo->pCtx, pOperator->numOfOutput); + copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfExprs); + resetResultRowEntryResult(pInfo->pCtx, pOperator->numOfExprs); return pRes; } } @@ -5060,480 +3994,53 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { - bool prevVal = *newgroup; - // The downstream exec may change the value of the newgroup, so use a local variable instead. publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { - *newgroup = prevVal; setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); break; } // Return result of the previous group in the firstly. - if (*newgroup) { + if (false) { if (pRes->info.rows > 0) { pProjectInfo->existDataBlock = pBlock; break; } else { // init output buffer for a new group data - initCtxOutputBuffer(pInfo->pCtx, pOperator->numOfOutput); + initCtxOutputBuffer(pInfo->pCtx, pOperator->numOfExprs); } } // todo dynamic set tags // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; // if (pTableQueryInfo != NULL) { - // setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); + // setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfExprs); // } - // the pDataBlock are always the same one, no need to call this again - int32_t order = getTableScanOrder(pOperator->pDownstream[0]); - - setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, false); - blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); - - projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfOutput, - pProjectInfo->pPseudoColInfo); - - int32_t status = handleLimitOffset(pOperator, pBlock); - if (status == PROJECT_RETRIEVE_CONTINUE) { - continue; - } else if (status == PROJECT_RETRIEVE_DONE) { - break; - } - } - - pProjectInfo->curOutput += pInfo->pRes->info.rows; - - // copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); - return (pInfo->pRes->info.rows > 0) ? pInfo->pRes : NULL; -} - -static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { - if (OPTR_IS_OPENED(pOperator)) { - return TSDB_CODE_SUCCESS; - } - - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - STableIntervalOperatorInfo* pInfo = pOperator->info; - - int32_t order = TSDB_ORDER_ASC; - // STimeWindow win = {0}; - bool newgroup = false; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, &newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); - STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; - - setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId); - -#if 0 // test for encode/decode result info - if(pOperator->encodeResultRow){ - char *result = NULL; - int32_t length = 0; - SAggSupporter *pSup = &pInfo->aggSup; - pOperator->encodeResultRow(pOperator, pSup, &pInfo->binfo, &result, &length); - taosHashClear(pSup->pResultRowHashTable); - pInfo->binfo.resultRowInfo.size = 0; - pOperator->decodeResultRow(pOperator, pSup, &pInfo->binfo, result, length); - if(result){ - taosMemoryFree(result); - } - } -#endif - } - - closeAllResultRows(&pInfo->binfo.resultRowInfo); - finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, - &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); - OPTR_SET_OPENED(pOperator); - return TSDB_CODE_SUCCESS; -} - -static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgroup) { - STableIntervalOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SSDataBlock* pBlock = pInfo->binfo.pRes; - - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - return pOperator->fpSet.getStreamResFn(pOperator, newgroup); - } else { - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - return NULL; - } - - blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBlock, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); - - if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pBlock->info.rows == 0 ? NULL : pBlock; - } -} - -static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { - STableIntervalOperatorInfo* pInfo = pOperator->info; - int32_t order = TSDB_ORDER_ASC; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; - } - - // STimeWindow win = {0}; - *newgroup = false; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - SArray* pUpdated = NULL; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - // The timewindows that overlaps the timestamps of the input pBlock need to be recalculated and return to the - // caller. Note that all the time window are not close till now. - - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); - pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); - } - - finalizeUpdatedResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, pUpdated, - pInfo->binfo.rowCellInfoOffset); - - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); - - ASSERT(pInfo->binfo.pRes->info.rows > 0); - pOperator->status = OP_RES_TO_RETURN; - - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; -} - -static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STimeSliceOperatorInfo* pSliceInfo = pOperator->info; - if (pOperator->status == OP_RES_TO_RETURN) { - // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pSliceInfo->binfo.pRes; - } - - int32_t order = TSDB_ORDER_ASC; - // STimeWindow win = pQueryAttr->window; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - if (pBlock == NULL) { - break; - } - - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, true); - // hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0); - } - - // restore the value - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pSliceInfo->binfo.resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - // finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput); - -// initGroupedResultInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); - // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes); - - if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pSliceInfo->binfo.pRes->info.rows == 0 ? NULL : pSliceInfo->binfo.pRes; -} - -static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - STableIntervalOperatorInfo* pInfo = pOperator->info; - if (pOperator->status == OP_RES_TO_RETURN) { - int64_t st = taosGetTimestampUs(); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; - } - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - // setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, TSDB_ORDER_ASC, true); - STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; - - setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); - // hashIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pBlock->info.groupId); - } - - closeAllResultRows(&pInfo->binfo.resultRowInfo); - finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, - &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - -// initGroupedResultInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); - OPTR_SET_OPENED(pOperator); - - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); - - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; -} - -static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SOptrBasicInfo* pBInfo = &pInfo->binfo; - - SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); - int64_t gid = pBlock->info.groupId; - - bool masterScan = true; - int32_t numOfOutput = pOperator->numOfOutput; - - int16_t bytes = pStateColInfoData->info.bytes; - int16_t type = pStateColInfoData->info.type; - - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - TSKEY* tsList = (TSKEY*)pColInfoData->pData; - - SWindowRowsSup* pRowSup = &pInfo->winSup; - pRowSup->numOfRows = 0; - - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pBlock->pBlockAgg)) { - continue; - } - - char* val = colDataGetData(pStateColInfoData, j); - - if (!pInfo->hasKey) { - memcpy(pInfo->stateKey.pData, val, bytes); - pInfo->hasKey = true; - - doKeepNewWindowStartInfo(pRowSup, tsList, j); - doKeepTuple(pRowSup, tsList[j]); - } else if (memcmp(pInfo->stateKey.pData, val, bytes) == 0) { - doKeepTuple(pRowSup, tsList[j]); - if (j == 0 && pRowSup->startRowIndex != 0) { - pRowSup->startRowIndex = 0; - } - } else { // a new state window started - SResultRow* pResult = NULL; - - // keep the time window for the closed time window. - STimeWindow window = pRowSup->win; - - pRowSup->win.ekey = pRowSup->win.skey; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &window, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); - } - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); - doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, - pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - - // here we start a new session window - doKeepNewWindowStartInfo(pRowSup, tsList, j); - doKeepTuple(pRowSup, tsList[j]); - } - } - - SResultRow* pResult = NULL; - pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &pRowSup->win, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); - } - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); - doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, - pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); -} - -static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SStateWindowOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SOptrBasicInfo* pBInfo = &pInfo->binfo; - - if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - return NULL; - } - - return pBInfo->pRes; - } - - int32_t order = TSDB_ORDER_ASC; - STimeWindow win = pTaskInfo->window; - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); - doStateWindowAggImpl(pOperator, pInfo, pBlock); - } - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pBInfo->resultRowInfo); - finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, - pBInfo->rowCellInfoOffset); - - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); - blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; -} - -static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SSessionAggOperatorInfo* pInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pInfo->binfo; - - if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - return NULL; - } + // the pDataBlock are always the same one, no need to call this again + int32_t order = getTableScanOrder(pOperator->pDownstream[0]); - return pBInfo->pRes; - } + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, false); + blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); - int32_t order = TSDB_ORDER_ASC; - SOperatorInfo* downstream = pOperator->pDownstream[0]; + projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs, + pProjectInfo->pPseudoColInfo); - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - if (pBlock == NULL) { + int32_t status = handleLimitOffset(pOperator, pBlock); + if (status == PROJECT_RETRIEVE_CONTINUE) { + continue; + } else if (status == PROJECT_RETRIEVE_DONE) { break; } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); - doSessionWindowAggImpl(pOperator, pInfo, pBlock); } - // restore the value - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pBInfo->resultRowInfo); - finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, - pBInfo->rowCellInfoOffset); - - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); - blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } + pProjectInfo->curOutput += pInfo->pRes->info.rows; - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; + // copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfExprs); + return (pInfo->pRes->info.rows > 0) ? pInfo->pRes : NULL; } static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo* pInfo, SResultInfo* pResultInfo, bool* newgroup, @@ -5547,7 +4054,7 @@ static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo* pInfo, SResult taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->existNewGroupBlock); - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pResultInfo->capacity, pInfo->p); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pResultInfo->capacity); pInfo->existNewGroupBlock = NULL; *newgroup = true; } @@ -5556,7 +4063,7 @@ static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo* pInfo, SResultInf SExecTaskInfo* pTaskInfo) { if (taosFillHasMoreResults(pInfo->pFillInfo)) { *newgroup = false; - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pResultInfo->capacity, pInfo->p); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pResultInfo->capacity); if (pInfo->pRes->info.rows > pResultInfo->threshold || (!pInfo->multigroupResult)) { return; } @@ -5568,7 +4075,7 @@ static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo* pInfo, SResultInf } } -static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doFill(SOperatorInfo* pOperator) { SFillOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -5580,6 +4087,9 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { return NULL; } + // todo handle different group data interpolation + bool n = false; + bool *newgroup = &n; doHandleRemainBlockFromNewGroup(pInfo, pResultInfo, newgroup, pTaskInfo); if (pResBlock->info.rows > pResultInfo->threshold || (!pInfo->multigroupResult && pResBlock->info.rows > 0)) { return pResBlock; @@ -5588,7 +4098,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { SOperatorInfo* pDownstream = pOperator->pDownstream[0]; while (1) { publishOperatorProfEvent(pDownstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream, newgroup); + SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); publishOperatorProfEvent(pDownstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (*newgroup) { @@ -5617,7 +4127,8 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { } } - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pResBlock, pOperator->resultInfo.capacity, pInfo->p); + blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pResBlock, pOperator->resultInfo.capacity); // current group has no more result to return if (pResBlock->info.rows > 0) { @@ -5661,7 +4172,7 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { } if (pOperator->fpSet.closeFn != NULL) { - pOperator->fpSet.closeFn(pOperator->info, pOperator->numOfOutput); + pOperator->fpSet.closeFn(pOperator->info, pOperator->numOfExprs); } if (pOperator->pDownstream != NULL) { @@ -5684,8 +4195,8 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t)); pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); -// pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); -// pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); + // pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); + // pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); if (pAggSup->keyBuf == NULL /*|| pAggSup->pResultRowArrayList == NULL || pAggSup->pResultRowListSet == NULL*/ || pAggSup->pResultRowHashTable == NULL) { @@ -5700,11 +4211,9 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n return TSDB_CODE_SUCCESS; } -static void cleanupAggSup(SAggSupporter* pAggSup) { +void cleanupAggSup(SAggSupporter* pAggSup) { taosMemoryFreeClear(pAggSup->keyBuf); taosHashCleanup(pAggSup->pResultRowHashTable); -// taosHashCleanup(pAggSup->pResultRowListSet); -// taosArrayDestroy(pAggSup->pResultRowArrayList); destroyDiskbasedBuf(pAggSup->pResultBuf); } @@ -5715,7 +4224,7 @@ int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInf doInitAggInfoSup(pAggSup, pBasicInfo->pCtx, numOfCols, keyBufSize, pkey); - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { pBasicInfo->pCtx[i].pBuf = pAggSup->pResultBuf; } @@ -5732,6 +4241,10 @@ void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows) { } static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInfo) { + if (pTableGroupInfo->numOfTables == 0) { + return NULL; + } + STableQueryInfo* pTableQueryInfo = taosMemoryCalloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); if (pTableQueryInfo == NULL) { return NULL; @@ -5751,13 +4264,14 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf } STimeWindow win = {0, INT64_MAX}; - createTableQueryInfo(pTableQueryInfo, false, win); + createTableQueryInfo(pTableQueryInfo, win); return pTableQueryInfo; } SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, - int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { + int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, + const STableGroupInfo* pTableGroupInfo) { SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5771,7 +4285,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResultBlock, keyBufSize, pTaskInfo->id.str); pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); - if (code != TSDB_CODE_SUCCESS || pInfo->pTableQueryInfo == NULL) { + if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -5790,11 +4304,11 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->name = "TableAggregate"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_AGG; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; + pOperator->numOfExprs = numOfCols; pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, NULL, destroyAggOperatorInfo, @@ -5829,28 +4343,11 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { doDestroyBasicInfo(pInfo, numOfOutput); } -void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { - SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - taosMemoryFreeClear(pInfo->stateKey.pData); -} - void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); } -void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*)param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - cleanupAggSup(&pInfo->aggSup); -} - -void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { - SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); -} - void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { SFillOperatorInfo* pInfo = (SFillOperatorInfo*)param; pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); @@ -5863,14 +4360,6 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { doDestroyBasicInfo(&pInfo->binfo, numOfOutput); } -static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { - SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param; - pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock); - - taosArrayDestroy(pInfo->pSortInfo); - taosArrayDestroy(pInfo->inputSlotMap); -} - void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) { SExchangeInfo* pExInfo = (SExchangeInfo*)param; taosArrayDestroy(pExInfo->pSources); @@ -5920,14 +4409,14 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p pOperator->name = "ProjectOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; - pOperator->blockingOptr = false; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = num; + pOperator->numOfExprs = num; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, NULL, destroyProjectOperatorInfo, - NULL, NULL, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, NULL, + destroyProjectOperatorInfo, NULL, NULL, NULL); pOperator->pTaskInfo = pTaskInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); @@ -5942,263 +4431,17 @@ _error: return NULL; } -SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, - SExecTaskInfo* pTaskInfo) { - STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; -// pInfo->execModel = OPTR_EXEC_MODEL_STREAM; - pInfo->execModel = pTaskInfo->execModel; - pInfo->win = pTaskInfo->window; - pInfo->twAggSup = *pTwAggSupp; - pInfo->primaryTsIndex = primaryTsSlotId; - - int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - initResultSizeInfo(pOperator, numOfRows); - int32_t code = - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); - - // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); - if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { - goto _error; - } - - initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - - pOperator->name = "TimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, destroyIntervalOperatorInfo, - aggEncodeResultRow, aggDecodeResultRow, NULL); - - code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - return pOperator; - -_error: - destroyIntervalOperatorInfo(pInfo, numOfCols); - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - -SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { - STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; - pInfo->execModel = OPTR_EXEC_MODEL_STREAM; - pInfo->win = pTaskInfo->window; - pInfo->twAggSup = *pTwAggSupp; - pInfo->primaryTsIndex = primaryTsSlotId; - - int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - initResultSizeInfo(pOperator, numOfRows); - int32_t code = - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); - - // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); - if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { - goto _error; - } - - initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - - pOperator->name = "StreamTimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL, destroyIntervalOperatorInfo, - aggEncodeResultRow, aggDecodeResultRow, NULL); - - code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - return pOperator; - - _error: - destroyIntervalOperatorInfo(pInfo, numOfCols); - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; - -} - -SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo) { - STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pOperator == NULL || pInfo == NULL) { - goto _error; - } - - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - - pOperator->name = "TimeSliceOperator"; - // pOperator->operatorType = OP_AllTimeWindow; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doAllIntervalAgg, NULL, NULL, destroyBasicOperatorInfo, - NULL, NULL, NULL); - - int32_t code = appendDownstream(pOperator, &downstream, 1); - return pOperator; - -_error: - taosMemoryFree(pInfo); - taosMemoryFree(pOperator); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - return NULL; -} - -SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, - SExecTaskInfo* pTaskInfo) { - SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - pInfo->colIndex = -1; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - initResultSizeInfo(pOperator, 4096); - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExpr, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - - pInfo->twAggSup = *pTwAggSup; - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - - pOperator->name = "StateWindowOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfCols; - - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL, destroyStateWindowOperatorInfo, - aggEncodeResultRow, aggDecodeResultRow, NULL); - - int32_t code = appendDownstream(pOperator, &downstream, 1); - return pOperator; - -_error: - pTaskInfo->code = TSDB_CODE_SUCCESS; - return NULL; -} - -SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp* pTwAggSupp, - SExecTaskInfo* pTaskInfo) { - SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - initResultSizeInfo(pOperator, numOfRows); - int32_t code = - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->twAggSup = *pTwAggSupp; - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - - pInfo->gap = gap; - pInfo->binfo.pRes = pResBlock; - pInfo->winSup.prevTs = INT64_MIN; - pInfo->reptScan = false; - pOperator->name = "SessionWindowAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL, destroySWindowOperatorInfo, - aggEncodeResultRow, aggDecodeResultRow, NULL); - pOperator->pTaskInfo = pTaskInfo; - - code = appendDownstream(pOperator, &downstream, 1); - return pOperator; - -_error: - if (pInfo != NULL) { - destroySWindowOperatorInfo(pInfo, numOfCols); - } - - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - -static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, int64_t* fillVal, +static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SNodeListNode* pValNode, STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType) { - SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, NULL); + SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pValNode); - // TODO set correct time precision STimeWindow w = TSWINDOW_INITIALIZER; - getAlignQueryTimeWindow(pInterval, TSDB_TIME_PRECISION_MILLI, win.skey, &w); + getAlignQueryTimeWindow(pInterval, pInterval->precision, win.skey, &w); int32_t order = TSDB_ORDER_ASC; pInfo->pFillInfo = taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval, fillType, pColInfo, id); pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); - if (pInfo->pFillInfo == NULL || pInfo->p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } else { @@ -6207,14 +4450,13 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t } SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SInterval* pInterval, SSDataBlock* pResBlock, int32_t fillType, char* fillVal, + SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* pValueNode, bool multigroupResult, SExecTaskInfo* pTaskInfo) { SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pInfo->pRes = pResBlock; pInfo->multigroupResult = multigroupResult; - pInfo->intervalInfo = *pInterval; int32_t type = TSDB_FILL_NONE; switch (fillType) { @@ -6243,22 +4485,22 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp SResultInfo* pResultInfo = &pOperator->resultInfo; initResultSizeInfo(pOperator, 4096); - int32_t code = initFillInfo(pInfo, pExpr, numOfCols, (int64_t*)fillVal, pTaskInfo->window, pResultInfo->capacity, + int32_t code = initFillInfo(pInfo, pExpr, numOfCols, pValueNode, *pWindow, pResultInfo->capacity, pTaskInfo->id.str, pInterval, type); if (code != TSDB_CODE_SUCCESS) { goto _error; } pOperator->name = "FillOperator"; - pOperator->blockingOptr = false; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; - // pOperator->operatorType = OP_Fill; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_FILL; pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfCols; + pOperator->numOfExprs = numOfCols; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, NULL, destroySFillOperatorInfo, - NULL, NULL, NULL); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, NULL, destroySFillOperatorInfo, NULL, NULL, NULL); pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -6269,7 +4511,6 @@ _error: return NULL; } - static int32_t getColumnIndexInSource(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr, SColumnInfo* pTagCols) { int32_t j = 0; @@ -6460,18 +4701,19 @@ static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo); static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols); -static SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget); +static SArray* createSortInfo(SNodeList* pNodeList); static SArray* createIndexMap(SNodeList* pNodeList); static SArray* extractPartitionColInfo(SNodeList* pNodeList); static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode); +static void setJoinColumnInfo(SColumnInfo* pInfo, const SColumnNode* pLeftNode); static SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode) { SInterval interval = { - .interval = pTableScanNode->interval, - .sliding = pTableScanNode->sliding, + .interval = pTableScanNode->interval, + .sliding = pTableScanNode->sliding, .intervalUnit = pTableScanNode->intervalUnit, - .slidingUnit = pTableScanNode->slidingUnit, - .offset = pTableScanNode->offset, + .slidingUnit = pTableScanNode->slidingUnit, + .offset = pTableScanNode->offset, }; return interval; @@ -6486,7 +4728,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; - int32_t numOfCols = 0; + int32_t numOfCols = 0; tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); if (pDataReader == NULL && terrno != 0) { return NULL; @@ -6503,8 +4745,12 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } SInterval interval = extractIntervalInfo(pTableScanNode); - return createTableScanOperatorInfo(pDataReader, &cond, numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq, pColList, - pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo); + SOperatorInfo* pOperator = createTableScanOperatorInfo(pDataReader, &cond, numOfCols, pTableScanNode->dataRequired, + pTableScanNode->scanSeq, pColList, pResBlock, pScanPhyNode->node.pConditions, + &interval, pTableScanNode->ratio, pTaskInfo); + STableScanInfo* pScanInfo = pOperator->info; + pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; + return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc); @@ -6519,20 +4765,41 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t numOfCols = 0; SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo, + pScanPhyNode->node.pConditions); taosArrayDestroy(tableIdList); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; - SScanPhysiNode* pScanNode = &pSysScanPhyNode->scan; + SScanPhysiNode* pScanNode = &pSysScanPhyNode->scan; SSDataBlock* pResBlock = createResDataBlock(pScanNode->node.pOutputDataBlockDesc); int32_t numOfOutputCols = 0; - SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pScanNode->node.pOutputDataBlockDesc, &numOfOutputCols); + SArray* colList = + extractColMatchInfo(pScanNode->pScanCols, pScanNode->node.pOutputDataBlockDesc, &numOfOutputCols); SOperatorInfo* pOperator = createSysTableScanOperatorInfo( - pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, - colList, pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId); + pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, colList, + pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId); + return pOperator; + } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { + STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*) pPhyNode; + SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); + + int32_t code = + doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } + + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pScanPhyNode->pScanPseudoCols, NULL, &num); + + int32_t numOfOutputCols = 0; + SArray* colList = + extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfOutputCols); + + SOperatorInfo* pOperator = createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableGroupInfo, pTaskInfo); return pOperator; } else { ASSERT(0); @@ -6595,31 +4862,34 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STimeWindowAggSupp as = {.waterMark = pIntervalPhyNode->window.watermark, .calTrigger = pIntervalPhyNode->window.triggerType}; - int32_t primaryTsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; - pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, primaryTsSlotId, &as, - pTableGroupInfo, pTaskInfo); - - if (pIntervalPhyNode->pFill != NULL) { - pOptr = createFillOperatorInfo(pOptr, pExprInfo, num, &interval, pResBlock, pIntervalPhyNode->pFill->mode, NULL, - false, pTaskInfo); - } + int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; + pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTableGroupInfo, + pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { SSortPhysiNode* pSortPhyNode = (SSortPhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - SArray* info = createSortInfo(pSortPhyNode->pSortKeys, pSortPhyNode->pTargets); - SArray* slotMap = createIndexMap(pSortPhyNode->pTargets); - pOptr = createSortOperatorInfo(ops[0], pResBlock, info, slotMap, pTaskInfo); + SArray* info = createSortInfo(pSortPhyNode->pSortKeys); + + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols); + + int32_t numOfOutputCols = 0; + SArray* pColList = + extractColMatchInfo(pSortPhyNode->pTargets, pSortPhyNode->node.pOutputDataBlockDesc, &numOfOutputCols); + + pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, pColList, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; - STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark, - .calTrigger = pSessionNode->window.triggerType}; + STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType}; SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, &as, pTaskInfo); + int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId; + + pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { SPartitionPhysiNode* pPartNode = (SPartitionPhysiNode*)pPhyNode; SArray* pColList = extractPartitionColInfo(pPartNode->pPartitionKeys); @@ -6634,13 +4904,22 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, pTaskInfo); + int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; + + pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_JOIN == type) { SJoinPhysiNode* pJoinNode = (SJoinPhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); SExprInfo* pExprInfo = createExprInfo(pJoinNode->pTargets, NULL, &num); pOptr = createJoinOperatorInfo(ops, size, pExprInfo, num, pResBlock, pJoinNode->pOnConditions, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) { + SFillPhysiNode* pFillNode = (SFillPhysiNode*)pPhyNode; + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + SExprInfo* pExprInfo = createExprInfo(pFillNode->pTargets, NULL, &num); + + SInterval* pInterval = &((SIntervalAggOperatorInfo*)ops[0]->info)->interval; + pOptr = createFillOperatorInfo(ops[0], pExprInfo, num, pInterval, &pFillNode->timeRange, pResBlock, pFillNode->mode, (SNodeListNode*)pFillNode->pValues, false, pTaskInfo); } else { ASSERT(0); } @@ -6649,8 +4928,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return pOptr; } - -static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) { +static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) { pCond->loadExternalRows = false; pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; @@ -6664,7 +4942,7 @@ static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableS pCond->twindow = pTableScanNode->scanRange; #if 1 - //todo work around a problem, remove it later + // todo work around a problem, remove it later if ((pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey > pCond->twindow.ekey) || (pCond->order == TSDB_ORDER_DESC && pCond->twindow.skey < pCond->twindow.ekey)) { TSWAP(pCond->twindow.skey, pCond->twindow.ekey); @@ -6708,22 +4986,22 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { // todo extract method SColumn c = {0}; - c.slotId = pColNode->slotId; - c.colId = pColNode->colId; - c.type = pColNode->node.resType.type; - c.bytes = pColNode->node.resType.bytes; - c.scale = pColNode->node.resType.scale; + c.slotId = pColNode->slotId; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; + c.scale = pColNode->node.resType.scale; c.precision = pColNode->node.resType.precision; taosArrayPush(pList, &c); } else if (nodeType(pNode->pExpr) == QUERY_NODE_VALUE) { - SValueNode* pValNode = (SValueNode*) pNode->pExpr; - SColumn c = {0}; + SValueNode* pValNode = (SValueNode*)pNode->pExpr; + SColumn c = {0}; c.slotId = pNode->slotId; - c.colId = pNode->slotId; - c.type = pValNode->node.type; - c.bytes = pValNode->node.resType.bytes; - c.scale = pValNode->node.resType.scale; + c.colId = pNode->slotId; + c.type = pValNode->node.type; + c.bytes = pValNode->node.resType.bytes; + c.scale = pValNode->node.resType.scale; c.precision = pValNode->node.resType.precision; taosArrayPush(pList, &c); @@ -6759,7 +5037,7 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList) { return pList; } -SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) { +SArray* createSortInfo(SNodeList* pNodeList) { size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo)); if (pList == NULL) { @@ -6774,22 +5052,7 @@ SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) { bi.nullFirst = (pSortKey->nullOrder == NULL_ORDER_FIRST); SColumnNode* pColNode = (SColumnNode*)pSortKey->pExpr; - - bool found = false; - for (int32_t j = 0; j < LIST_LENGTH(pNodeListTarget); ++j) { - STargetNode* pTarget = (STargetNode*)nodesListGetNode(pNodeListTarget, j); - - SColumnNode* pColNodeT = (SColumnNode*)pTarget->pExpr; - if (pColNode->slotId == pColNodeT->slotId) { // to find slotId in PhysiSort OutputDataBlockDesc - bi.slotId = pTarget->slotId; - found = true; - break; - } - } - - if (!found) { - qError("sort slot id does not found"); - } + bi.slotId = pColNode->slotId; taosArrayPush(pList, &bi); } @@ -6828,7 +5091,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod SColMatchInfo c = {0}; c.output = true; - c.colId = pColNode->colId; + c.colId = pColNode->colId; c.targetSlotId = pNode->slotId; taosArrayPush(pList, &c); } @@ -6907,7 +5170,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* goto _error; } - return tsdbQueryTables(pHandle->reader, &cond, pTableGroupInfo, queryId, taskId); + return tsdbQueryTables(pHandle->vnode, &cond, pTableGroupInfo, queryId, taskId); _error: terrno = code; @@ -6925,7 +5188,8 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead goto _complete; } - (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoGroupInfo); + (*pTaskInfo)->pRoot = + createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoGroupInfo); if (NULL == (*pTaskInfo)->pRoot) { code = terrno; goto _complete; @@ -7140,9 +5404,8 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo return TSDB_CODE_SUCCESS; } -static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { SJoinOperatorInfo* pJoinInfo = pOperator->info; - // SOptrBasicInfo* pInfo = &pJoinInfo->binfo; SSDataBlock* pRes = pJoinInfo->pRes; blockDataCleanup(pRes); @@ -7151,12 +5414,10 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) int32_t nrows = 0; while (1) { - bool prevVal = *newgroup; - if (pJoinInfo->pLeft == NULL || pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { SOperatorInfo* ds1 = pOperator->pDownstream[0]; publishOperatorProfEvent(ds1, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1, newgroup); + pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1); publishOperatorProfEvent(ds1, QUERY_PROF_AFTER_OPERATOR_EXEC); pJoinInfo->leftPos = 0; @@ -7169,7 +5430,7 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) if (pJoinInfo->pRight == NULL || pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { SOperatorInfo* ds2 = pOperator->pDownstream[1]; publishOperatorProfEvent(ds2, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2, newgroup); + pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2); publishOperatorProfEvent(ds2, QUERY_PROF_AFTER_OPERATOR_EXEC); pJoinInfo->rightPos = 0; @@ -7188,7 +5449,7 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) // only the timestamp match support for ordinary table ASSERT(pLeftCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); if (*(int64_t*)pLeftVal == *(int64_t*)pRightVal) { - for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { + for (int32_t i = 0; i < pOperator->numOfExprs; ++i) { SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, i); SExprInfo* pExprInfo = &pOperator->pExpr[i]; @@ -7247,25 +5508,29 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOf goto _error; } - pOperator->resultInfo.capacity = 4096; - pOperator->resultInfo.threshold = 4096 * 0.75; - - // initResultRowInf - // o(&pInfo->binfo.resultRowInfo, 8); - pInfo->pRes = pResBlock; + initResultSizeInfo(pOperator, 4096); - pOperator->name = "JoinOperator"; + pInfo->pRes = pResBlock; + pOperator->name = "MergeJoinOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfExprs = numOfCols; + pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyBasicOperatorInfo, - NULL, NULL, NULL); + SOperatorNode* pNode = (SOperatorNode*)pOnCondition; + setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft); + setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight); + + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyBasicOperatorInfo, NULL, NULL, NULL); int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + return pOperator; _error: @@ -7274,3 +5539,11 @@ _error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + +void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { + pColumn->slotId = pColumnNode->slotId; + pColumn->type = pColumnNode->node.resType.type; + pColumn->bytes = pColumnNode->node.resType.bytes; + pColumn->precision = pColumnNode->node.resType.precision; + pColumn->scale = pColumnNode->node.resType.scale; +} diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index beee11ec189680adaf5388d44e56bc523e5ca91a..58efd75a0bf6dc2129950fbc5ecd80e0a15bd0dc 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -77,7 +77,7 @@ static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlo SColumn* pCol = taosArrayGet(pGroupCols, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); if (pBlock->pBlockAgg != NULL) { - pColAgg = &pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? } bool isNull = colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg); @@ -118,7 +118,7 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); if (pBlock->pBlockAgg != NULL) { - pColAgg = &pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? } SGroupKeys* pkey = taosArrayGet(pGroupColVals, i); @@ -227,16 +227,16 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); - int32_t ret = setGroupResultOutputBuf(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, 0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); + int32_t ret = setGroupResultOutputBuf(&(pInfo->binfo), pOperator->numOfExprs, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, 0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } int32_t rowIndex = j - num; - doApplyFunctions(pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfExprs, TSDB_ORDER_ASC); // assign the group keys or user input constant values if required - doAssignGroupKeys(pCtx, pOperator->numOfOutput, pBlock->info.rows, rowIndex); + doAssignGroupKeys(pCtx, pOperator->numOfExprs, pBlock->info.rows, rowIndex); recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); num = 1; } @@ -244,19 +244,19 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { if (num > 0) { len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); int32_t ret = - setGroupResultOutputBuf(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, + setGroupResultOutputBuf(&(pInfo->binfo), pOperator->numOfExprs, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, 0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } int32_t rowIndex = pBlock->info.rows - num; - doApplyFunctions(pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfOutput, TSDB_ORDER_ASC); - doAssignGroupKeys(pCtx, pOperator->numOfOutput, pBlock->info.rows, rowIndex); + doApplyFunctions(pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfExprs, TSDB_ORDER_ASC); + doAssignGroupKeys(pCtx, pOperator->numOfExprs, pBlock->info.rows, rowIndex); } } -static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -265,7 +265,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -277,7 +277,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -291,19 +291,19 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou projectApplyFunctions(pInfo->pScalarExprInfo, pBlock, pBlock, pInfo->pScalarFuncCtx, pInfo->numOfScalarExpr, NULL); } - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); + // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfExprs); doHashGroupbyAgg(pOperator, pBlock); } pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pInfo->binfo.resultRowInfo); - finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, + finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); // if (!stableQuery) { // finalize include the update of result rows - // finalizeQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput); + // finalizeQueryResult(pInfo->binfo.pCtx, pOperator->numOfExprs); // } else { - // updateNumOfRowsInResultRows(pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, + // updateNumOfRowsInResultRows(pInfo->binfo.pCtx, pOperator->numOfExprs, &pInfo->binfo.resultRowInfo, // pInfo->binfo.rowCellInfoOffset); // } @@ -311,7 +311,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, false); while(1) { - doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); doFilter(pInfo->pCondition, pRes); bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo); @@ -353,11 +353,11 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); pOperator->name = "GroupbyAggOperator"; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; // pOperator->operatorType = OP_Groupby; pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; + pOperator->numOfExprs = numOfCols; pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; @@ -392,7 +392,7 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t* rows = (int32_t*) pPage; - size_t numOfCols = pOperator->numOfOutput; + size_t numOfCols = pOperator->numOfExprs; for(int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExpr = &pOperator->pExpr[i]; int32_t slotId = pExpr->base.pParam[0].pCol->slotId; @@ -537,11 +537,12 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { pInfo->pageIndex += 1; + blockDataUpdateTsWindow(pInfo->binfo.pRes); pInfo->binfo.pRes->info.groupId = pGroupInfo->groupId; return pInfo->binfo.pRes; } -static SSDataBlock* hashPartition(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -558,13 +559,13 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator, bool* newgroup) { while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; } - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); + // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfExprs); doHashPartition(pOperator, pBlock); } @@ -611,11 +612,11 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* } pOperator->name = "PartitionOperator"; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; pInfo->binfo.pRes = pResultBlock; - pOperator->numOfOutput = numOfCols; + pOperator->numOfExprs = numOfCols; pOperator->pExpr = pExprInfo; pOperator->info = pInfo; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8291826e69aa1bf99639ec0e0e1649c011b95b75..b728daa3bba0e4b28f48f6ab5c5084cd3ecc6a56 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,15 +13,16 @@ * along with this program. If not, see . */ -#include +#include #include "filter.h" #include "function.h" #include "functionMgt.h" #include "os.h" #include "querynodes.h" +#include "systable.h" #include "tglobal.h" #include "tname.h" -#include "vnode.h" +#include "ttime.h" #include "tdatablock.h" #include "tmsg.h" @@ -36,7 +37,10 @@ #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) -void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { +static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); +static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, const char* dbName); + +static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t i = 0; i < numOfOutput; ++i) { SWITCH_ORDER(pCtx[i].order); } @@ -65,21 +69,6 @@ static void setupQueryRangeForReverseScan(STableScanInfo* pTableScanInfo) { #endif } -// relocated the column data according to the slotId -static void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols) { - int32_t numOfCols = pBlock->info.numOfCols; - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* p = taosArrayGet(pCols, i); - SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); - if (!pmInfo->output) { - continue; - } - - ASSERT(pmInfo->colId == p->info.colId); - taosArraySet(pBlock->pDataBlock, pmInfo->targetSlotId, p); - } -} - static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) { int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { @@ -170,11 +159,11 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn return false; } -int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { +static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableScanInfo* pInfo = pOperator->info; - STaskCostInfo* pCost = &pTaskInfo->cost; + SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder; pCost->totalBlocks += 1; pCost->totalRows += pBlock->info.rows; @@ -200,14 +189,18 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; - SColumnDataAgg* pColAgg = NULL; - tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg); + bool allColumnsHaveAgg = true; + SColumnDataAgg** pColAgg = NULL; + tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); - if (pColAgg != NULL) { + if (allColumnsHaveAgg == true) { int32_t numOfCols = pBlock->info.numOfCols; // todo create this buffer during creating operator - pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, sizeof(SColumnDataAgg)); + if (pBlock->pBlockAgg == NULL) { + pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES); + } + for (int32_t i = 0; i < numOfCols; ++i) { SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i); if (!pColMatchInfo->output) { @@ -244,11 +237,7 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, } relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols); - - // reset the block to be 0 by default, this blockId is assigned by physical plan and is used by direct upstream - // operator. - pBlock->info.blockId = 0; - + // todo record the filter time cost doFilter(pTableScanInfo->pFilterNode, pBlock); if (pBlock->info.rows == 0) { pCost->filterOutBlocks += 1; @@ -270,18 +259,15 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction pTableScanInfo->cond.order = TSDB_ORDER_DESC; } -static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; - SSDataBlock* pBlock = pTableScanInfo->pResBlock; - *newgroup = false; while (tsdbNextDataBlock(pTableScanInfo->dataReader)) { if (isTaskKilled(pOperator->pTaskInfo)) { longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } - pTableScanInfo->numOfBlocks += 1; tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info); uint32_t status = 0; @@ -302,7 +288,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { return NULL; } -static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -311,17 +297,16 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { return NULL; } - *newgroup = false; - - while (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) { - SSDataBlock* p = doTableScanImpl(pOperator, newgroup); + // do the ascending order traverse in the first place. + while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { + SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { return p; } - pTableScanInfo->current += 1; + pTableScanInfo->scanTimes += 1; - if (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) { + if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; @@ -336,7 +321,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { } int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc; - if (pTableScanInfo->current < total) { + if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) { prepareForDescendingScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond); @@ -346,21 +331,20 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { qDebug("%s start to descending order scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); - while (pTableScanInfo->current < total) { - SSDataBlock* p = doTableScanImpl(pOperator, newgroup); + while (pTableScanInfo->scanTimes < total) { + SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { return p; } - pTableScanInfo->current += 1; + pTableScanInfo->scanTimes += 1; - if (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) { + if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; qDebug("%s start to repeat descending order scan data blocks due to query func required, qrange:%" PRId64 - "-%" PRId64, - GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); + "-%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); // do prepare for the next round table scan operation tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond); @@ -389,58 +373,59 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCon pInfo->cond = *pCond; pInfo->scanInfo = (SScanInfo){.numOfAsc = scanInfo[0], .numOfDesc = scanInfo[1]}; - pInfo->interval = *pInterval; - pInfo->sampleRatio = sampleRatio; + pInfo->interval = *pInterval; + pInfo->sampleRatio = sampleRatio; pInfo->dataBlockLoadFlag = dataLoadFlag; - pInfo->pResBlock = pResBlock; - pInfo->pFilterNode = pCondition; - pInfo->dataReader = pDataReader; - pInfo->current = 0; - pInfo->scanFlag = MAIN_SCAN; - pInfo->pColMatchInfo = pColMatchInfo; - - pOperator->name = "TableScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->fpSet.getNextFn = doTableScan; - pOperator->pTaskInfo = pTaskInfo; + pInfo->pResBlock = pResBlock; + pInfo->pFilterNode = pCondition; + pInfo->dataReader = pDataReader; + pInfo->scanFlag = MAIN_SCAN; + pInfo->pColMatchInfo = pColMatchInfo; + + pOperator->name = "TableScanOperator"; // for dubug purpose + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfExprs = numOfOutput; + pOperator->pTaskInfo = pTaskInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, NULL, NULL, NULL, NULL); static int32_t cost = 0; - pOperator->cost.openCost = ++cost; + + // for non-blocking operator, the open cost is always 0 + pOperator->cost.openCost = 0; pOperator->cost.totalCost = ++cost; pOperator->resultInfo.totalRows = ++cost; return pOperator; } -SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle) { +SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pTaskInfo) { STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - pInfo->dataReader = pTsdbReadHandle; - pInfo->current = 0; - pInfo->prevGroupId = -1; + pInfo->dataReader = pReadHandle; +// pInfo->prevGroupId = -1; - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableSeqScanOperator"; + pOperator->name = "TableSeqScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->fpSet.getNextFn = doTableScanImpl; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL, NULL, NULL, NULL); return pOperator; } -static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } STableScanInfo* pTableScanInfo = pOperator->info; - *newgroup = false; STableBlockDistInfo tableBlockDist = {0}; tableBlockDist.numOfTables = 1; // TODO set the correct number of tables @@ -500,7 +485,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pOperator->name = "DataBlockInfoScanOperator"; // pOperator->operatorType = OP_TableBlockInfoScan; - pOperator->blockingOptr = false; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->fpSet._openFn = operatorDummyOpenFn; pOperator->fpSet.getNextFn = doBlockInfoScan; @@ -527,10 +512,11 @@ static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) { taosArrayClear(pInfo->pBlockLists); } -static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamBlockScanInfo* pInfo = pOperator->info; + int32_t rows = 0; pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS || pOperator->status == OP_EXEC_DONE) { @@ -597,6 +583,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) pTaskInfo->code = terrno; return NULL; } + rows = pBlockInfo->rows; + doFilter(pInfo->pCondition, pInfo->pRes); break; } @@ -605,16 +593,16 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) pInfo->numOfExec++; pInfo->numOfRows += pBlockInfo->rows; - if (pBlockInfo->rows == 0) { + if (rows == 0) { pOperator->status = OP_EXEC_DONE; } - return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; + return (rows == 0) ? NULL : pInfo->pRes; } } SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, - SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { + SArray* pTableIdList, SExecTaskInfo* pTaskInfo, SNode* pCondition) { SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -651,19 +639,22 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* } pInfo->readerHandle = streamReadHandle; - pInfo->pRes = pResBlock; - - pOperator->name = "StreamBlockScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->fpSet._openFn = operatorDummyOpenFn; + pInfo->pRes = pResBlock; + pInfo->pCondition = pCondition; + + pOperator->name = "StreamBlockScanOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfExprs = pResBlock->info.numOfCols; + pOperator->fpSet._openFn = operatorDummyOpenFn; pOperator->fpSet.getNextFn = doStreamBlockScan; - pOperator->fpSet.closeFn = operatorDummyCloseFn; + pOperator->fpSet.closeFn = operatorDummyCloseFn; pOperator->pTaskInfo = pTaskInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamBlockScan, NULL, NULL, operatorDummyCloseFn, NULL, NULL, NULL); + return pOperator; } @@ -685,14 +676,12 @@ EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { switch (nType) { case QUERY_NODE_OPERATOR: { SOperatorNode* node = (SOperatorNode*)pNode; - if (OP_TYPE_EQUAL == node->opType) { *(int32_t*)pContext = 1; return DEAL_RES_CONTINUE; } *(int32_t*)pContext = 0; - return DEAL_RES_IGNORE_CHILD; } case QUERY_NODE_COLUMN: { @@ -723,19 +712,17 @@ EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { default: break; } - return DEAL_RES_CONTINUE; } -void getDBNameFromCondition(SNode* pCondition, char* dbName) { +static void getDBNameFromCondition(SNode* pCondition, const char* dbName) { if (NULL == pCondition) { return; } - - nodesWalkExpr(pCondition, getDBNameFromConditionWalker, dbName); + nodesWalkExpr(pCondition, getDBNameFromConditionWalker, (char*) dbName); } -static int32_t loadSysTableContentCb(void* param, const SDataBuf* pMsg, int32_t code) { +static int32_t loadSysTableCallback(void* param, const SDataBuf* pMsg, int32_t code) { SOperatorInfo* operator=(SOperatorInfo*) param; SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo*)operator->info; if (TSDB_CODE_SUCCESS == code) { @@ -760,13 +747,14 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { } SFilterInfo* filter = NULL; - int32_t code = filterInitFromNode(pInfo->pCondition, &filter, 0); + + int32_t code = filterInitFromNode(pInfo->pCondition, &filter, 0); SFilterColumnParam param1 = {.numOfCols = pInfo->pRes->info.numOfCols, .pDataBlock = pInfo->pRes->pDataBlock}; code = filterSetDataFromSlotId(filter, ¶m1); int8_t* rowRes = NULL; - bool keep = filterExecute(filter, pInfo->pRes, &rowRes, NULL, param1.numOfCols); + bool keep = filterExecute(filter, pInfo->pRes, &rowRes, NULL, param1.numOfCols); filterFreeInfo(filter); SSDataBlock* px = createOneDataBlock(pInfo->pRes, false); @@ -788,7 +776,12 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { continue; } - colDataAppend(pDest, numOfRow, colDataGetData(pSrc, j), false); + if (colDataIsNull_s(pSrc, j)) { + colDataAppendNULL(pDest, numOfRow); + } else { + colDataAppend(pDest, numOfRow, colDataGetData(pSrc, j), false); + } + numOfRow += 1; } } else { @@ -802,123 +795,174 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } -static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { - // build message and send to mnode to fetch the content of system tables. - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; +static SSDataBlock* buildSysTableMetaBlock() { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - // retrieve local table list info from vnode - const char* name = tNameGetTableName(&pInfo->name); - if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { - if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + size_t size = 0; + const SSysTableMeta *pMeta = NULL; + getInfosDbMeta(&pMeta, &size); + + int32_t index = 0; + for(int32_t i = 0; i < size; ++i) { + if(strcmp(pMeta[i].name, TSDB_INS_TABLE_USER_TABLES) == 0) { + index = i; + break; } + } - blockDataCleanup(pInfo->pRes); + pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - int32_t tableNameSlotId = 1; - SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, tableNameSlotId); + for(int32_t i = 0; i < pMeta[index].colNum; ++i) { + SColumnInfoData colInfoData = {0}; + colInfoData.info.colId = i + 1; + colInfoData.info.type = pMeta[index].schema[i].type; + colInfoData.info.bytes = pMeta[index].schema[i].bytes; + taosArrayPush(pBlock->pDataBlock, &colInfoData); + } - int32_t numOfRows = 0; + pBlock->info.numOfCols = pMeta[index].colNum; + pBlock->info.hasVarCol = true; - const char* db = NULL; - int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + return pBlock; +} - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT|T_NAME_DB); +static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { + // build message and send to mnode to fetch the content of system tables. + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); + // retrieve local table list info from vnode + const char* name = tNameGetTableName(&pInfo->name); + if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { + // the retrieve is executed on the mnode, so return tables that belongs to the information schema database. + if (pInfo->readHandle.mnd != NULL) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - char n[TSDB_TABLE_NAME_LEN] = {0}; - while (metaTbCursorNext(pInfo->pCur) == 0) { - STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); - colDataAppend(pTableNameCol, numOfRows, n, false); + buildSysDbTableInfo(pInfo, pOperator->resultInfo.capacity); - int32_t tableType = pInfo->pCur->mr.me.type; + doFilterResult(pInfo); + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - // database name - SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, dbname, false); + pOperator->status = OP_EXEC_DONE; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } else { + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + } - // vgId - pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 6); - colDataAppend(pColInfoData, numOfRows, (char*) &vgId, false); + blockDataCleanup(pInfo->pRes); - // table comment - // todo: set the correct comment - pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 8); - colDataAppendNULL(pColInfoData, numOfRows); + int32_t numOfRows = 0; - char str[256] = {0}; - if (tableType == TSDB_CHILD_TABLE) { - SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, 0); - metaGetTableEntryByUid(&mr, pInfo->pCur->mr.me.ctbEntry.suid); + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*) &mr.me.stbEntry.schema.nCols, false); + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - // create time - int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; - pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*) &ts, false); + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); - // super table name - STR_TO_VARSTR(str, mr.me.name); - pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 4); - colDataAppend(pColInfoData, numOfRows, str, false); - metaReaderClear(&mr); + SSDataBlock* p = buildSysTableMetaBlock(); + blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); - // uid - pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*) &pInfo->pCur->mr.me.uid, false); + char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + while (metaTbCursorNext(pInfo->pCur) == 0) { + STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); - // ttl - pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*) &pInfo->pCur->mr.me.ctbEntry.ttlDays, false); + // table name + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, n, false); - STR_TO_VARSTR(str, "CHILD_TABLE"); - } else if (tableType == TSDB_NORMAL_TABLE) { - // create time - pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*) &pInfo->pCur->mr.me.ntbEntry.ctime, false); + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); - // number of columns - pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*) &pInfo->pCur->mr.me.ntbEntry.schema.nCols, false); + // vgId + pColInfoData = taosArrayGet(p->pDataBlock, 6); + colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); - // super table name - pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 4); + // table comment + // todo: set the correct comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); colDataAppendNULL(pColInfoData, numOfRows); - // uid - pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*) &pInfo->pCur->mr.me.uid, false); + char str[256] = {0}; + int32_t tableType = pInfo->pCur->mr.me.type; + if (tableType == TSDB_CHILD_TABLE) { + // create time + int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); + + SMetaReader mr = {0}; + metaReaderInit(&mr, pInfo->readHandle.meta, 0); + metaGetTableEntryByUid(&mr, pInfo->pCur->mr.me.ctbEntry.suid); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schema.nCols, false); + + // super table name + STR_TO_VARSTR(str, mr.me.name); + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppend(pColInfoData, numOfRows, str, false); + metaReaderClear(&mr); + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); + + STR_TO_VARSTR(str, "CHILD_TABLE"); + } else if (tableType == TSDB_NORMAL_TABLE) { + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schema.nCols, false); + + // super table name + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppendNULL(pColInfoData, numOfRows); + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); + + STR_TO_VARSTR(str, "NORMAL_TABLE"); + } - // ttl - pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*) &pInfo->pCur->mr.me.ntbEntry.ttlDays, false); + pColInfoData = taosArrayGet(p->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, str, false); - STR_TO_VARSTR(str, "NORMAL_TABLE"); + if (++numOfRows >= pOperator->resultInfo.capacity) { + break; + } } - pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, str, false); - - if (++numOfRows >= pInfo->capacity) { - break; - } - } + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; - pInfo->loadInfo.totalRows += numOfRows; - pInfo->pRes->info.rows = numOfRows; + relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock); + doFilterResult(pInfo); - // pInfo->elapsedTime; - // pInfo->totalBytes; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } } else { // load the meta from mnode of the given epset if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -950,7 +994,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { pMsgSendInfo->msgInfo.pData = buf1; pMsgSendInfo->msgInfo.len = contLen; pMsgSendInfo->msgType = TDMT_MND_SYSTABLE_RETRIEVE; - pMsgSendInfo->fp = loadSysTableContentCb; + pMsgSendInfo->fp = loadSysTableCallback; int64_t transporterId = 0; int32_t code = asyncSendMsgToServer(pInfo->pTransporter, &pInfo->epSet, &transporterId, pMsgSendInfo); @@ -977,7 +1021,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { SRetrieveMetaTableRsp* pTableRsp = pInfo->pRsp; setSDataBlockFromFetchRsp(pInfo->pRes, &pInfo->loadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pOperator->numOfOutput, startTs, NULL, pInfo->scanCols); + pTableRsp->compLen, pOperator->numOfExprs, startTs, NULL, pInfo->scanCols); // todo log the filter info doFilterResult(pInfo); @@ -988,6 +1032,67 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { } } +int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { + SSDataBlock* p = buildSysTableMetaBlock(); + blockDataEnsureCapacity(p, capacity); + + size_t size = 0; + const SSysTableMeta* pSysDbTableMeta = NULL; + + getInfosDbMeta(&pSysDbTableMeta, &size); + p->info.rows = buildDbTableInfoBlock(p, pSysDbTableMeta, size, TSDB_INFORMATION_SCHEMA_DB); + + getPerfDbMeta(&pSysDbTableMeta, &size); + p->info.rows = buildDbTableInfoBlock(p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB); + + relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock); +// blockDataDestroy(p); todo handle memory leak + + pInfo->pRes->info.rows = p->info.rows; + return p->info.rows; +} + +int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, const char* dbName) { + char n[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t numOfRows = p->info.rows; + + for(int32_t i = 0; i < size; ++i) { + const SSysTableMeta* pm = &pSysDbTableMeta[i]; + + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + + STR_TO_VARSTR(n, pm->name); + colDataAppend(pColInfoData, numOfRows, n, false); + + // database name + STR_TO_VARSTR(n, dbName); + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, n, false); + + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppendNULL(pColInfoData, numOfRows); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&pm->colNum, false); + + for(int32_t j = 4; j <= 8; ++j) { + pColInfoData = taosArrayGet(p->pDataBlock, j); + colDataAppendNULL(pColInfoData, numOfRows); + } + + STR_TO_VARSTR(n, "SYSTEM_TABLE"); + + pColInfoData = taosArrayGet(p->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, n, false); + + numOfRows += 1; + } + + return numOfRows; +} + SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pResBlock, const SName* pName, SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId) { @@ -1000,18 +1105,19 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe return NULL; } - pInfo->accountId = accountId; + pInfo->accountId = accountId; pInfo->showRewrite = showRewrite; - pInfo->pRes = pResBlock; - pInfo->capacity = 4096; - pInfo->pCondition = pCondition; - pInfo->scanCols = colList; + pInfo->pRes = pResBlock; + pInfo->pCondition = pCondition; + pInfo->scanCols = colList; + + initResultSizeInfo(pOperator, 4096); tNameAssign(&pInfo->name, pName); const char* name = tNameGetTableName(&pInfo->name); if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { pInfo->readHandle = *(SReadHandle*) readHandle; - blockDataEnsureCapacity(pInfo->pRes, pInfo->capacity); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); } else { tsem_init(&pInfo->ready, 0, 0); pInfo->epSet = epset; @@ -1040,12 +1146,12 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe #endif } - pOperator->name = "SysTableScanOperator"; + pOperator->name = "SysTableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pResBlock->info.numOfCols; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfExprs = pResBlock->info.numOfCols; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, NULL, NULL, NULL); pOperator->pTaskInfo = pTaskInfo; @@ -1053,18 +1159,18 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe return pOperator; } -static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) { -#if 0 - SOperatorInfo* pOperator = (SOperatorInfo*) param; +static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + +#if 0 int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; STagScanInfo *pInfo = pOperator->info; SSDataBlock *pRes = pInfo->pRes; - *newgroup = false; int32_t count = 0; SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); @@ -1133,55 +1239,54 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) { pOperator->status = OP_EXEC_DONE; //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); } else { // return only the tags|table name etc. - SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo +#endif - count = 0; - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; + STagScanInfo* pInfo = pOperator->info; + SExprInfo* pExprInfo = &pOperator->pExpr[0]; + SSDataBlock* pRes = pInfo->pRes; - STableQueryInfo* item = taosArrayGetP(pa, i); + SArray* pa = taosArrayGetP(pInfo->pTableGroups->pGroupList, 0); - char *data = NULL, *dst = NULL; - int16_t type = 0, bytes = 0; - for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { - // not assign value in case of user defined constant output column - if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) { - continue; - } + char str[512] = {0}; + int32_t count = 0; + SMetaReader mr = {0}; - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); - type = pExprInfo[j].base.resSchema.type; - bytes = pExprInfo[j].base.resSchema.bytes; + while (pInfo->curPos < pInfo->pTableGroups->numOfTables && count < pOperator->resultInfo.capacity) { + STableKeyInfo* item = taosArrayGet(pa, pInfo->curPos); - if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); - } + for (int32_t j = 0; j < pOperator->numOfExprs; ++j) { + SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId); - dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; - doSetTagValueToResultBuf(dst, data, type, bytes); + // refactor later + if (fmIsScanPseudoColumnFunc(pExprInfo[j].pExpr->_function.functionId)) { + metaReaderInit(&mr, pInfo->readHandle.meta, 0); + metaGetTableEntryByUid(&mr, item->uid); + + STR_TO_VARSTR(str, mr.me.name); + metaReaderClear(&mr); + + colDataAppend(pDst, count, str, false); + + // data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); + // dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; + // doSetTagValueToResultBuf(dst, data, type, bytes); } count += 1; } - if (pInfo->curPos >= pInfo->totalTables) { + if (++pInfo->curPos >= pInfo->pTableGroups->numOfTables) { pOperator->status = OP_EXEC_DONE; } - - //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); } + // qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); if (pOperator->status == OP_EXEC_DONE) { - setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED); + setTaskStatus(pTaskInfo, TASK_COMPLETED); } pRes->info.rows = count; - return (pRes->info.rows == 0)? NULL:pInfo->pRes; - -#endif - return TSDB_CODE_SUCCESS; + return (pRes->info.rows == 0) ? NULL : pInfo->pRes; } static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { @@ -1189,29 +1294,36 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, - SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, + SSDataBlock* pResBlock, SArray* pColMatchInfo, STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->pReader = pReaderHandle; - pInfo->curPos = 0; - pOperator->name = "TagScanOperator"; + pInfo->pTableGroups = pTableGroupInfo; + pInfo->pColMatchInfo = pColMatchInfo; + pInfo->pRes = pResBlock; + pInfo->readHandle = *pReadHandle; + pInfo->curPos = 0; + pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfExprs = numOfOutput; + pOperator->pTaskInfo = pTaskInfo; + + initResultSizeInfo(pOperator, 4096); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, NULL, destroyTagScanOperatorInfo, NULL, NULL, NULL); - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pTaskInfo = pTaskInfo; return pOperator; + _error: taosMemoryFree(pInfo); taosMemoryFree(pOperator); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c new file mode 100644 index 0000000000000000000000000000000000000000..619651f11f9d03d1e712dd721909d0a1e15cfa2c --- /dev/null +++ b/source/libs/executor/src/sortoperator.c @@ -0,0 +1,160 @@ +#include "tdatablock.h" +#include "executorimpl.h" + +static SSDataBlock* doSort(SOperatorInfo* pOperator); +static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); + +SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols, + SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) { + SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + int32_t rowSize = pResBlock->info.rowSize; + + if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) { + goto _error; + } + + pOperator->pExpr = pExprInfo; + pOperator->numOfExprs = numOfCols; + pInfo->binfo.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = pResBlock; + + initResultSizeInfo(pOperator, 1024); + + pInfo->pSortInfo = pSortInfo; + pInfo->pColMatchInfo= pColMatchColInfo; + pOperator->name = "SortOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + + // lazy evaluation for the following parameter since the input datablock is not known till now. +// pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + header +// pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer + + pOperator->pTaskInfo = pTaskInfo; + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL, NULL); + + int32_t code = appendDownstream(pOperator, &downstream, 1); + return pOperator; + + _error: + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + return NULL; +} + +void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + bool isNull = tsortIsNullVal(pTupleHandle, i); + if (isNull) { + colDataAppendNULL(pColInfo, pBlock->info.rows); + } else { + char* pData = tsortGetValue(pTupleHandle, i); + colDataAppend(pColInfo, pBlock->info.rows, pData, false); + } + } + + pBlock->info.rows += 1; +} + +SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo) { + blockDataCleanup(pDataBlock); + ASSERT(taosArrayGetSize(pColMatchInfo) == pDataBlock->info.numOfCols); + + SSDataBlock* p = tsortGetSortedDataBlock(pHandle); + blockDataEnsureCapacity(p, capacity); + + while (1) { + STupleHandle* pTupleHandle = tsortNextTuple(pHandle); + if (pTupleHandle == NULL) { + break; + } + + appendOneRowToDataBlock(p, pTupleHandle); + if (p->info.rows >= capacity) { + return pDataBlock; + } + } + + if (p->info.rows > 0) { + int32_t numOfCols = taosArrayGetSize(pColMatchInfo); + for(int32_t i = 0; i < numOfCols; ++i) { + SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); + + for(int32_t j = 0; j < p->info.numOfCols; ++j) { + SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, j); + if (pSrc->info.colId == pmInfo->colId) { + SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); + colDataAssign(pDst, pSrc, p->info.rows); + break; + } + } + } + + pDataBlock->info.rows = p->info.rows; + pDataBlock->info.capacity = p->info.rows; + } + + blockDataDestroy(p); + return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; +} + +SSDataBlock* loadNextDataBlock(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*)param; + return pOperator->fpSet.getNextFn(pOperator); +} + +// todo refactor: merged with fetch fp +void applyScalarFunction(SSDataBlock* pBlock, void* param) { + SOperatorInfo* pOperator = param; + SSortOperatorInfo* pSort = pOperator->info; + if (pOperator->pExpr != NULL) { + projectApplyFunctions(pOperator->pExpr, pBlock, pBlock, pSort->binfo.pCtx, pOperator->numOfExprs, NULL); + } +} + +SSDataBlock* doSort(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSortOperatorInfo* pInfo = pOperator->info; + + if (pOperator->status == OP_RES_TO_RETURN) { + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); + } + +// pInfo->binfo.pRes is not equalled to the input datablock. +// int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_SINGLESOURCE_SORT, + -1, -1, NULL, pTaskInfo->id.str); + + tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator); + + SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); + ps->param = pOperator->pDownstream[0]; + tsortAddSource(pInfo->pSortHandle, ps); + + int32_t code = tsortOpen(pInfo->pSortHandle); + taosMemoryFreeClear(ps); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, terrno); + } + + pOperator->status = OP_RES_TO_RETURN; + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); +} + +void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { + SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param; + pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); + + taosArrayDestroy(pInfo->pSortInfo); + taosArrayDestroy(pInfo->pColMatchInfo); +} diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index aeed07c636cdd31cf31a50441f9c85f985b60fc4..018f0016a22db237b70bdd43135492d835147ee1 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -13,20 +13,21 @@ * along with this program. If not, see . */ -#include "function.h" #include "os.h" -#include "querynodes.h" - #include "taosdef.h" #include "tmsg.h" #include "ttypes.h" -#include "tfill.h" -#include "function.h" #include "tcommon.h" #include "thash.h" #include "ttime.h" +#include "function.h" +#include "tdatablock.h" +#include "executorInt.h" +#include "querynodes.h" +#include "tfill.h" + #define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC) #define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1)))) @@ -37,168 +38,208 @@ static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { continue; } - char* val1 = elePtrAt(data[j], pCol->col.bytes, genRows); + SResSchema* pSchema = &pCol->pExpr->base.resSchema; + char* val1 = elePtrAt(data[j], pSchema->bytes, genRows); assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags); SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - -// assert (pTag->col.colId == pCol->col.colId); - assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type); + assignVal(val1, pTag->tagVal, pSchema->bytes, pSchema->type); } } -static void setNullValueForRow(SFillInfo* pFillInfo, void** data, int32_t numOfCol, int32_t rowIndex) { +static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) { // the first are always the timestamp column, so start from the second column. - for (int32_t i = 1; i < numOfCol; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - - char* output = elePtrAt(data[i], pCol->col.bytes, rowIndex); - setNull(output, pCol->col.type, pCol->col.bytes); + for (int32_t i = 1; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, i); + colDataAppendNULL(p, rowIndex); } } -static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData, int64_t ts, bool outOfBound) { - char* prev = pFillInfo->prevValues; - char* next = pFillInfo->nextValues; +#define GET_DEST_SLOT_ID(_p) ((_p)->pExpr->base.resSchema.slotId) +#define GET_SRC_SLOT_ID(_p) ((_p)->pExpr->base.pParam[0].pCol->slotId) + +static void doSetVal(SColumnInfoData* pDstColInfoData, int32_t rowIndex, const SGroupKeys* pKey); +static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock *pBlock, SSDataBlock* pSrcBlock, int64_t ts, bool outOfBound) { SPoint point1, point2, point; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); // set the primary timestamp column value int32_t index = pFillInfo->numOfCurrent; - char* val = elePtrAt(data[0], TSDB_KEYSIZE, index); + SColumnInfoData *pCol0 = taosArrayGet(pBlock->pDataBlock, 0); + char* val = colDataGetData(pCol0, index); + *(TSKEY*) val = pFillInfo->currentKey; // set the other values if (pFillInfo->type == TSDB_FILL_PREV) { - char* p = FILL_IS_ASC_FILL(pFillInfo) ? prev : next; - - if (p != NULL) { - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)) { - continue; - } + SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev : pFillInfo->next; - char* output = elePtrAt(data[i], pCol->col.bytes, index); -// assignVal(output, p + pCol->offset, pCol->col.bytes, pCol->col.type); + for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { + SFillColInfo* pCol = &pFillInfo->pFillCol[i]; + if (TSDB_COL_IS_TAG(pCol->flag)) { + continue; } - } else { // no prev value yet, set the value for NULL - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); + + SGroupKeys* pKey = taosArrayGet(p, i); + SColumnInfoData* pDstColInfoData = taosArrayGet(pBlock->pDataBlock, GET_DEST_SLOT_ID(pCol)); + doSetVal(pDstColInfoData, index, pKey); } } else if (pFillInfo->type == TSDB_FILL_NEXT) { - char* p = FILL_IS_ASC_FILL(pFillInfo)? next : prev; - - if (p != NULL) { - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)) { - continue; - } + SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->next : pFillInfo->prev; - char* output = elePtrAt(data[i], pCol->col.bytes, index); -// assignVal(output, p + pCol->offset, pCol->col.bytes, pCol->col.type); + for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { + SFillColInfo* pCol = &pFillInfo->pFillCol[i]; + if (TSDB_COL_IS_TAG(pCol->flag)) { + continue; } - } else { // no prev value yet, set the value for NULL - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); + + SGroupKeys* pKey = taosArrayGet(p, i); + SColumnInfoData* pDstColInfoData = taosArrayGet(pBlock->pDataBlock, GET_DEST_SLOT_ID(pCol)); + doSetVal(pDstColInfoData, index, pKey); } } else if (pFillInfo->type == TSDB_FILL_LINEAR) { // TODO : linear interpolation supports NULL value - if (prev != NULL && !outOfBound) { + if (outOfBound) { + setNullRow(pBlock, pFillInfo->numOfCols, index); + } else { for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; if (TSDB_COL_IS_TAG(pCol->flag)) { continue; } - int16_t type = pCol->col.type; - int16_t bytes = pCol->col.bytes; + int32_t srcSlotId = GET_SRC_SLOT_ID(pCol); - char *val1 = elePtrAt(data[i], pCol->col.bytes, index); - if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) { - setNull(val1, pCol->col.type, bytes); + int32_t dstSlotId = GET_DEST_SLOT_ID(pCol); + SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); + + int16_t type = pCol->pExpr->base.resSchema.type; + SGroupKeys* pKey = taosArrayGet(pFillInfo->prev, i); + if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pKey->isNull) { + colDataAppendNULL(pDstCol, index); continue; } - point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->offset}; - point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes}; - point = (SPoint){.key = pFillInfo->currentKey, .val = val1}; + SGroupKeys* pKey1 = taosArrayGet(pFillInfo->prev, 0); + int64_t prevTs = *(int64_t*)pKey1->pData; + + SColumnInfoData* pSrcCol = taosArrayGet(pSrcBlock->pDataBlock, srcSlotId); + char* data = colDataGetData(pSrcCol, pFillInfo->index); + + point1 = (SPoint){.key = prevTs, .val = pKey->pData}; + point2 = (SPoint){.key = ts, .val = data}; + + int64_t out = 0; + point = (SPoint){.key = pFillInfo->currentKey, .val = &out}; taosGetLinearInterpolationVal(&point, type, &point1, &point2, type); + + colDataAppend(pDstCol, index, (const char*)&out, false); } - } else { - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); } - } else { // fill the default value */ + } else if (pFillInfo->type == TSDB_FILL_NULL) { // fill with NULL + setNullRow(pBlock, pFillInfo->numOfCols, index); + } else { // fill with user specified value for each column for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) { + if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->schema.type)*/) { continue; } - char* val1 = elePtrAt(data[i], pCol->col.bytes, index); - assignVal(val1, (char*)&pCol->val, pCol->col.bytes, pCol->col.type); + SVariant* pVar = &pFillInfo->pFillCol[i].fillVal; + + SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); + if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); + colDataAppend(pDst, index, (char*)&v, false); + } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); + colDataAppend(pDst, index, (char*)&v, false); + } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); + colDataAppend(pDst, index, (char*)&v, false); + } } } - setTagsValue(pFillInfo, data, index); - pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, - pFillInfo->interval.precision); +// setTagsValue(pFillInfo, data, index); + SInterval* pInterval = &pFillInfo->interval; + pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision); pFillInfo->numOfCurrent++; } -static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) { - if (*next != NULL) { +void doSetVal(SColumnInfoData* pDstCol, int32_t rowIndex, const SGroupKeys* pKey) { + if (pKey->isNull) { + colDataAppendNULL(pDstCol, rowIndex); + } else { + colDataAppend(pDstCol, rowIndex, pKey->pData, false); + } +} + +static void initBeforeAfterDataBuf(SFillInfo* pFillInfo) { + if (taosArrayGetSize(pFillInfo->next) > 0) { return; } - *next = taosMemoryCalloc(1, pFillInfo->rowSize); - for (int i = 1; i < pFillInfo->numOfCols; i++) { + for (int i = 0; i < pFillInfo->numOfCols; i++) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - setNull(*next + pCol->offset, pCol->col.type, pCol->col.bytes); + + SGroupKeys key = {0}; + SResSchema* pSchema = &pCol->pExpr->base.resSchema; + key.pData = taosMemoryMalloc(pSchema->bytes); + key.isNull = true; + key.bytes = pSchema->bytes; + key.type = pSchema->type; + + taosArrayPush(pFillInfo->next, &key); + + key.pData = taosMemoryMalloc(pSchema->bytes); + taosArrayPush(pFillInfo->prev, &key); } } -static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* buf) { - int32_t rowIndex = pFillInfo->index; +static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull); + +static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray* pRow) { for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - memcpy(buf + pCol->offset, srcData[i] + rowIndex * pCol->col.bytes, pCol->col.bytes); + int32_t srcSlotId = GET_SRC_SLOT_ID(&pFillInfo->pFillCol[i]); + + SColumnInfoData* pSrcCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, srcSlotId); + + bool isNull = colDataIsNull_s(pSrcCol, rowIndex); + char* p = colDataGetData(pSrcCol, rowIndex); + saveColData(pRow, i, p, isNull); } } -static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputRows) { +static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t outputRows) { pFillInfo->numOfCurrent = 0; - char** srcData = pFillInfo->pData; - char** prev = &pFillInfo->prevValues; - char** next = &pFillInfo->nextValues; + // todo make sure the first column is always the primary timestamp column? + SColumnInfoData* pTsCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, 0); int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); + bool ascFill = FILL_IS_ASC_FILL(pFillInfo); - if (FILL_IS_ASC_FILL(pFillInfo)) { - assert(pFillInfo->currentKey >= pFillInfo->start); - } else { - assert(pFillInfo->currentKey <= pFillInfo->start); - } +#if 0 + ASSERT(ascFill && (pFillInfo->currentKey >= pFillInfo->start) || (!ascFill && (pFillInfo->currentKey <= pFillInfo->start))); +#endif while (pFillInfo->numOfCurrent < outputRows) { - int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->index]; + int64_t ts = ((int64_t*)pTsCol->pData)[pFillInfo->index]; // set the next value for interpolation - if ((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || - (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) { - initBeforeAfterDataBuf(pFillInfo, next); - copyCurrentRowIntoBuf(pFillInfo, srcData, *next); + if ((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) { + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pFillInfo->next); } - if (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) && - pFillInfo->numOfCurrent < outputRows) { - - // fill the gap between two actual input rows - while (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || - (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) && - pFillInfo->numOfCurrent < outputRows) { - doFillOneRowResult(pFillInfo, data, srcData, ts, false); + if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && pFillInfo->numOfCurrent < outputRows) { + // fill the gap between two input rows + while (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && pFillInfo->numOfCurrent < outputRows) { + doFillOneRowResult(pFillInfo, pBlock, pFillInfo->pSrcBlock, ts, false); } // output buffer is full, abort @@ -208,61 +249,66 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputR } } else { assert(pFillInfo->currentKey == ts); - initBeforeAfterDataBuf(pFillInfo, prev); + if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) { - initBeforeAfterDataBuf(pFillInfo, next); ++pFillInfo->index; - copyCurrentRowIntoBuf(pFillInfo, srcData, *next); + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pFillInfo->next); --pFillInfo->index; } // assign rows to dst buffer for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) { + if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->schema.type)*/) { continue; } - char* output = elePtrAt(data[i], pCol->col.bytes, pFillInfo->numOfCurrent); - char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index); + int32_t srcSlotId = GET_SRC_SLOT_ID(pCol); + int32_t dstSlotId = GET_DEST_SLOT_ID(pCol); - if (i == 0 || (pCol->functionId != FUNCTION_COUNT && !isNull(src, pCol->col.type)) || - (pCol->functionId == FUNCTION_COUNT && GET_INT64_VAL(src) != 0)) { - assignVal(output, src, pCol->col.bytes, pCol->col.type); - memcpy(*prev + pCol->offset, src, pCol->col.bytes); + SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, dstSlotId); + SColumnInfoData* pSrc = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, srcSlotId); + + char* src = colDataGetData(pSrc, pFillInfo->index); + if (i == 0 || (/*pCol->functionId != FUNCTION_COUNT &&*/ !colDataIsNull_s(pSrc, pFillInfo->index)) /*|| + (pCol->functionId == FUNCTION_COUNT && GET_INT64_VAL(src) != 0)*/) { + bool isNull = colDataIsNull_s(pSrc, pFillInfo->index); + colDataAppend(pDst, pFillInfo->numOfCurrent, src, isNull); + saveColData(pFillInfo->prev, i, src, isNull); } else { // i > 0 and data is null , do interpolation if (pFillInfo->type == TSDB_FILL_PREV) { - assignVal(output, *prev + pCol->offset, pCol->col.bytes, pCol->col.type); + SGroupKeys *pKey = taosArrayGet(pFillInfo->prev, i); + doSetVal(pDst, pFillInfo->numOfCurrent, pKey); } else if (pFillInfo->type == TSDB_FILL_LINEAR) { - assignVal(output, src, pCol->col.bytes, pCol->col.type); - memcpy(*prev + pCol->offset, src, pCol->col.bytes); + bool isNull = colDataIsNull_s(pSrc, pFillInfo->index); + colDataAppend(pDst, pFillInfo->numOfCurrent, src, isNull); + saveColData(pFillInfo->prev, i, src, isNull); + } else if (pFillInfo->type == TSDB_FILL_NULL) { + colDataAppendNULL(pDst, pFillInfo->numOfCurrent); } else if (pFillInfo->type == TSDB_FILL_NEXT) { - if (*next) { - assignVal(output, *next + pCol->offset, pCol->col.bytes, pCol->col.type); - } else { - setNull(output, pCol->col.type, pCol->col.bytes); - } + SGroupKeys *pKey = taosArrayGet(pFillInfo->next, i); + doSetVal(pDst, pFillInfo->numOfCurrent, pKey); } else { - assignVal(output, (char*)&pCol->val, pCol->col.bytes, pCol->col.type); + SVariant* pVar = &pFillInfo->pFillCol[i].fillVal; + colDataAppend(pDst, pFillInfo->numOfCurrent, (char*)&pVar->i, false); } } } // set the tag value for final result - setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent); +// setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent); + SInterval *pInterval = &pFillInfo->interval; + pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision); - pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, - pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); pFillInfo->index += 1; pFillInfo->numOfCurrent += 1; } if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) { /* the raw data block is exhausted, next value does not exists */ - if (pFillInfo->index >= pFillInfo->numOfRows) { - taosMemoryFreeClear(*next); - } - +// if (pFillInfo->index >= pFillInfo->numOfRows) { +// taosMemoryFreeClear(*next); +// } pFillInfo->numOfTotal += pFillInfo->numOfCurrent; return pFillInfo->numOfCurrent; } @@ -271,14 +317,24 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputR return pFillInfo->numOfCurrent; } -static int64_t appendFilledResult(SFillInfo* pFillInfo, void** output, int64_t resultCapacity) { +static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull) { + SGroupKeys *pKey = taosArrayGet(rowBuf, columnIndex); + if (isNull) { + pKey->isNull = true; + } else { + memcpy(pKey->pData, src, pKey->bytes); + pKey->isNull = false; + } +} + +static int64_t appendFilledResult(SFillInfo* pFillInfo, SSDataBlock* pBlock, int64_t resultCapacity) { /* * These data are generated according to fill strategy, since the current timestamp is out of the time window of * real result set. Note that we need to keep the direct previous result rows, to generated the filled data. */ pFillInfo->numOfCurrent = 0; while (pFillInfo->numOfCurrent < resultCapacity) { - doFillOneRowResult(pFillInfo, output, pFillInfo->pData, pFillInfo->start, true); + doFillOneRowResult(pFillInfo, pBlock, pFillInfo->pSrcBlock, pFillInfo->start, true); } pFillInfo->numOfTotal += pFillInfo->numOfCurrent; @@ -295,15 +351,15 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t int32_t k = 0; for (int32_t i = 0; i < numOfCols; ++i) { SFillColInfo* pColInfo = &pFillInfo->pFillCol[i]; - pFillInfo->pData[i] = NULL; + SResSchema* pSchema = &pColInfo->pExpr->base.resSchema; - if (TSDB_COL_IS_TAG(pColInfo->flag) || pColInfo->col.type == TSDB_DATA_TYPE_BINARY) { + if (TSDB_COL_IS_TAG(pColInfo->flag) || pSchema->type == TSDB_DATA_TYPE_BINARY) { numOfTags += 1; bool exists = false; int32_t index = -1; for (int32_t j = 0; j < k; ++j) { - if (pFillInfo->pTags[j].col.colId == pColInfo->col.slotId) { + if (pFillInfo->pTags[j].col.colId == pSchema->slotId) { exists = true; index = j; break; @@ -311,12 +367,12 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t } if (!exists) { - SSchema* pSchema = &pFillInfo->pTags[k].col; - pSchema->colId = pColInfo->col.slotId; - pSchema->type = pColInfo->col.type; - pSchema->bytes = pColInfo->col.bytes; + SSchema* pSchema1 = &pFillInfo->pTags[k].col; + pSchema1->colId = pSchema->slotId; + pSchema1->type = pSchema->type; + pSchema1->bytes = pSchema->bytes; - pFillInfo->pTags[k].tagVal = taosMemoryCalloc(1, pColInfo->col.bytes); + pFillInfo->pTags[k].tagVal = taosMemoryCalloc(1, pSchema->bytes); pColInfo->tagIndex = k; k += 1; @@ -325,7 +381,7 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t } } - rowsize += pColInfo->col.bytes; + rowsize += pSchema->bytes; } pFillInfo->numOfTags = numOfTags; @@ -355,7 +411,6 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag } taosResetFillInfo(pFillInfo, skey); - pFillInfo->order = order; switch(fillType) { @@ -364,6 +419,7 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag case FILL_MODE_NULL: pFillInfo->type = TSDB_FILL_NULL; break; case FILL_MODE_LINEAR: pFillInfo->type = TSDB_FILL_LINEAR;break; case FILL_MODE_NEXT: pFillInfo->type = TSDB_FILL_NEXT; break; + case FILL_MODE_VALUE: pFillInfo->type = TSDB_FILL_SET_VALUE; break; default: terrno = TSDB_CODE_INVALID_PARA; return NULL; @@ -376,7 +432,6 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag pFillInfo->alloc = capacity; pFillInfo->id = id; pFillInfo->interval = *pInterval; - pFillInfo->pData = taosMemoryMalloc(POINTER_BYTES * numOfCols); // if (numOfTags > 0) { pFillInfo->pTags = taosMemoryCalloc(numOfCols, sizeof(SFillTagColInfo)); @@ -385,6 +440,11 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag } // } + pFillInfo->next = taosArrayInit(numOfCols, sizeof(SGroupKeys)); + pFillInfo->prev = taosArrayInit(numOfCols, sizeof(SGroupKeys)); + + initBeforeAfterDataBuf(pFillInfo); + pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc); assert(pFillInfo->rowSize > 0); return pFillInfo; @@ -405,18 +465,15 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) { return NULL; } - taosMemoryFreeClear(pFillInfo->prevValues); - taosMemoryFreeClear(pFillInfo->nextValues); + taosArrayDestroy(pFillInfo->prev); + taosArrayDestroy(pFillInfo->next); for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) { taosMemoryFreeClear(pFillInfo->pTags[i].tagVal); } taosMemoryFreeClear(pFillInfo->pTags); - - taosMemoryFreeClear(pFillInfo->pData); taosMemoryFreeClear(pFillInfo->pFillCol); - taosMemoryFreeClear(pFillInfo); return NULL; } @@ -436,18 +493,7 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) } void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) { - for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - - SColumnInfoData* pColData = taosArrayGet(pInput->pDataBlock, i); - pFillInfo->pData[i] = pColData->pData; - - if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer - SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - assert (pTag->col.colId == pCol->col.slotId); - memcpy(pTag->tagVal, pColData->pData, pCol->col.bytes); // TODO not memcpy?? - } - } + pFillInfo->pSrcBlock = (SSDataBlock*) pInput; } bool taosFillHasMoreResults(SFillInfo* pFillInfo) { @@ -465,8 +511,9 @@ bool taosFillHasMoreResults(SFillInfo* pFillInfo) { } int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) { - int64_t* tsList = (int64_t*) pFillInfo->pData[0]; + SColumnInfoData* pCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, 0); + int64_t* tsList = (int64_t*) pCol->pData; int32_t numOfRows = taosNumOfRemainRows(pFillInfo); TSKEY ekey1 = ekey; @@ -513,7 +560,7 @@ int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* return TSDB_CODE_SUCCESS; } -int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity) { +int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity) { int32_t remain = taosNumOfRemainRows(pFillInfo); int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity); @@ -521,9 +568,9 @@ int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t cap // no data existed for fill operation now, append result according to the fill strategy if (remain == 0) { - appendFilledResult(pFillInfo, output, numOfRes); + appendFilledResult(pFillInfo, p, numOfRes); } else { - fillResultImpl(pFillInfo, output, (int32_t) numOfRes); + fillResultImpl(pFillInfo, p, (int32_t) numOfRes); assert(numOfRes == pFillInfo->numOfCurrent); } @@ -538,28 +585,30 @@ int64_t getFillInfoStart(struct SFillInfo *pFillInfo) { return pFillInfo->start; } -struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SValueNode* val) { - int32_t offset = 0; - - struct SFillColInfo* pFillCol = taosMemoryCalloc(numOfOutput, sizeof(SFillColInfo)); +SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SNodeListNode* pValNode) { + SFillColInfo* pFillCol = taosMemoryCalloc(numOfOutput, sizeof(SFillColInfo)); if (pFillCol == NULL) { return NULL; } + size_t len = (pValNode != NULL)? LIST_LENGTH(pValNode->pNodeList):0; for(int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExprInfo = &pExpr[i]; + SExprInfo* pExprInfo = &pExpr[i]; + pFillCol[i].pExpr = pExprInfo; + pFillCol[i].tagIndex = -2; - pFillCol[i].col = pExprInfo->base.resSchema; - pFillCol[i].offset = offset; - pFillCol[i].tagIndex = -2; + // todo refactor + if (len > 0) { + // if the user specified value is less than the column, alway use the last one as the fill value + int32_t index = (i >= len)? (len - 1):i; + + SValueNode* pv = (SValueNode*)nodesListGetNode(pValNode->pNodeList, index); + valueNodeToVariant(pv, &pFillCol[i].fillVal); + } if (pExprInfo->base.numOfParams > 0) { pFillCol[i].flag = pExprInfo->base.pParam[0].pCol->flag; // always be the normal column for table query } -// pFillCol[i].functionId = pExprInfo->pExpr->_function.functionId; -// pFillCol[i].val.d = *val; - - offset += pExprInfo->base.resSchema.bytes; } return pFillCol; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c new file mode 100644 index 0000000000000000000000000000000000000000..224f3db9128a60dae13794a9421b4a851b95b5a9 --- /dev/null +++ b/source/libs/executor/src/timewindowoperator.c @@ -0,0 +1,1462 @@ +#include "ttime.h" +#include "tdatablock.h" +#include "executorimpl.h" + +typedef enum SResultTsInterpType { + RESULT_ROW_START_INTERP = 1, + RESULT_ROW_END_INTERP = 2, +} SResultTsInterpType; + +/* + * There are two cases to handle: + * + * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including + * pQueryAttr->lastKey, pQueryAttr->window.skey, and pQueryAttr->eKey. + * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be + * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there + * is a previous result generated or not. + */ +static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) { + // SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; + // if (pResultRowInfo->curPos != -1) { + // return; + // } + + // pTableQueryInfo->win.skey = key; + // STimeWindow win = {.skey = key, .ekey = pQRange->ekey}; + + /** + * In handling the both ascending and descending order super table query, we need to find the first qualified + * timestamp of this table, and then set the first qualified start timestamp. + * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional + * operations involve. + */ + // STimeWindow w = TSWINDOW_INITIALIZER; + // + // TSKEY sk = TMIN(win.skey, win.ekey); + // TSKEY ek = TMAX(win.skey, win.ekey); + // getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); + + // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { + // if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { + // assert(win.ekey == pQueryAttr->window.ekey); + // } + // + // pResultRowInfo->prevSKey = w.skey; + // } + + // pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; +} + +static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows, bool ascQuery) { + TSKEY ts = TSKEY_INITIAL_VAL; + if (tsCols == NULL) { + ts = ascQuery ? win->skey : win->ekey; + } else { + int32_t offset = ascQuery ? 0 : rows - 1; + ts = tsCols[offset]; + } + + return ts; +} + +static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, + bool ascQuery) { + if (ascQuery) { + getAlignQueryTimeWindow(pInterval, precision, ts, w); + } else { + // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + getAlignQueryTimeWindow(pInterval, precision, ts, w); + + int64_t key = w->skey; + while (key < ts) { // moving towards end + key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision); + if (key >= ts) { + break; + } + + w->skey = key; + } + } +} + +// get the correct time window according to the handled timestamp +static STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, + SInterval* pInterval, int32_t precision, STimeWindow* win) { + STimeWindow w = {0}; + + if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value + getInitialStartTimeWindow(pInterval, precision, ts, &w, true); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win; + } + + if (w.skey > ts || w.ekey < ts) { + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + w.skey = taosTimeTruncate(ts, pInterval, precision); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + int64_t st = w.skey; + + if (st > ts) { + st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + int64_t et = st + pInterval->interval - 1; + if (et < ts) { + st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + w.skey = st; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } + } + return w; +} + +static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan, + SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset, SAggSupporter* pAggSup, + SExecTaskInfo* pTaskInfo) { + assert(win->skey <= win->ekey); + SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE, + masterscan, tableGroupId, pTaskInfo, true, pAggSup); + + if (pResultRow == NULL) { + *pResult = NULL; + return TSDB_CODE_SUCCESS; + } + + // set time window for current result + pResultRow->win = (*win); + *pResult = pResultRow; + setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); + return TSDB_CODE_SUCCESS; +} + +static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) { + int64_t* ts = (int64_t*)pColData->pData; + int32_t delta = includeEndpoint ? 1 : 0; + + int64_t duration = pWin->ekey - pWin->skey + delta; + ts[2] = duration; // set the duration + ts[3] = pWin->skey; // window start key + ts[4] = pWin->ekey + delta; // window end key +} + +static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts) { + pRowSup->win.ekey = ts; + pRowSup->prevTs = ts; + pRowSup->numOfRows += 1; +} + +static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex) { + pRowSup->startRowIndex = rowIndex; + pRowSup->numOfRows = 0; + pRowSup->win.skey = tsList[rowIndex]; +} + +static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, + int16_t pos, int16_t order, int64_t* pData) { + int32_t forwardStep = 0; + + if (order == TSDB_ORDER_ASC) { + int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order); + if (end >= 0) { + forwardStep = end; + + if (pData[end + pos] == ekey) { + forwardStep += 1; + } + } + } else { + int32_t end = searchFn((char*)pData, pos + 1, ekey, order); + if (end >= 0) { + forwardStep = pos - end; + + if (pData[end] == ekey) { + forwardStep += 1; + } + } + } + + assert(forwardStep >= 0); + return forwardStep; +} + +static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { + int32_t midPos = -1; + int32_t numOfRows; + + if (num <= 0) { + return -1; + } + + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + + TSKEY* keyList = (TSKEY*)pValue; + int32_t firstPos = 0; + int32_t lastPos = num - 1; + + if (order == TSDB_ORDER_DESC) { + // find the first position which is smaller than the key + while (1) { + if (key >= keyList[lastPos]) return lastPos; + if (key == keyList[firstPos]) return firstPos; + if (key < keyList[firstPos]) return firstPos - 1; + + numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + + } else { + // find the first position which is bigger than the key + while (1) { + if (key <= keyList[firstPos]) return firstPos; + if (key == keyList[lastPos]) return lastPos; + + if (key > keyList[lastPos]) { + lastPos = lastPos + 1; + if (lastPos >= num) + return -1; + else + return lastPos; + } + + numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1u) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + } + + return midPos; +} + +static int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, + TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, + int32_t order) { + assert(startPos >= 0 && startPos < pDataBlockInfo->rows); + + int32_t num = -1; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + + if (order == TSDB_ORDER_ASC) { + if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) { + num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); + if (item != NULL) { + item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; + } + } else { + num = pDataBlockInfo->rows - startPos; + if (item != NULL) { + item->lastKey = pDataBlockInfo->window.ekey + step; + } + } + } else { // desc + if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) { + num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); + if (item != NULL) { + item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; + } + } else { + num = startPos + 1; + if (item != NULL) { + item->lastKey = pDataBlockInfo->window.skey + step; + } + } + } + + assert(num >= 0); + return num; +} + +static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); + if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { + tw->skey += pInterval->sliding * factor; + tw->ekey = tw->skey + pInterval->interval - 1; + return; + } + + int64_t key = tw->skey, interval = pInterval->interval; + // convert key to second + key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000; + + if (pInterval->intervalUnit == 'y') { + interval *= 12; + } + + struct tm tm; + time_t t = (time_t)key; + taosLocalTime(&t, &tm); + + int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); + + mon = (int)(mon + interval); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); + + tw->ekey -= 1; +} + +void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, + int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { + SExprInfo* pExpr = pOperator->pExpr; + + SqlFunctionCtx* pCtx = pInfo->pCtx; + + for (int32_t k = 0; k < pOperator->numOfExprs; ++k) { + int32_t functionId = pCtx[k].functionId; + if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { + pCtx[k].start.key = INT64_MIN; + continue; + } + + SColIndex* pColIndex = NULL /*&pExpr[k].base.colInfo*/; + int16_t index = pColIndex->colIndex; + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, index); + + // assert(pColInfo->info.colId == pColIndex->info.colId && curTs != windowKey); + double v1 = 0, v2 = 0, v = 0; + + if (prevRowIndex == -1) { + // GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pRuntimeEnv->prevRow[index]); + } else { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); + } + + GET_TYPED_DATA(v2, double, pColInfo->info.type, (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes); + + if (functionId == FUNCTION_INTERP) { + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = prevTs; + pCtx[k].start.val = v1; + + pCtx[k].end.key = curTs; + pCtx[k].end.val = v2; + + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + if (prevRowIndex == -1) { + // pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index]; + } else { + pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes; + } + + pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes; + } + } + } else if (functionId == FUNCTION_TWA) { + SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; + SPoint point2 = (SPoint){.key = curTs, .val = &v2}; + SPoint point = (SPoint){.key = windowKey, .val = &v}; + + taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = point.key; + pCtx[k].start.val = v; + } else { + pCtx[k].end.key = point.key; + pCtx[k].end.val = v; + } + } + } +} + +static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { + if (type == RESULT_ROW_START_INTERP) { + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].start.key = INT64_MIN; + } + } else { + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].end.key = INT64_MIN; + } + } +} + +static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos, + int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, + STimeWindow* win) { + bool ascQuery = true; + TSKEY curTs = tsCols[pos]; + TSKEY lastTs = 0; //*(TSKEY*)pRuntimeEnv->prevRow[0]; + + // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. + // start exactly from this point, no need to do interpolation + TSKEY key = ascQuery ? win->skey : win->ekey; + if (key == curTs) { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_START_INTERP); + return true; + } + + if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_START_INTERP); + return true; + } + + int32_t step = 1; // GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery)) ? lastTs : tsCols[pos - step]; + + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, + RESULT_ROW_START_INTERP); + return true; +} + +static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex, + SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, + STimeWindow* win) { + int32_t order = TSDB_ORDER_ASC; + int32_t numOfOutput = pOperatorInfo->numOfExprs; + + TSKEY actualEndKey = tsCols[endRowIndex]; + TSKEY key = order ? win->ekey : win->skey; + + // not ended in current data block, do not invoke interpolation + if ((key > blockEkey /*&& QUERY_IS_ASC_QUERY(pQueryAttr)*/) || + (key < blockEkey /*&& !QUERY_IS_ASC_QUERY(pQueryAttr)*/)) { + setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + return false; + } + + // there is actual end point of current time window, no interpolation need + if (key == actualEndKey) { + setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + return true; + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + int32_t nextRowIndex = endRowIndex + step; + assert(nextRowIndex >= 0); + + TSKEY nextKey = tsCols[nextRowIndex]; + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey, + nextRowIndex, key, RESULT_ROW_END_INTERP); + return true; +} + +static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, + TSKEY* primaryKeys, int32_t prevPosition, SIntervalAggOperatorInfo* pInfo) { + int32_t order = pInfo->order; + bool ascQuery = (order == TSDB_ORDER_ASC); + + int32_t precision = pInterval->precision; + getNextTimeWindow(pInterval, precision, order, pNext); + + // next time window is not in current block + if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) || + (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) { + return -1; + } + + TSKEY startKey = ascQuery ? pNext->skey : pNext->ekey; + int32_t startPos = 0; + + // tumbling time window query, a special case of sliding time window query + if (pInterval->sliding == pInterval->interval && prevPosition != -1) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); + startPos = prevPosition + factor; + } else { + if (startKey <= pDataBlockInfo->window.skey && ascQuery) { + startPos = 0; + } else if (startKey >= pDataBlockInfo->window.ekey && !ascQuery) { + startPos = pDataBlockInfo->rows - 1; + } else { + startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, startKey, order); + } + } + + /* interp query with fill should not skip time window */ + // if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) { + // return startPos; + // } + + /* + * This time window does not cover any data, try next time window, + * this case may happen when the time window is too small + */ + if (primaryKeys == NULL) { + if (ascQuery) { + assert(pDataBlockInfo->window.skey <= pNext->ekey); + } else { + assert(pDataBlockInfo->window.ekey >= pNext->skey); + } + } else { + if (ascQuery && primaryKeys[startPos] > pNext->ekey) { + TSKEY next = primaryKeys[startPos]; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + pNext->skey = pNext->ekey - pInterval->interval + 1; + } + } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) { + TSKEY next = primaryKeys[startPos]; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + pNext->ekey = pNext->skey + pInterval->interval - 1; + } + } + } + + return startPos; +} + +static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { + assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); + if (type == RESULT_ROW_START_INTERP) { + return pResult->startInterp == true; + } else { + return pResult->endInterp == true; + } +} + +static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { + assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); + if (type == RESULT_ROW_START_INTERP) { + pResult->startInterp = true; + } else { + pResult->endInterp = true; + } +} + + +static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SqlFunctionCtx* pCtx, + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep, + int32_t order, bool timeWindowInterpo) { + if (!timeWindowInterpo) { + return; + } + + assert(pBlock != NULL); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + + if (pBlock->pDataBlock == NULL) { + // tscError("pBlock->pDataBlock == NULL"); + return; + } + + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); + + TSKEY* tsCols = (TSKEY*)(pColInfo->pData); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { // it is not interpolated, now start to generated the interpolated value + int32_t startRowIndex = startPos; + bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, + pBlock->pDataBlock, tsCols, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } + } else { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_START_INTERP); + } + + // point interpolation does not require the end key time window interpolation. + // if (pointInterpQuery) { + // return; + // } + + // interpolation query does not generate the time window end interpolation + done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + if (!done) { + int32_t endRowIndex = startPos + (forwardStep - 1) * step; + + TSKEY endKey = (order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey; + bool interp = + setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + } else { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_END_INTERP); + } +} + +static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowIndex, int32_t numOfCols) { + if (pDataBlock == NULL) { + return; + } + + for (int32_t k = 0; k < numOfCols; ++k) { + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, k); + memcpy(pRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); + } +} + +static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, + int32_t tableGroupId) { + SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; + + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + int32_t numOfOutput = pOperatorInfo->numOfExprs; + + SArray* pUpdated = NULL; + if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + pUpdated = taosArrayInit(4, POINTER_BYTES); + } + + int32_t step = 1; + bool ascScan = true; + + // int32_t prevIndex = pResultRowInfo->curPos; + + TSKEY* tsCols = NULL; + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + } + + int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan); + + STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, + pInfo->interval.precision, &pInfo->win); + bool masterScan = true; + SResultRow* pResult = NULL; + + int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*)pos->key = pResult->win.skey; + + taosArrayPush(pUpdated, &pos); + } + + int32_t forwardStep = 0; + TSKEY ekey = win.ekey; + forwardStep = + getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + + // prev time window not interpolation yet. + // int32_t curIndex = pResultRowInfo->curPos; + +#if 0 + if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) { + for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. + SResultRow* pRes = getResultRow(pResultRowInfo, j); + if (pRes->closed) { + assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); + continue; + } + + STimeWindow w = pRes->win; + ret = setTimeWindowOutputBuf(pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult, tableGroupId, + pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, + pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + doTimeWindowInterpolation(pOperatorInfo, &pInfo->binfo, pSDataBlock->pDataBlock, *(TSKEY*)pInfo->pRow[0], -1, + tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); + + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pInfo->binfo.pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_START_INTERP); + + doApplyFunctions(pInfo->binfo.pCtx, &w, &pInfo->timeWindowData, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + } + + // restore current time window + ret = setTimeWindowOutputBuf(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, + pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, + pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } +#endif + + // window start key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, + pInfo->order, false); + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); + doApplyFunctions(pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, + pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + + STimeWindow nextWin = win; + while (1) { + int32_t prevEndPos = (forwardStep - 1) * step + startPos; + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo); + if (startPos < 0) { + break; + } + + // null data, failed to allocate more memory buffer + int32_t code = + setTimeWindowOutputBuf(pResultRowInfo, &nextWin, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*)pos->key = pResult->win.skey; + + taosArrayPush(pUpdated, &pos); + } + + ekey = nextWin.ekey; // reviseWindowEkey(pQueryAttr, &nextWin); + forwardStep = + getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, + pInfo->order, false); + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, + pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + } + + if (pInfo->timeWindowInterpo) { + int32_t rowIndex = ascScan ? (pSDataBlock->info.rows - 1) : 0; + saveDataBlockLastRow(pInfo->pRow, pSDataBlock->pDataBlock, rowIndex, pSDataBlock->info.numOfCols); + } + + return pUpdated; + // updateResultRowInfoActiveIndex(pResultRowInfo, &pInfo->win, pRuntimeEnv->current->lastKey, true, false); +} + +static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; + } + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SIntervalAggOperatorInfo* pInfo = pOperator->info; + + int32_t order = TSDB_ORDER_ASC; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfExprs); + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); + STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; + + setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId); + +#if 0 // test for encode/decode result info + if(pOperator->encodeResultRow){ + char *result = NULL; + int32_t length = 0; + SAggSupporter *pSup = &pInfo->aggSup; + pOperator->encodeResultRow(pOperator, pSup, &pInfo->binfo, &result, &length); + taosHashClear(pSup->pResultRowHashTable); + pInfo->binfo.resultRowInfo.size = 0; + pOperator->decodeResultRow(pOperator, pSup, &pInfo->binfo, result, length); + if(result){ + taosMemoryFree(result); + } + } +#endif + } + + closeAllResultRows(&pInfo->binfo.resultRowInfo); + finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfExprs, pInfo->aggSup.pResultBuf, + &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); + OPTR_SET_OPENED(pOperator); + return TSDB_CODE_SUCCESS; +} + +static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); + int64_t gid = pBlock->info.groupId; + + bool masterScan = true; + int32_t numOfOutput = pOperator->numOfExprs; + int16_t bytes = pStateColInfoData->info.bytes; + + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId); + TSKEY* tsList = (TSKEY*)pColInfoData->pData; + + SWindowRowsSup* pRowSup = &pInfo->winSup; + pRowSup->numOfRows = 0; + + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pBlock->pBlockAgg[pInfo->colIndex])) { + continue; + } + + char* val = colDataGetData(pStateColInfoData, j); + + if (!pInfo->hasKey) { + memcpy(pInfo->stateKey.pData, val, bytes); + pInfo->hasKey = true; + + doKeepNewWindowStartInfo(pRowSup, tsList, j); + doKeepTuple(pRowSup, tsList[j]); + } else if (memcmp(pInfo->stateKey.pData, val, bytes) == 0) { + doKeepTuple(pRowSup, tsList[j]); + if (j == 0 && pRowSup->startRowIndex != 0) { + pRowSup->startRowIndex = 0; + } + } else { // a new state window started + SResultRow* pResult = NULL; + + // keep the time window for the closed time window. + STimeWindow window = pRowSup->win; + + pRowSup->win.ekey = pRowSup->win.skey; + int32_t ret = + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); + doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + + // here we start a new session window + doKeepNewWindowStartInfo(pRowSup, tsList, j); + doKeepTuple(pRowSup, tsList[j]); + } + } + + SResultRow* pResult = NULL; + pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; + int32_t ret = + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); + doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); +} + +static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SStateWindowOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + + if (pOperator->status == OP_RES_TO_RETURN) { + doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + return NULL; + } + + return pBInfo->pRes; + } + + int32_t order = TSDB_ORDER_ASC; + STimeWindow win = pTaskInfo->window; + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); + doStateWindowAggImpl(pOperator, pInfo, pBlock); + } + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pBInfo->resultRowInfo); + finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, + pBInfo->rowCellInfoOffset); + + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); + blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); + doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; +} + +static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { + SIntervalAggOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SSDataBlock* pBlock = pInfo->binfo.pRes; + + if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + return pOperator->fpSet.getStreamResFn(pOperator); + } else { + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + return NULL; + } + + blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + + if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pBlock->info.rows == 0 ? NULL : pBlock; + } +} + +// todo merged with the build group result. +static void finalizeUpdatedResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList, + int32_t* rowCellInfoOffset) { + size_t num = taosArrayGetSize(pUpdateList); + + for (int32_t i = 0; i < num; ++i) { + SResKeyPos* pPos = taosArrayGetP(pUpdateList, i); + + SFilePage* bufPage = getBufPage(pBuf, pPos->pos.pageId); + SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset); + + for (int32_t j = 0; j < numOfOutput; ++j) { + SResultRowEntryInfo* pEntry = getResultCell(pRow, j, rowCellInfoOffset); + if (pRow->numOfRows < pEntry->numOfRes) { + pRow->numOfRows = pEntry->numOfRes; + } + } + + releaseBufPage(pBuf, bufPage); + } +} + +static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { + SIntervalAggOperatorInfo* pInfo = pOperator->info; + int32_t order = TSDB_ORDER_ASC; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + if (pOperator->status == OP_RES_TO_RETURN) { + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; + } + + // STimeWindow win = {0}; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + SArray* pUpdated = NULL; + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + // The timewindows that overlaps the timestamps of the input pBlock need to be recalculated and return to the + // caller. Note that all the time window are not close till now. + + // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfExprs); + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); + pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); + } + + finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); + + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + + ASSERT(pInfo->binfo.pRes->info.rows > 0); + pOperator->status = OP_RES_TO_RETURN; + + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; +} + +static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { + SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + taosMemoryFreeClear(pInfo->stateKey.pData); +} + +void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { + SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + cleanupAggSup(&pInfo->aggSup); +} + +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, + SExecTaskInfo* pTaskInfo) { + SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; + // pInfo->execModel = OPTR_EXEC_MODEL_STREAM; + pInfo->execModel = pTaskInfo->execModel; + pInfo->win = pTaskInfo->window; + pInfo->twAggSup = *pTwAggSupp; + pInfo->primaryTsIndex = primaryTsSlotId; + + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + initResultSizeInfo(pOperator, 4096); + + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); + + // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); + + pOperator->name = "TimeIntervalAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfExprs = numOfCols; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, + destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + + _error: + destroyIntervalOperatorInfo(pInfo, numOfCols); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, + SExecTaskInfo* pTaskInfo) { + SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; + pInfo->execModel = OPTR_EXEC_MODEL_STREAM; + pInfo->win = pTaskInfo->window; + pInfo->twAggSup = *pTwAggSupp; + pInfo->primaryTsIndex = primaryTsSlotId; + + int32_t numOfRows = 4096; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + + initResultSizeInfo(pOperator, numOfRows); + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); + + // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); + + pOperator->name = "StreamTimeIntervalAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfExprs = numOfCols; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL, + destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + + _error: + destroyIntervalOperatorInfo(pInfo, numOfCols); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +// todo handle multiple tables cases. +static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId); + + bool masterScan = true; + int32_t numOfOutput = pOperator->numOfExprs; + int64_t gid = pBlock->info.groupId; + + int64_t gap = pInfo->gap; + + if (!pInfo->reptScan) { + pInfo->reptScan = true; + pInfo->winSup.prevTs = INT64_MIN; + } + + SWindowRowsSup* pRowSup = &pInfo->winSup; + pRowSup->numOfRows = 0; + + // In case of ascending or descending order scan data, only one time window needs to be kepted for each table. + TSKEY* tsList = (TSKEY*)pColInfoData->pData; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + if (pInfo->winSup.prevTs == INT64_MIN) { + doKeepNewWindowStartInfo(pRowSup, tsList, j); + doKeepTuple(pRowSup, tsList[j]); + } else if (tsList[j] - pRowSup->prevTs <= gap && (tsList[j] - pRowSup->prevTs) >= 0) { + // The gap is less than the threshold, so it belongs to current session window that has been opened already. + doKeepTuple(pRowSup, tsList[j]); + if (j == 0 && pRowSup->startRowIndex != 0) { + pRowSup->startRowIndex = 0; + } + } else { // start a new session window + SResultRow* pResult = NULL; + + // keep the time window for the closed time window. + STimeWindow window = pRowSup->win; + + pRowSup->win.ekey = pRowSup->win.skey; + int32_t ret = + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + // pInfo->numOfRows data belong to the current session window + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); + doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + + // here we start a new session window + doKeepNewWindowStartInfo(pRowSup, tsList, j); + doKeepTuple(pRowSup, tsList[j]); + } + } + + SResultRow* pResult = NULL; + pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; + int32_t ret = + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); + doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); +} + +static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SSessionAggOperatorInfo* pInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + + if (pOperator->status == OP_RES_TO_RETURN) { + doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + return NULL; + } + + return pBInfo->pRes; + } + + int32_t order = TSDB_ORDER_ASC; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (pBlock == NULL) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); + doSessionWindowAggImpl(pOperator, pInfo, pBlock); + } + + // restore the value + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pBInfo->resultRowInfo); + finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, + pBInfo->rowCellInfoOffset); + + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); + blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); + doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; +} + +static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STimeSliceOperatorInfo* pSliceInfo = pOperator->info; + if (pOperator->status == OP_RES_TO_RETURN) { + // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pSliceInfo->binfo.pRes; + } + + int32_t order = TSDB_ORDER_ASC; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (pBlock == NULL) { + break; + } + + // setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfExprs); + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, true); + // hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0); + } + + // restore the value + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pSliceInfo->binfo.resultRowInfo); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); + // finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfExprs); + + // initGroupedResultInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); + // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes); + + if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pSliceInfo->binfo.pRes->info.rows == 0 ? NULL : pSliceInfo->binfo.pRes; +} + +SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo) { + STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pOperator == NULL || pInfo == NULL) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + + pOperator->name = "TimeSliceOperator"; + // pOperator->operatorType = OP_AllTimeWindow; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfExprs = numOfCols; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doAllIntervalAgg, NULL, NULL, destroyBasicOperatorInfo, + NULL, NULL, NULL); + + int32_t code = appendDownstream(pOperator, &downstream, 1); + return pOperator; + + _error: + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + return NULL; +} + +SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, + SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, int32_t tsSlotId, + SExecTaskInfo* pTaskInfo) { + SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->colIndex = -1; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + + initResultSizeInfo(pOperator, 4096); + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExpr, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + + pInfo->twAggSup = *pTwAggSup; + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + + pInfo->tsSlotId = tsSlotId; + pOperator->name = "StateWindowOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExpr; + pOperator->numOfExprs = numOfCols; + pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL, + destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + + int32_t code = appendDownstream(pOperator, &downstream, 1); + return pOperator; + + _error: + pTaskInfo->code = TSDB_CODE_SUCCESS; + return NULL; +} + +void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { + SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); +} + +SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, + SExecTaskInfo* pTaskInfo) { + SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + initResultSizeInfo(pOperator, 4096); + + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->twAggSup = *pTwAggSupp; + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + + pInfo->tsSlotId = tsSlotId; + pInfo->gap = gap; + pInfo->binfo.pRes = pResBlock; + pInfo->winSup.prevTs = INT64_MIN; + pInfo->reptScan = false; + pOperator->name = "SessionWindowAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfExprs = numOfCols; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL, + destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + pOperator->pTaskInfo = pTaskInfo; + + code = appendDownstream(pOperator, &downstream, 1); + return pOperator; + + _error: + if (pInfo != NULL) { + destroySWindowOperatorInfo(pInfo, numOfCols); + } + + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} \ No newline at end of file diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 2ac28200e79d3b365fc75d055f785e0a99963355..040ee8c7f5a73c9817ccc353e63a1129369ff9fa 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -42,11 +42,7 @@ struct SSortHandle { _sort_fetch_block_fn_t fetchfp; _sort_merge_compar_fn_t comparFn; - - void *pParam; SMultiwayMergeTreeInfo *pMergeTree; - int32_t numOfCols; - int64_t startTs; uint64_t sortElapsed; uint64_t totalElapsed; @@ -61,29 +57,15 @@ struct SSortHandle { bool inMemSort; bool needAdjust; STupleHandle tupleHandle; + + void *param; + void (*beforeFp)(SSDataBlock* pBlock, void* param); }; static int32_t msortComparFn(const void *pLeft, const void *pRight, void *param); -static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = numOfCols; - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - - colInfo.info.type = pSchema[i].type; - colInfo.info.bytes = pSchema[i].bytes; - colInfo.info.colId = pSchema[i].colId; - taosArrayPush(pBlock->pDataBlock, &colInfo); - - if (IS_VAR_DATA_TYPE(colInfo.info.type)) { - pBlock->info.hasVarCol = true; - } - } - - return pBlock; +SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle) { + return createOneDataBlock(pSortHandle->pDataBlock, false); } /** @@ -99,7 +81,10 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, SArray* pIndexMap, int32_t pSortHandle->numOfPages = numOfPages; pSortHandle->pSortInfo = pSortInfo; pSortHandle->pIndexMap = pIndexMap; - pSortHandle->pDataBlock = createOneDataBlock(pBlock, false); + + if (pBlock != NULL) { + pSortHandle->pDataBlock = createOneDataBlock(pBlock, false); + } pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES); pSortHandle->cmpParam.orderInfo = pSortInfo; @@ -374,18 +359,17 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) { for(int32_t i = 0; i < pInfo->size; ++i) { SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); - SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId); bool leftNull = false; if (pLeftColInfoData->hasNull) { - leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg); + leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg[pOrder->slotId]); } SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->slotId); bool rightNull = false; if (pRightColInfoData->hasNull) { - rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, pRightBlock->pBlockAgg); + rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, pRightBlock->pBlockAgg[pOrder->slotId]); } if (leftNull && rightNull) { @@ -532,8 +516,25 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { if (pHandle->pDataBlock == NULL) { pHandle->pDataBlock = createOneDataBlock(pBlock, false); + + // calculate the buffer pages according to the total available buffers. + int32_t rowSize = blockDataGetRowSize(pBlock); + if (rowSize * 4 > 4096) { + pHandle->pageSize = rowSize * 4; + } else { + pHandle->pageSize = 4096; + } + // todo!! + pHandle->numOfPages = 1024; + sortBufSize = pHandle->numOfPages * pHandle->pageSize; + } + + // perform the scalar function calculation before apply the sort + if (pHandle->beforeFp != NULL) { + pHandle->beforeFp(pBlock, pHandle->param); } + // todo relocate the columns int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock, pHandle->pIndexMap); if (code != 0) { return code; @@ -624,8 +625,10 @@ int32_t tsortClose(SSortHandle* pHandle) { return TSDB_CODE_SUCCESS; } -int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fp) { - pHandle->fetchfp = fp; +int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fetchFp, void (*fp)(SSDataBlock*, void*), void* param) { + pHandle->fetchfp = fetchFp; + pHandle->beforeFp = fp; + pHandle->param = param; return TSDB_CODE_SUCCESS; } @@ -682,7 +685,7 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle) { bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex) { SColumnInfoData* pColInfoSrc = taosArrayGet(pVHandle->pBlock->pDataBlock, colIndex); - return colDataIsNull(pColInfoSrc, 0, pVHandle->rowIndex, NULL); + return colDataIsNull_s(pColInfoSrc, pVHandle->rowIndex); } void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) { diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index de8df7b91641f7ffe21a059793b90fed519e082a..f534cf091780122d5f913000fb479bf5c09605b9 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -55,7 +55,7 @@ typedef struct SDummyInputInfo { SSDataBlock* pBlock; } SDummyInputInfo; -SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { +SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) { SDummyInputInfo* pInfo = static_cast(pOperator->info); if (pInfo->current >= pInfo->totalPages) { return NULL; @@ -121,7 +121,7 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { return pBlock; } -SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { +SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { SDummyInputInfo* pInfo = static_cast(pOperator->info); if (pInfo->current >= pInfo->totalPages) { return NULL; diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index efa0581fd0b09893054c09a9429cb589411d3e91..c037fae75f258411c3cecc32b4d3032bc48d332f 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -164,13 +164,13 @@ int32_t docomp(const void* p1, const void* p2, void* param) { bool leftNull = false; if (pLeftColInfoData->hasNull) { - leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg); + leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg[pOrder->slotId]); } SColumnInfoData* pRightColInfoData = (SColumnInfoData*) TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->slotId); bool rightNull = false; if (pRightColInfoData->hasNull) { - rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, pRightBlock->pBlockAgg); + rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, pRightBlock->pBlockAgg[pOrder->slotId]); } if (leftNull && rightNull) { @@ -210,7 +210,7 @@ TEST(testCase, inMem_sort_Test) { taosArrayPush(orderInfo, &oi); SSortHandle* phandle = tsortCreateSortHandle(orderInfo, NULL, SORT_SINGLESOURCE_SORT, 1024, 5, NULL, "test_abc"); - tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); + tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock, NULL, NULL); _info* pInfo = (_info*) taosMemoryCalloc(1, sizeof(_info)); pInfo->startVal = 0; @@ -299,7 +299,7 @@ TEST(testCase, external_mem_sort_Test) { taosArrayPush(orderInfo, &oi); SSortHandle* phandle = tsortCreateSortHandle(orderInfo, NULL, SORT_SINGLESOURCE_SORT, 128, 3, NULL, "test_abc"); - tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); + tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock, NULL, NULL); SSortSource* ps = static_cast(taosMemoryCalloc(1, sizeof(SSortSource))); ps->param = &pInfo[i]; @@ -366,7 +366,7 @@ TEST(testCase, ordered_merge_sort_Test) { } SSortHandle* phandle = tsortCreateSortHandle(orderInfo, NULL, SORT_MULTISOURCE_MERGE, 1024, 5, pBlock,"test_abc"); - tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); + tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock, NULL, NULL); tsortSetComparFp(phandle, docomp); SSortSource* p[10] = {0}; diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index 15813b3cb05cc796594dd7be27ad10a0557359b7..c31cabda1960780331650c1db90912db3d314632 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -48,8 +48,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - udf1 PUBLIC os -) + udf1 PUBLIC os) add_library(udf2 MODULE test/udf2.c) target_include_directories( diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index d419feca45085c185cfbc9b92a05a3597666e3dd..abb9525cc5c68f6cb9b817a5eea3cdb1bbfe8861 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -68,6 +68,16 @@ bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); int32_t topFunction(SqlFunctionCtx *pCtx); int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +int32_t spreadFunction(SqlFunctionCtx* pCtx); +int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + +bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +int32_t histogramFunction(SqlFunctionCtx* pCtx); +int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + #ifdef __cplusplus } #endif diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index e19a332e669d283b1cdd2ce11d02b09ddcbef12a..3869a5d7b2898bd1ea7e621513f6f13ea6d25d4b 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -39,6 +39,7 @@ extern "C" { #define FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED FUNC_MGT_FUNC_CLASSIFICATION_MASK(10) #define FUNC_MGT_MULTI_RES_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(11) #define FUNC_MGT_SCAN_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(12) +#define FUNC_MGT_SELECT_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(13) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) diff --git a/source/libs/function/inc/tudfInt.h b/source/libs/function/inc/tudfInt.h index 8aedd59c335f2876a5fc4c6a9f958f4ea81dc464..6f82542aee6966d8b4fd7c613263a0639be190a3 100644 --- a/source/libs/function/inc/tudfInt.h +++ b/source/libs/function/inc/tudfInt.h @@ -19,9 +19,6 @@ extern "C" { #endif -//TODO replaces them with fnDebug -//#define debugPrint(...) taosPrintLog("Function", DEBUG_INFO, 135, __VA_ARGS__) -#define debugPrint(...) {fprintf(stderr, __VA_ARGS__);fprintf(stderr, "\n");} enum { UDF_TASK_SETUP = 0, UDF_TASK_CALL = 1, @@ -107,7 +104,7 @@ void* decodeUdfRequest(const void *buf, SUdfRequest* request); int32_t encodeUdfResponse(void **buf, const SUdfResponse *response); void* decodeUdfResponse(const void* buf, SUdfResponse *response); -void freeUdfColumnData(SUdfColumnData *data); +void freeUdfColumnData(SUdfColumnData *data, SUdfColumnMeta *meta); void freeUdfColumn(SUdfColumn* col); void freeUdfDataDataBlock(SUdfDataBlock *block); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 45cfbae1ad13d348b92d8225af797f9db400fe78..8fa4bd0c4ddd9bcb3320b60d13ad04cace9d7a65 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -212,7 +212,36 @@ static int32_t translateBottom(SFunctionNode* pFunc, char* pErrBuf, int32_t len) } static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - // todo + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_NUMERIC_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE}; + return TSDB_CODE_SUCCESS; +} + +static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (4 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_NUMERIC_TYPE(colType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY || + ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BINARY || + ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type != TSDB_DATA_TYPE_BIGINT) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE }; return TSDB_CODE_SUCCESS; } @@ -233,8 +262,7 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l "The parameters of first/last can only be columns"); } - uint8_t paraType = ((SExprNode*)pPara)->resType.type; - pFunc->node.resType = (SDataType){.bytes = tDataTypes[paraType].bytes, .type = paraType}; + pFunc->node.resType = ((SExprNode*)pPara)->resType; return TSDB_CODE_SUCCESS; } @@ -247,8 +275,7 @@ static int32_t translateLength(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = - (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_SMALLINT].bytes, .type = TSDB_DATA_TYPE_SMALLINT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } @@ -430,436 +457,569 @@ static int32_t translateToJson(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return TSDB_CODE_SUCCESS; } +static int32_t translateSelectValue(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; + return TSDB_CODE_SUCCESS; +} + +// clang-format off const SBuiltinFuncDefinition funcMgtBuiltins[] = { - {.name = "count", - .type = FUNCTION_TYPE_COUNT, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, - .translateFunc = translateCount, - .dataRequiredFunc = countDataRequired, - .getEnvFunc = getCountFuncEnv, - .initFunc = functionSetup, - .processFunc = countFunction, - .finalizeFunc = functionFinalize}, - {.name = "sum", - .type = FUNCTION_TYPE_SUM, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, - .translateFunc = translateSum, - .dataRequiredFunc = statisDataRequired, - .getEnvFunc = getSumFuncEnv, - .initFunc = functionSetup, - .processFunc = sumFunction, - .finalizeFunc = functionFinalize}, - {.name = "min", - .type = FUNCTION_TYPE_MIN, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, - .translateFunc = translateInOutNum, - .dataRequiredFunc = statisDataRequired, - .getEnvFunc = getMinmaxFuncEnv, - .initFunc = minFunctionSetup, - .processFunc = minFunction, - .finalizeFunc = functionFinalize}, - {.name = "max", - .type = FUNCTION_TYPE_MAX, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, - .translateFunc = translateInOutNum, - .dataRequiredFunc = statisDataRequired, - .getEnvFunc = getMinmaxFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, - .finalizeFunc = functionFinalize}, - {.name = "stddev", - .type = FUNCTION_TYPE_STDDEV, - .classification = FUNC_MGT_AGG_FUNC, - .translateFunc = translateInNumOutDou, - .getEnvFunc = getStddevFuncEnv, - .initFunc = stddevFunctionSetup, - .processFunc = stddevFunction, - .finalizeFunc = stddevFinalize}, - {.name = "avg", - .type = FUNCTION_TYPE_AVG, - .classification = FUNC_MGT_AGG_FUNC, - .translateFunc = translateInNumOutDou, - .getEnvFunc = getAvgFuncEnv, - .initFunc = avgFunctionSetup, - .processFunc = avgFunction, - .finalizeFunc = avgFinalize}, - {.name = "percentile", - .type = FUNCTION_TYPE_PERCENTILE, - .classification = FUNC_MGT_AGG_FUNC, - .translateFunc = translatePercentile, - .getEnvFunc = getPercentileFuncEnv, - .initFunc = percentileFunctionSetup, - .processFunc = percentileFunction, - .finalizeFunc = percentileFinalize}, - {.name = "apercentile", - .type = FUNCTION_TYPE_APERCENTILE, - .classification = FUNC_MGT_AGG_FUNC, - .translateFunc = translateApercentile, - .getEnvFunc = getMinmaxFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, - .finalizeFunc = functionFinalize}, - { - .name = "top", - .type = FUNCTION_TYPE_TOP, - .classification = FUNC_MGT_AGG_FUNC, - .translateFunc = translateTop, - .getEnvFunc = getTopBotFuncEnv, - .initFunc = functionSetup, - .processFunc = topFunction, - .finalizeFunc = topBotFinalize, - }, - {.name = "bottom", - .type = FUNCTION_TYPE_BOTTOM, - .classification = FUNC_MGT_AGG_FUNC, - .translateFunc = translateBottom, - .getEnvFunc = getMinmaxFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, - .finalizeFunc = functionFinalize}, - {.name = "spread", - .type = FUNCTION_TYPE_SPREAD, - .classification = FUNC_MGT_AGG_FUNC, - .translateFunc = translateSpread, - .getEnvFunc = getMinmaxFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, - .finalizeFunc = functionFinalize}, - {.name = "last_row", - .type = FUNCTION_TYPE_LAST_ROW, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC, - .translateFunc = translateLastRow, - .getEnvFunc = getMinmaxFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, - .finalizeFunc = functionFinalize}, - {.name = "first", - .type = FUNCTION_TYPE_FIRST, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC, - .translateFunc = translateFirstLast, - .getEnvFunc = getFirstLastFuncEnv, - .initFunc = functionSetup, - .processFunc = firstFunction, - .finalizeFunc = functionFinalize}, - {.name = "last", - .type = FUNCTION_TYPE_LAST, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC, - .translateFunc = translateFirstLast, - .getEnvFunc = getFirstLastFuncEnv, - .initFunc = functionSetup, - .processFunc = lastFunction, - .finalizeFunc = functionFinalize}, - {.name = "diff", - .type = FUNCTION_TYPE_DIFF, - .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC, - .translateFunc = translateInOutNum, - .getEnvFunc = getDiffFuncEnv, - .initFunc = diffFunctionSetup, - .processFunc = diffFunction, - .finalizeFunc = functionFinalize}, - {.name = "abs", - .type = FUNCTION_TYPE_ABS, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateInOutNum, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = absFunction, - .finalizeFunc = NULL}, - {.name = "log", - .type = FUNCTION_TYPE_LOG, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateIn2NumOutDou, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = logFunction, - .finalizeFunc = NULL}, - {.name = "pow", - .type = FUNCTION_TYPE_POW, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateIn2NumOutDou, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = powFunction, - .finalizeFunc = NULL}, - {.name = "sqrt", - .type = FUNCTION_TYPE_SQRT, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateInNumOutDou, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = sqrtFunction, - .finalizeFunc = NULL}, - {.name = "ceil", - .type = FUNCTION_TYPE_CEIL, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateInOutNum, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = ceilFunction, - .finalizeFunc = NULL}, - {.name = "floor", - .type = FUNCTION_TYPE_FLOOR, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateInOutNum, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = floorFunction, - .finalizeFunc = NULL}, - {.name = "round", - .type = FUNCTION_TYPE_ROUND, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateInOutNum, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = roundFunction, - .finalizeFunc = NULL}, - {.name = "sin", - .type = FUNCTION_TYPE_SIN, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateInNumOutDou, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = sinFunction, - .finalizeFunc = NULL}, - {.name = "cos", - .type = FUNCTION_TYPE_COS, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateInNumOutDou, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = cosFunction, - .finalizeFunc = NULL}, - {.name = "tan", - .type = FUNCTION_TYPE_TAN, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateInNumOutDou, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = tanFunction, - .finalizeFunc = NULL}, - {.name = "asin", - .type = FUNCTION_TYPE_ASIN, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateInNumOutDou, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = asinFunction, - .finalizeFunc = NULL}, - {.name = "acos", - .type = FUNCTION_TYPE_ACOS, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateInNumOutDou, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = acosFunction, - .finalizeFunc = NULL}, - {.name = "atan", - .type = FUNCTION_TYPE_ATAN, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateInNumOutDou, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = atanFunction, - .finalizeFunc = NULL}, - {.name = "length", - .type = FUNCTION_TYPE_LENGTH, - .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, - .translateFunc = translateLength, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = lengthFunction, - .finalizeFunc = NULL}, - {.name = "char_length", - .type = FUNCTION_TYPE_CHAR_LENGTH, - .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, - .translateFunc = translateLength, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = charLengthFunction, - .finalizeFunc = NULL}, - {.name = "concat", - .type = FUNCTION_TYPE_CONCAT, - .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, - .translateFunc = translateConcat, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = concatFunction, - .finalizeFunc = NULL}, - {.name = "concat_ws", - .type = FUNCTION_TYPE_CONCAT_WS, - .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, - .translateFunc = translateConcatWs, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = concatWsFunction, - .finalizeFunc = NULL}, - {.name = "lower", - .type = FUNCTION_TYPE_LOWER, - .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, - .translateFunc = translateInOutStr, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = lowerFunction, - .finalizeFunc = NULL}, - {.name = "upper", - .type = FUNCTION_TYPE_UPPER, - .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, - .translateFunc = translateInOutStr, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = upperFunction, - .finalizeFunc = NULL}, - {.name = "ltrim", - .type = FUNCTION_TYPE_LTRIM, - .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, - .translateFunc = translateInOutStr, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = ltrimFunction, - .finalizeFunc = NULL}, - {.name = "rtrim", - .type = FUNCTION_TYPE_RTRIM, - .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, - .translateFunc = translateInOutStr, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = rtrimFunction, - .finalizeFunc = NULL}, - {.name = "substr", - .type = FUNCTION_TYPE_SUBSTR, - .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, - .translateFunc = translateSubstr, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = substrFunction, - .finalizeFunc = NULL}, - {.name = "cast", - .type = FUNCTION_TYPE_CAST, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateCast, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = castFunction, - .finalizeFunc = NULL}, - {.name = "to_iso8601", - .type = FUNCTION_TYPE_TO_ISO8601, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateToIso8601, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = toISO8601Function, - .finalizeFunc = NULL}, - {.name = "to_unixtimestamp", - .type = FUNCTION_TYPE_TO_UNIXTIMESTAMP, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateToUnixtimestamp, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = toUnixtimestampFunction, - .finalizeFunc = NULL}, - {.name = "timetruncate", - .type = FUNCTION_TYPE_TIMETRUNCATE, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateTimeTruncate, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = timeTruncateFunction, - .finalizeFunc = NULL}, - {.name = "timediff", - .type = FUNCTION_TYPE_TIMEDIFF, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateTimeDiff, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = timeDiffFunction, - .finalizeFunc = NULL}, - {.name = "now", - .type = FUNCTION_TYPE_NOW, - .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC, - .translateFunc = translateTimePseudoColumn, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = nowFunction, - .finalizeFunc = NULL}, - {.name = "today", - .type = FUNCTION_TYPE_TODAY, - .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC, - .translateFunc = translateTimePseudoColumn, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = todayFunction, - .finalizeFunc = NULL}, - {.name = "timezone", - .type = FUNCTION_TYPE_TIMEZONE, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateTimezone, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = timezoneFunction, - .finalizeFunc = NULL}, - {.name = "_rowts", - .type = FUNCTION_TYPE_ROWTS, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, - .translateFunc = translateTimePseudoColumn, - .getEnvFunc = getTimePseudoFuncEnv, - .initFunc = NULL, - .sprocessFunc = NULL, - .finalizeFunc = NULL}, - {.name = "tbname", - .type = FUNCTION_TYPE_TBNAME, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC, - .translateFunc = translateTbnameColumn, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = NULL, - .finalizeFunc = NULL}, - {.name = "_qstartts", - .type = FUNCTION_TYPE_QSTARTTS, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC, - .translateFunc = translateTimePseudoColumn, - .getEnvFunc = getTimePseudoFuncEnv, - .initFunc = NULL, - .sprocessFunc = qStartTsFunction, - .finalizeFunc = NULL}, - {.name = "_qendts", - .type = FUNCTION_TYPE_QENDTS, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC, - .translateFunc = translateTimePseudoColumn, - .getEnvFunc = getTimePseudoFuncEnv, - .initFunc = NULL, - .sprocessFunc = qEndTsFunction, - .finalizeFunc = NULL}, - {.name = "_wstartts", - .type = FUNCTION_TYPE_WSTARTTS, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC, - .translateFunc = translateTimePseudoColumn, - .getEnvFunc = getTimePseudoFuncEnv, - .initFunc = NULL, - .sprocessFunc = winStartTsFunction, - .finalizeFunc = NULL}, - {.name = "_wendts", - .type = FUNCTION_TYPE_WENDTS, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC, - .translateFunc = translateTimePseudoColumn, - .getEnvFunc = getTimePseudoFuncEnv, - .initFunc = NULL, - .sprocessFunc = winEndTsFunction, - .finalizeFunc = NULL}, - {.name = "_wduration", - .type = FUNCTION_TYPE_WDURATION, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC, - .translateFunc = translateWduration, - .getEnvFunc = getTimePseudoFuncEnv, - .initFunc = NULL, - .sprocessFunc = winDurFunction, - .finalizeFunc = NULL}, - {.name = "to_json", - .type = FUNCTION_TYPE_TO_JSON, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateToJson, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = toJsonFunction, - .finalizeFunc = NULL}}; + { + .name = "count", + .type = FUNCTION_TYPE_COUNT, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, + .translateFunc = translateCount, + .dataRequiredFunc = countDataRequired, + .getEnvFunc = getCountFuncEnv, + .initFunc = functionSetup, + .processFunc = countFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "sum", + .type = FUNCTION_TYPE_SUM, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, + .translateFunc = translateSum, + .dataRequiredFunc = statisDataRequired, + .getEnvFunc = getSumFuncEnv, + .initFunc = functionSetup, + .processFunc = sumFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "min", + .type = FUNCTION_TYPE_MIN, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC, + .translateFunc = translateInOutNum, + .dataRequiredFunc = statisDataRequired, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = minFunctionSetup, + .processFunc = minFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "max", + .type = FUNCTION_TYPE_MAX, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC, + .translateFunc = translateInOutNum, + .dataRequiredFunc = statisDataRequired, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "stddev", + .type = FUNCTION_TYPE_STDDEV, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = getStddevFuncEnv, + .initFunc = stddevFunctionSetup, + .processFunc = stddevFunction, + .finalizeFunc = stddevFinalize + }, + { + .name = "avg", + .type = FUNCTION_TYPE_AVG, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = getAvgFuncEnv, + .initFunc = avgFunctionSetup, + .processFunc = avgFunction, + .finalizeFunc = avgFinalize + }, + { + .name = "percentile", + .type = FUNCTION_TYPE_PERCENTILE, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translatePercentile, + .getEnvFunc = getPercentileFuncEnv, + .initFunc = percentileFunctionSetup, + .processFunc = percentileFunction, + .finalizeFunc = percentileFinalize + }, + { + .name = "apercentile", + .type = FUNCTION_TYPE_APERCENTILE, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateApercentile, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "top", + .type = FUNCTION_TYPE_TOP, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateTop, + .getEnvFunc = getTopBotFuncEnv, + .initFunc = functionSetup, + .processFunc = topFunction, + .finalizeFunc = topBotFinalize, + }, + { + .name = "bottom", + .type = FUNCTION_TYPE_BOTTOM, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateBottom, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "spread", + .type = FUNCTION_TYPE_SPREAD, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateSpread, + .dataRequiredFunc = statisDataRequired, + .getEnvFunc = getSpreadFuncEnv, + .initFunc = spreadFunctionSetup, + .processFunc = spreadFunction, + .finalizeFunc = spreadFinalize + }, + { + .name = "last_row", + .type = FUNCTION_TYPE_LAST_ROW, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC, + .translateFunc = translateLastRow, + .getEnvFunc = getMinmaxFuncEnv, + .initFunc = maxFunctionSetup, + .processFunc = maxFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "first", + .type = FUNCTION_TYPE_FIRST, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLast, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = firstFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "last", + .type = FUNCTION_TYPE_LAST, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLast, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "diff", + .type = FUNCTION_TYPE_DIFF, + .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateInOutNum, + .getEnvFunc = getDiffFuncEnv, + .initFunc = diffFunctionSetup, + .processFunc = diffFunction, + .finalizeFunc = functionFinalize + }, + { + .name = "histogram", + .type = FUNCTION_TYPE_HISTOGRAM, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateHistogram, + .getEnvFunc = getHistogramFuncEnv, + .initFunc = histogramFunctionSetup, + .processFunc = histogramFunction, + .finalizeFunc = histogramFinalize + }, + { + .name = "abs", + .type = FUNCTION_TYPE_ABS, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateInOutNum, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = absFunction, + .finalizeFunc = NULL + }, + { + .name = "log", + .type = FUNCTION_TYPE_LOG, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateIn2NumOutDou, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = logFunction, + .finalizeFunc = NULL + }, + { + .name = "pow", + .type = FUNCTION_TYPE_POW, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateIn2NumOutDou, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = powFunction, + .finalizeFunc = NULL + }, + { + .name = "sqrt", + .type = FUNCTION_TYPE_SQRT, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = sqrtFunction, + .finalizeFunc = NULL + }, + { + .name = "ceil", + .type = FUNCTION_TYPE_CEIL, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateInOutNum, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = ceilFunction, + .finalizeFunc = NULL + }, + { + .name = "floor", + .type = FUNCTION_TYPE_FLOOR, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateInOutNum, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = floorFunction, + .finalizeFunc = NULL + }, + { + .name = "round", + .type = FUNCTION_TYPE_ROUND, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateInOutNum, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = roundFunction, + .finalizeFunc = NULL + }, + { + .name = "sin", + .type = FUNCTION_TYPE_SIN, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = sinFunction, + .finalizeFunc = NULL + }, + { + .name = "cos", + .type = FUNCTION_TYPE_COS, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = cosFunction, + .finalizeFunc = NULL + }, + { + .name = "tan", + .type = FUNCTION_TYPE_TAN, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = tanFunction, + .finalizeFunc = NULL + }, + { + .name = "asin", + .type = FUNCTION_TYPE_ASIN, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = asinFunction, + .finalizeFunc = NULL + }, + { + .name = "acos", + .type = FUNCTION_TYPE_ACOS, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = acosFunction, + .finalizeFunc = NULL + }, + { + .name = "atan", + .type = FUNCTION_TYPE_ATAN, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = atanFunction, + .finalizeFunc = NULL + }, + { + .name = "length", + .type = FUNCTION_TYPE_LENGTH, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .translateFunc = translateLength, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = lengthFunction, + .finalizeFunc = NULL + }, + { + .name = "char_length", + .type = FUNCTION_TYPE_CHAR_LENGTH, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .translateFunc = translateLength, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = charLengthFunction, + .finalizeFunc = NULL + }, + { + .name = "concat", + .type = FUNCTION_TYPE_CONCAT, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .translateFunc = translateConcat, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = concatFunction, + .finalizeFunc = NULL + }, + { + .name = "concat_ws", + .type = FUNCTION_TYPE_CONCAT_WS, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .translateFunc = translateConcatWs, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = concatWsFunction, + .finalizeFunc = NULL + }, + { + .name = "lower", + .type = FUNCTION_TYPE_LOWER, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .translateFunc = translateInOutStr, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = lowerFunction, + .finalizeFunc = NULL + }, + { + .name = "upper", + .type = FUNCTION_TYPE_UPPER, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .translateFunc = translateInOutStr, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = upperFunction, + .finalizeFunc = NULL + }, + { + .name = "ltrim", + .type = FUNCTION_TYPE_LTRIM, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .translateFunc = translateInOutStr, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = ltrimFunction, + .finalizeFunc = NULL + }, + { + .name = "rtrim", + .type = FUNCTION_TYPE_RTRIM, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .translateFunc = translateInOutStr, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = rtrimFunction, + .finalizeFunc = NULL + }, + { + .name = "substr", + .type = FUNCTION_TYPE_SUBSTR, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .translateFunc = translateSubstr, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = substrFunction, + .finalizeFunc = NULL + }, + { + .name = "cast", + .type = FUNCTION_TYPE_CAST, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateCast, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = castFunction, + .finalizeFunc = NULL + }, + { + .name = "to_iso8601", + .type = FUNCTION_TYPE_TO_ISO8601, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateToIso8601, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = toISO8601Function, + .finalizeFunc = NULL + }, + { + .name = "to_unixtimestamp", + .type = FUNCTION_TYPE_TO_UNIXTIMESTAMP, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateToUnixtimestamp, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = toUnixtimestampFunction, + .finalizeFunc = NULL + }, + { + .name = "timetruncate", + .type = FUNCTION_TYPE_TIMETRUNCATE, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateTimeTruncate, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = timeTruncateFunction, + .finalizeFunc = NULL + }, + { + .name = "timediff", + .type = FUNCTION_TYPE_TIMEDIFF, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateTimeDiff, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = timeDiffFunction, + .finalizeFunc = NULL + }, + { + .name = "now", + .type = FUNCTION_TYPE_NOW, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC, + .translateFunc = translateTimePseudoColumn, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = nowFunction, + .finalizeFunc = NULL + }, + { + .name = "today", + .type = FUNCTION_TYPE_TODAY, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC, + .translateFunc = translateTimePseudoColumn, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = todayFunction, + .finalizeFunc = NULL + }, + { + .name = "timezone", + .type = FUNCTION_TYPE_TIMEZONE, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateTimezone, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = timezoneFunction, + .finalizeFunc = NULL + }, + { + .name = "_rowts", + .type = FUNCTION_TYPE_ROWTS, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, + .translateFunc = translateTimePseudoColumn, + .getEnvFunc = getTimePseudoFuncEnv, + .initFunc = NULL, + .sprocessFunc = NULL, + .finalizeFunc = NULL + }, + { + .name = "tbname", + .type = FUNCTION_TYPE_TBNAME, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC, + .translateFunc = translateTbnameColumn, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = qTbnameFunction, + .finalizeFunc = NULL + }, + { + .name = "_qstartts", + .type = FUNCTION_TYPE_QSTARTTS, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC, + .translateFunc = translateTimePseudoColumn, + .getEnvFunc = getTimePseudoFuncEnv, + .initFunc = NULL, + .sprocessFunc = qStartTsFunction, + .finalizeFunc = NULL + }, + { + .name = "_qendts", + .type = FUNCTION_TYPE_QENDTS, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC, + .translateFunc = translateTimePseudoColumn, + .getEnvFunc = getTimePseudoFuncEnv, + .initFunc = NULL, + .sprocessFunc = qEndTsFunction, + .finalizeFunc = NULL + }, + { + .name = "_wstartts", + .type = FUNCTION_TYPE_WSTARTTS, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC, + .translateFunc = translateTimePseudoColumn, + .getEnvFunc = getTimePseudoFuncEnv, + .initFunc = NULL, + .sprocessFunc = winStartTsFunction, + .finalizeFunc = NULL + }, + { + .name = "_wendts", + .type = FUNCTION_TYPE_WENDTS, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC, + .translateFunc = translateTimePseudoColumn, + .getEnvFunc = getTimePseudoFuncEnv, + .initFunc = NULL, + .sprocessFunc = winEndTsFunction, + .finalizeFunc = NULL + }, + { + .name = "_wduration", + .type = FUNCTION_TYPE_WDURATION, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC, + .translateFunc = translateWduration, + .getEnvFunc = getTimePseudoFuncEnv, + .initFunc = NULL, + .sprocessFunc = winDurFunction, + .finalizeFunc = NULL + }, + { + .name = "to_json", + .type = FUNCTION_TYPE_TO_JSON, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateToJson, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = toJsonFunction, + .finalizeFunc = NULL + }, + { + .name = "_select_value", + .type = FUNCTION_TYPE_SELECT_VALUE, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC, + .translateFunc = translateSelectValue, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = NULL, + .finalizeFunc = NULL + } +}; +// clang-format on const int32_t funcMgtBuiltinsNum = (sizeof(funcMgtBuiltins) / sizeof(SBuiltinFuncDefinition)); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 43fe6a06edbb0f522534173dbb85f679561655e8..3c9eca85ddc4fc521227d215414cdff417dc319f 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -20,6 +20,8 @@ #include "tdatablock.h" #include "tpercentile.h" +#define HISTOGRAM_MAX_BINS_NUM 100 + typedef struct SSumRes { union { int64_t isum; @@ -36,27 +38,33 @@ typedef struct SAvgRes { typedef struct STopBotResItem { SVariant v; - uint64_t uid; // it is a table uid, used to extract tag data during building of the final result for the tag data + uint64_t uid; // it is a table uid, used to extract tag data during building of the final result for the tag data struct { - int32_t pageId; - int32_t offset; - } tuplePos; // tuple data of this chosen row + int32_t pageId; + int32_t offset; + } tuplePos; // tuple data of this chosen row } STopBotResItem; typedef struct STopBotRes { - STopBotResItem *pItems; + STopBotResItem* pItems; } STopBotRes; typedef struct SStddevRes { double result; int64_t count; - union {double quadraticDSum; int64_t quadraticISum;}; - union {double dsum; int64_t isum;}; + union { + double quadraticDSum; + int64_t quadraticISum; + }; + union { + double dsum; + int64_t isum; + }; } SStddevRes; typedef struct SPercentileInfo { double result; - tMemBucket *pMemBucket; + tMemBucket* pMemBucket; int32_t stage; double minval; double maxval; @@ -64,19 +72,47 @@ typedef struct SPercentileInfo { } SPercentileInfo; typedef struct SDiffInfo { - bool hasPrev; - bool includeNull; - bool ignoreNegative; - bool firstOutput; - union { int64_t i64; double d64;} prev; + bool hasPrev; + bool includeNull; + bool ignoreNegative; + bool firstOutput; + union { + int64_t i64; + double d64; + } prev; + + int64_t prevTs; } SDiffInfo; -#define SET_VAL(_info, numOfElem, res) \ - do { \ - if ((numOfElem) <= 0) { \ - break; \ - } \ - (_info)->numOfRes = (res); \ +typedef struct SSpreadInfo { + double result; + bool hasResult; + double min; + double max; +} SSpreadInfo; + +typedef struct SHistoFuncBin { + double lower; + double upper; + union { + int64_t count; + double percentage; + }; +} SHistoFuncBin; + +typedef struct SHistoFuncInfo { + int32_t numOfBins; + bool normalized; + SHistoFuncBin bins[]; +} SHistoFuncInfo; + + +#define SET_VAL(_info, numOfElem, res) \ + do { \ + if ((numOfElem) <= 0) { \ + break; \ + } \ + (_info)->numOfRes = (res); \ } while (0) #define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) @@ -85,7 +121,7 @@ typedef struct SDiffInfo { #define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ do { \ for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ + SqlFunctionCtx* __ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ __ctx->fpSet.process(__ctx); \ } \ } while (0); @@ -101,7 +137,7 @@ typedef struct SDiffInfo { #define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num) \ do { \ - _t *d = (_t *)((_col)->pData); \ + _t* d = (_t*)((_col)->pData); \ for (int32_t i = (_start); i < (_nrow) + (_start); ++i) { \ if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \ continue; \ @@ -111,8 +147,7 @@ typedef struct SDiffInfo { } \ } while (0) - -bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +bool functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { if (pResultInfo->initialized) { return false; } @@ -126,12 +161,12 @@ bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { } int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { - int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - pResInfo->isNullRes = (pResInfo->numOfRes == 0)? 1:0; - cleanupResultRowEntry(pResInfo); + pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; + /*cleanupResultRowEntry(pResInfo);*/ char* in = GET_ROWCELL_INTERBUF(pResInfo); colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); @@ -140,11 +175,11 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult) { - int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - pResInfo->isNullRes = (pResInfo->numOfRes == 0)? 1:0; + pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; cleanupResultRowEntry(pResInfo); char* in = finalResult; @@ -170,7 +205,7 @@ bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { * count function does need the finalize, if data is missing, the default value, which is 0, is used * count function does not use the pCtx->interResBuf to keep the intermediate buffer */ -int32_t countFunction(SqlFunctionCtx *pCtx) { +int32_t countFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; /* @@ -179,7 +214,7 @@ int32_t countFunction(SqlFunctionCtx *pCtx) { * 3. for primary key column, pInputCol->hasNull always be false, pInput->colDataAggIsSet == false; */ SInputColumnInfoData* pInput = &pCtx->input; - SColumnInfoData* pInputCol = pInput->pData[0]; + SColumnInfoData* pInputCol = pInput->pData[0]; if (pInput->colDataAggIsSet && pInput->totalRows == pInput->numOfRows) { numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull; ASSERT(numOfElem >= 0); @@ -192,14 +227,15 @@ int32_t countFunction(SqlFunctionCtx *pCtx) { numOfElem += 1; } } else { - //when counting on the primary time stamp column and no statistics data is presented, use the size value directly. + // when counting on the primary time stamp column and no statistics data is presented, use the size value + // directly. numOfElem = pInput->numOfRows; } } SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - char* buf = GET_ROWCELL_INTERBUF(pResInfo); - *((int64_t *)buf) += numOfElem; + char* buf = GET_ROWCELL_INTERBUF(pResInfo); + *((int64_t*)buf) += numOfElem; SET_VAL(pResInfo, numOfElem, 1); return TSDB_CODE_SUCCESS; @@ -207,7 +243,7 @@ int32_t countFunction(SqlFunctionCtx *pCtx) { #define LIST_ADD_N(_res, _col, _start, _rows, _t, numOfElem) \ do { \ - _t *d = (_t *)(_col->pData); \ + _t* d = (_t*)(_col->pData); \ for (int32_t i = (_start); i < (_rows) + (_start); ++i) { \ if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \ continue; \ @@ -217,13 +253,13 @@ int32_t countFunction(SqlFunctionCtx *pCtx) { } \ } while (0) -int32_t sumFunction(SqlFunctionCtx *pCtx) { +int32_t sumFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; // Only the pre-computing information loaded and actual data does not loaded SInputColumnInfoData* pInput = &pCtx->input; - SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0]; - int32_t type = pInput->pData[0]->info.type; + SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0]; + int32_t type = pInput->pData[0]->info.type; SSumRes* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -241,7 +277,7 @@ int32_t sumFunction(SqlFunctionCtx *pCtx) { } else { // computing based on the true data block SColumnInfoData* pCol = pInput->pData[0]; - int32_t start = pInput->startRowIndex; + int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) { @@ -286,7 +322,7 @@ bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -bool avgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +bool avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { if (!functionSetup(pCtx, pResultInfo)) { return false; } @@ -313,21 +349,21 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { switch (type) { case TSDB_DATA_TYPE_TINYINT: { - int8_t* plist = (int8_t*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { - continue; - } - - numOfElem += 1; - pAvgRes->count += 1; - pAvgRes->sum.isum += plist[i]; + int8_t* plist = (int8_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; } - break; + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.isum += plist[i]; } - case TSDB_DATA_TYPE_SMALLINT: { + break; + } + + case TSDB_DATA_TYPE_SMALLINT: { int16_t* plist = (int16_t*)pCol->pData; for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { @@ -409,22 +445,22 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SInputColumnInfoData* pInput = &pCtx->input; - int32_t type = pInput->pData[0]->info.type; - SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t type = pInput->pData[0]->info.type; + SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (IS_INTEGER_TYPE(type)) { - pAvgRes->result = pAvgRes->sum.isum / ((double) pAvgRes->count); + pAvgRes->result = pAvgRes->sum.isum / ((double)pAvgRes->count); } else { - pAvgRes->result = pAvgRes->sum.dsum / ((double) pAvgRes->count); + pAvgRes->result = pAvgRes->sum.dsum / ((double)pAvgRes->count); } return functionFinalize(pCtx, pBlock); } -EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow){ +EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { return FUNC_DATA_REQUIRED_STATIS_LOAD; } -bool maxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +bool maxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { if (!functionSetup(pCtx, pResultInfo)) { return false; } @@ -432,34 +468,34 @@ bool maxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { char* buf = GET_ROWCELL_INTERBUF(pResultInfo); switch (pCtx->resDataInfo.type) { case TSDB_DATA_TYPE_INT: - *((int32_t *)buf) = INT32_MIN; + *((int32_t*)buf) = INT32_MIN; break; case TSDB_DATA_TYPE_UINT: - *((uint32_t *)buf) = 0; + *((uint32_t*)buf) = 0; break; case TSDB_DATA_TYPE_FLOAT: - *((float *)buf) = -FLT_MAX; + *((float*)buf) = -FLT_MAX; break; case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_VAL(((double *)buf), -DBL_MAX); + SET_DOUBLE_VAL(((double*)buf), -DBL_MAX); break; case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)buf) = INT64_MIN; + *((int64_t*)buf) = INT64_MIN; break; case TSDB_DATA_TYPE_UBIGINT: - *((uint64_t *)buf) = 0; + *((uint64_t*)buf) = 0; break; case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)buf) = INT16_MIN; + *((int16_t*)buf) = INT16_MIN; break; case TSDB_DATA_TYPE_USMALLINT: - *((uint16_t *)buf) = 0; + *((uint16_t*)buf) = 0; break; case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)buf) = INT8_MIN; + *((int8_t*)buf) = INT8_MIN; break; case TSDB_DATA_TYPE_UTINYINT: - *((uint8_t *)buf) = 0; + *((uint8_t*)buf) = 0; break; case TSDB_DATA_TYPE_BOOL: *((int8_t*)buf) = 0; @@ -470,7 +506,7 @@ bool maxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { return true; } -bool minFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +bool minFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { if (!functionSetup(pCtx, pResultInfo)) { return false; // not initialized since it has been initialized } @@ -478,34 +514,34 @@ bool minFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { char* buf = GET_ROWCELL_INTERBUF(pResultInfo); switch (pCtx->resDataInfo.type) { case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)buf) = INT8_MAX; + *((int8_t*)buf) = INT8_MAX; break; case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *) buf = UINT8_MAX; + *(uint8_t*)buf = UINT8_MAX; break; case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)buf) = INT16_MAX; + *((int16_t*)buf) = INT16_MAX; break; case TSDB_DATA_TYPE_USMALLINT: - *((uint16_t *)buf) = UINT16_MAX; + *((uint16_t*)buf) = UINT16_MAX; break; case TSDB_DATA_TYPE_INT: - *((int32_t *)buf) = INT32_MAX; + *((int32_t*)buf) = INT32_MAX; break; case TSDB_DATA_TYPE_UINT: - *((uint32_t *)buf) = UINT32_MAX; + *((uint32_t*)buf) = UINT32_MAX; break; case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)buf) = INT64_MAX; + *((int64_t*)buf) = INT64_MAX; break; case TSDB_DATA_TYPE_UBIGINT: - *((uint64_t *)buf) = UINT64_MAX; + *((uint64_t*)buf) = UINT64_MAX; break; case TSDB_DATA_TYPE_FLOAT: - *((float *)buf) = FLT_MAX; + *((float*)buf) = FLT_MAX; break; case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_VAL(((double *)buf), DBL_MAX); + SET_DOUBLE_VAL(((double*)buf), DBL_MAX); break; case TSDB_DATA_TYPE_BOOL: *((int8_t*)buf) = 1; @@ -528,21 +564,21 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { #define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ do { \ for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ + SqlFunctionCtx* __ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ __ctx->fpSet.process(__ctx); \ } \ } while (0); -#define DO_UPDATE_SUBSID_RES(ctx, ts) \ - do { \ +#define DO_UPDATE_SUBSID_RES(ctx, ts) \ + do { \ for (int32_t _i = 0; _i < (ctx)->subsidiaries.num; ++_i) { \ - SqlFunctionCtx* __ctx = (ctx)->subsidiaries.pCtx[_i]; \ - if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ - __ctx->tag.i = (ts); \ - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ - } \ - __ctx->fpSet.process(__ctx); \ - } \ + SqlFunctionCtx* __ctx = (ctx)->subsidiaries.pCtx[_i]; \ + if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ + __ctx->tag.i = (ts); \ + __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ + } \ + __ctx->fpSet.process(__ctx); \ + } \ } while (0) #define UPDATE_DATA(ctx, left, right, num, sign, _ts) \ @@ -556,7 +592,7 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { #define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num) \ do { \ - _t *d = (_t *)((_col)->pData); \ + _t* d = (_t*)((_col)->pData); \ for (int32_t i = (_start); i < (_nrow) + (_start); ++i) { \ if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \ continue; \ @@ -566,17 +602,17 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { } \ } while (0) -int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { +int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { int32_t numOfElems = 0; SInputColumnInfoData* pInput = &pCtx->input; - SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0]; + SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0]; SColumnInfoData* pCol = pInput->pData[0]; - int32_t type = pCol->info.type; + int32_t type = pCol->info.type; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - char* buf = GET_ROWCELL_INTERBUF(pResInfo); + char* buf = GET_ROWCELL_INTERBUF(pResInfo); // data in current data block are qualified to the query if (pInput->colDataAggIsSet) { @@ -591,15 +627,15 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { int16_t index = 0; if (isMinFunc) { - tval = &pInput->pColumnDataAgg[0]->min; + tval = &pInput->pColumnDataAgg[0]->min; index = pInput->pColumnDataAgg[0]->minIndex; } else { - tval = &pInput->pColumnDataAgg[0]->max; + tval = &pInput->pColumnDataAgg[0]->max; index = pInput->pColumnDataAgg[0]->maxIndex; } // the index is the original position, not the relative position - TSKEY key = (pCtx->ptsList != NULL)? pCtx->ptsList[index]:TSKEY_INITIAL_VAL; + TSKEY key = (pCtx->ptsList != NULL) ? pCtx->ptsList[index] : TSKEY_INITIAL_VAL; if (IS_SIGNED_NUMERIC_TYPE(type)) { int64_t prev = 0; @@ -607,7 +643,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { int64_t val = GET_INT64_VAL(tval); if ((prev < val) ^ isMinFunc) { - *(int64_t*) buf = val; + *(int64_t*)buf = val; for (int32_t i = 0; i < (pCtx)->subsidiaries.num; ++i) { SqlFunctionCtx* __ctx = pCtx->subsidiaries.pCtx[i]; if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor @@ -624,7 +660,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { uint64_t val = GET_UINT64_VAL(tval); if ((prev < val) ^ isMinFunc) { - *(uint64_t*) buf = val; + *(uint64_t*)buf = val; for (int32_t i = 0; i < (pCtx)->subsidiaries.num; ++i) { SqlFunctionCtx* __ctx = pCtx->subsidiaries.pCtx[i]; if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor @@ -636,11 +672,11 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { } } } else if (type == TSDB_DATA_TYPE_DOUBLE) { - double val = GET_DOUBLE_VAL(tval); - UPDATE_DATA(pCtx, *(double*) buf, val, numOfElems, isMinFunc, key); + double val = GET_DOUBLE_VAL(tval); + UPDATE_DATA(pCtx, *(double*)buf, val, numOfElems, isMinFunc, key); } else if (type == TSDB_DATA_TYPE_FLOAT) { double val = GET_DOUBLE_VAL(tval); - UPDATE_DATA(pCtx, *(float*) buf, val, numOfElems, isMinFunc, key); + UPDATE_DATA(pCtx, *(float*)buf, val, numOfElems, isMinFunc, key); } return numOfElems; @@ -653,10 +689,10 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) { LOOPCHECK_N(*(int8_t*)buf, pCol, pCtx, int8_t, numOfRows, start, isMinFunc, numOfElems); } else if (type == TSDB_DATA_TYPE_SMALLINT) { - LOOPCHECK_N(*(int16_t*) buf, pCol, pCtx, int16_t, numOfRows, start, isMinFunc, numOfElems); + LOOPCHECK_N(*(int16_t*)buf, pCol, pCtx, int16_t, numOfRows, start, isMinFunc, numOfElems); } else if (type == TSDB_DATA_TYPE_INT) { - int32_t *pData = (int32_t*)pCol->pData; - int32_t *val = (int32_t*) buf; + int32_t* pData = (int32_t*)pCol->pData; + int32_t* val = (int32_t*)buf; for (int32_t i = start; i < start + numOfRows; ++i) { if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) { @@ -665,7 +701,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; - TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i) : 0; + TSKEY ts = (pCtx->ptsList != NULL) ? GET_TS_DATA(pCtx, i) : 0; DO_UPDATE_SUBSID_RES(pCtx, ts); } @@ -676,34 +712,34 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { qDebug("max value updated:%d", *retVal); #endif } else if (type == TSDB_DATA_TYPE_BIGINT) { - LOOPCHECK_N(*(int64_t*) buf, pCol, pCtx, int64_t, numOfRows, start, isMinFunc, numOfElems); + LOOPCHECK_N(*(int64_t*)buf, pCol, pCtx, int64_t, numOfRows, start, isMinFunc, numOfElems); } } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { if (type == TSDB_DATA_TYPE_UTINYINT) { - LOOPCHECK_N(*(uint8_t*) buf, pCol, pCtx, uint8_t, numOfRows, start, isMinFunc, numOfElems); + LOOPCHECK_N(*(uint8_t*)buf, pCol, pCtx, uint8_t, numOfRows, start, isMinFunc, numOfElems); } else if (type == TSDB_DATA_TYPE_USMALLINT) { - LOOPCHECK_N(*(uint16_t*) buf, pCol, pCtx, uint16_t, numOfRows, start, isMinFunc, numOfElems); + LOOPCHECK_N(*(uint16_t*)buf, pCol, pCtx, uint16_t, numOfRows, start, isMinFunc, numOfElems); } else if (type == TSDB_DATA_TYPE_UINT) { - LOOPCHECK_N(*(uint32_t*) buf, pCol, pCtx, uint32_t, numOfRows, start, isMinFunc, numOfElems); + LOOPCHECK_N(*(uint32_t*)buf, pCol, pCtx, uint32_t, numOfRows, start, isMinFunc, numOfElems); } else if (type == TSDB_DATA_TYPE_UBIGINT) { - LOOPCHECK_N(*(uint64_t*) buf, pCol, pCtx, uint64_t, numOfRows, start, isMinFunc, numOfElems); + LOOPCHECK_N(*(uint64_t*)buf, pCol, pCtx, uint64_t, numOfRows, start, isMinFunc, numOfElems); } } else if (type == TSDB_DATA_TYPE_DOUBLE) { - LOOPCHECK_N(*(double*) buf, pCol, pCtx, double, numOfRows, start, isMinFunc, numOfElems); + LOOPCHECK_N(*(double*)buf, pCol, pCtx, double, numOfRows, start, isMinFunc, numOfElems); } else if (type == TSDB_DATA_TYPE_FLOAT) { - LOOPCHECK_N(*(float*) buf, pCol, pCtx, float, numOfRows, start, isMinFunc, numOfElems); + LOOPCHECK_N(*(float*)buf, pCol, pCtx, float, numOfRows, start, isMinFunc, numOfElems); } return numOfElems; } -int32_t minFunction(SqlFunctionCtx *pCtx) { +int32_t minFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = doMinMaxHelper(pCtx, 1); SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); return TSDB_CODE_SUCCESS; } -int32_t maxFunction(SqlFunctionCtx *pCtx) { +int32_t maxFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = doMinMaxHelper(pCtx, 0); SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); return TSDB_CODE_SUCCESS; @@ -714,7 +750,7 @@ bool getStddevFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +bool stddevFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { if (!functionSetup(pCtx, pResultInfo)) { return false; } @@ -741,22 +777,22 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { switch (type) { case TSDB_DATA_TYPE_TINYINT: { - int8_t* plist = (int8_t*)pCol->pData; - for (int32_t i = start; i < numOfRows + start; ++i) { - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { - continue; - } - - numOfElem += 1; - pStddevRes->count += 1; - pStddevRes->isum += plist[i]; - pStddevRes->quadraticISum += plist[i] * plist[i]; + int8_t* plist = (int8_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + start; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; } - break; + numOfElem += 1; + pStddevRes->count += 1; + pStddevRes->isum += plist[i]; + pStddevRes->quadraticISum += plist[i] * plist[i]; } - case TSDB_DATA_TYPE_SMALLINT: { + break; + } + + case TSDB_DATA_TYPE_SMALLINT: { int16_t* plist = (int16_t*)pCol->pData; for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { @@ -843,15 +879,15 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SInputColumnInfoData* pInput = &pCtx->input; - int32_t type = pInput->pData[0]->info.type; - SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - double avg; + int32_t type = pInput->pData[0]->info.type; + SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + double avg; if (IS_INTEGER_TYPE(type)) { - avg = pStddevRes->isum / ((double) pStddevRes->count); - pStddevRes->result = sqrt(pStddevRes->quadraticISum/((double)pStddevRes->count) - avg*avg); + avg = pStddevRes->isum / ((double)pStddevRes->count); + pStddevRes->result = sqrt(pStddevRes->quadraticISum / ((double)pStddevRes->count) - avg * avg); } else { - avg = pStddevRes->dsum / ((double) pStddevRes->count); - pStddevRes->result = sqrt(pStddevRes->quadraticDSum/((double)pStddevRes->count) - avg*avg); + avg = pStddevRes->dsum / ((double)pStddevRes->count); + pStddevRes->result = sqrt(pStddevRes->quadraticDSum / ((double)pStddevRes->count) - avg * avg); } return functionFinalize(pCtx, pBlock); @@ -862,13 +898,13 @@ bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +bool percentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { if (!functionSetup(pCtx, pResultInfo)) { return false; } // in the first round, get the min-max value of all involved data - SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo); + SPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX); SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX); pInfo->numOfElems = 0; @@ -876,17 +912,17 @@ bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultI return true; } -int32_t percentileFunction(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); +int32_t percentileFunction(SqlFunctionCtx* pCtx) { + int32_t notNullElems = 0; + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; - SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0]; + SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0]; - SColumnInfoData *pCol = pInput->pData[0]; - int32_t type = pCol->info.type; + SColumnInfoData* pCol = pInput->pData[0]; + int32_t type = pCol->info.type; - SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + SPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { pInfo->stage += 1; @@ -931,7 +967,7 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) { continue; } - char *data = colDataGetData(pCol, i); + char* data = colDataGetData(pCol, i); double v = 0; GET_TYPED_DATA(v, double, pCtx->inputType, data); @@ -957,7 +993,7 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) { continue; } - char *data = colDataGetData(pCol, i); + char* data = colDataGetData(pCol, i); notNullElems += 1; tMemBucketPut(pInfo->pMemBucket, data, 1); @@ -969,12 +1005,12 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) { int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SVariant* pVal = &pCtx->param[1].param; - double v = pVal->nType == TSDB_DATA_TYPE_INT ? pVal->i : pVal->d; + double v = pVal->nType == TSDB_DATA_TYPE_INT ? pVal->i : pVal->d; - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo* ppInfo = (SPercentileInfo*)GET_ROWCELL_INTERBUF(pResInfo); - tMemBucket * pMemBucket = ppInfo->pMemBucket; + tMemBucket* pMemBucket = ppInfo->pMemBucket; if (pMemBucket != NULL && pMemBucket->total > 0) { // check for null SET_DOUBLE_VAL(&ppInfo->result, getPercentile(pMemBucket, v)); } @@ -994,19 +1030,19 @@ static FORCE_INLINE TSKEY getRowPTs(SColumnInfoData* pTsColInfo, int32_t rowInde return 0; } - return *(TSKEY*) colDataGetData(pTsColInfo, rowIndex); + return *(TSKEY*)colDataGetData(pTsColInfo, rowIndex); } // This ordinary first function does not care if current scan is ascending order or descending order scan // the OPTIMIZED version of first function will only handle the ascending order scan -int32_t firstFunction(SqlFunctionCtx *pCtx) { +int32_t firstFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - char* buf = GET_ROWCELL_INTERBUF(pResInfo); + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + char* buf = GET_ROWCELL_INTERBUF(pResInfo); SInputColumnInfoData* pInput = &pCtx->input; - SColumnInfoData* pInputCol = pInput->pData[0]; + SColumnInfoData* pInputCol = pInput->pData[0]; int32_t bytes = pInputCol->info.bytes; @@ -1016,12 +1052,12 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) { return 0; } - SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet)? pInput->pColumnDataAgg[0]:NULL; + SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet) ? pInput->pColumnDataAgg[0] : NULL; TSKEY startKey = getRowPTs(pInput->pPTS, 0); TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1); - int32_t blockDataOrder = (startKey <= endKey)? TSDB_ORDER_ASC:TSDB_ORDER_DESC; + int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; if (blockDataOrder == TSDB_ORDER_ASC) { // filter according to current result firstly @@ -1045,7 +1081,7 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) { if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) > cts) { memcpy(buf, data, bytes); *(TSKEY*)(buf + bytes) = cts; -// DO_UPDATE_TAG_COLUMNS(pCtx, ts); + // DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; break; @@ -1074,7 +1110,7 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) { if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) > cts) { memcpy(buf, data, bytes); *(TSKEY*)(buf + bytes) = cts; -// DO_UPDATE_TAG_COLUMNS(pCtx, ts); + // DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; break; } @@ -1085,14 +1121,14 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } -int32_t lastFunction(SqlFunctionCtx *pCtx) { +int32_t lastFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - char* buf = GET_ROWCELL_INTERBUF(pResInfo); + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + char* buf = GET_ROWCELL_INTERBUF(pResInfo); SInputColumnInfoData* pInput = &pCtx->input; - SColumnInfoData* pInputCol = pInput->pData[0]; + SColumnInfoData* pInputCol = pInput->pData[0]; int32_t bytes = pInputCol->info.bytes; @@ -1102,12 +1138,12 @@ int32_t lastFunction(SqlFunctionCtx *pCtx) { return 0; } - SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet)? pInput->pColumnDataAgg[0]:NULL; + SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet) ? pInput->pColumnDataAgg[0] : NULL; TSKEY startKey = getRowPTs(pInput->pPTS, 0); TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1); - int32_t blockDataOrder = (startKey <= endKey)? TSDB_ORDER_ASC:TSDB_ORDER_DESC; + int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; if (blockDataOrder == TSDB_ORDER_ASC) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { @@ -1141,7 +1177,7 @@ int32_t lastFunction(SqlFunctionCtx *pCtx) { memcpy(buf, data, bytes); *(TSKEY*)(buf + bytes) = cts; pResInfo->numOfRes = 1; -// DO_UPDATE_TAG_COLUMNS(pCtx, ts); + // DO_UPDATE_TAG_COLUMNS(pCtx, ts); } break; } @@ -1156,96 +1192,134 @@ bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { if (!functionSetup(pCtx, pResInfo)) { return false; } SDiffInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo); - pDiffInfo->hasPrev = false; + pDiffInfo->hasPrev = false; pDiffInfo->prev.i64 = 0; - pDiffInfo->ignoreNegative = false; // TODO set correct param + pDiffInfo->ignoreNegative = false; // TODO set correct param pDiffInfo->includeNull = false; pDiffInfo->firstOutput = false; return true; } -int32_t diffFunction(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SDiffInfo *pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo); +int32_t diffFunction(SqlFunctionCtx* pCtx) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SDiffInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo); SInputColumnInfoData* pInput = &pCtx->input; - SColumnInfoData* pInputCol = pInput->pData[0]; + SColumnInfoData* pInputCol = pInput->pData[0]; - bool isFirstBlock = (pDiffInfo->hasPrev == false); + bool isFirstBlock = (pDiffInfo->hasPrev == false); int32_t numOfElems = 0; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); -// int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - SColumnInfoData* pTsOutput = pCtx->pTsOutput; - TSKEY* tsList = (int64_t*)pInput->pPTS->pData; + TSKEY* tsList = (int64_t*)pInput->pPTS->pData; int32_t startOffset = pCtx->offset; switch (pInputCol->info.type) { case TSDB_DATA_TYPE_INT: { - SColumnInfoData *pOutput = (SColumnInfoData *)pCtx->pOutput; - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; + if (pCtx->order == TSDB_ORDER_ASC) { + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + int32_t pos = startOffset + (isFirstBlock ? (numOfElems - 1) : numOfElems); + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + if (pDiffInfo->includeNull) { + colDataSetNull_f(pOutput->nullbitmap, pos); + if (tsList != NULL) { + colDataAppendInt64(pTsOutput, pos, &tsList[i]); + } + + numOfElems += 1; + } + continue; + } - int32_t pos = startOffset + (isFirstBlock? (numOfElems-1):numOfElems); - if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - if (pDiffInfo->includeNull) { - colDataSetNull_f(pOutput->nullbitmap, pos); - if (tsList != NULL) { - colDataAppendInt64(pTsOutput, pos, &tsList[i]); + int32_t v = *(int32_t*)colDataGetData(pInputCol, i); + if (pDiffInfo->hasPrev) { + int32_t delta = (int32_t)(v - pDiffInfo->prev.i64); // direct previous may be null + if (delta < 0 && pDiffInfo->ignoreNegative) { + colDataSetNull_f(pOutput->nullbitmap, pos); + } else { + colDataAppendInt32(pOutput, pos, &delta); } - numOfElems += 1; + if (pTsOutput != NULL) { + colDataAppendInt64(pTsOutput, pos, &tsList[i]); + } } - continue; + + pDiffInfo->prev.i64 = v; + pDiffInfo->hasPrev = true; + numOfElems++; } + } else { + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + int32_t v = *(int32_t*)colDataGetData(pInputCol, i); + int32_t pos = startOffset + numOfElems; + + // there is a row of previous data block to be handled in the first place. + if (pDiffInfo->hasPrev) { + int32_t delta = (int32_t)(pDiffInfo->prev.i64 - v); // direct previous may be null + if (delta < 0 && pDiffInfo->ignoreNegative) { + colDataSetNull_f(pOutput->nullbitmap, pos); + } else { + colDataAppendInt32(pOutput, pos, &delta); + } - int32_t v = *(int32_t*) colDataGetData(pInputCol, i); - if (pDiffInfo->hasPrev) { - int32_t delta = (int32_t)(v - pDiffInfo->prev.i64); // direct previous may be null - if (delta < 0 && pDiffInfo->ignoreNegative) { - colDataSetNull_f(pOutput->nullbitmap, pos); - } else { - colDataAppendInt32(pOutput, pos, &delta); + if (pTsOutput != NULL) { + colDataAppendInt64(pTsOutput, pos, &pDiffInfo->prevTs); + } + pDiffInfo->hasPrev = false; } - if (pTsOutput != NULL) { - colDataAppendInt64(pTsOutput, pos, &tsList[i]); + // it is not the last row of current block + if (i < pInput->numOfRows + pInput->startRowIndex - 1) { + int32_t next = *(int32_t*)colDataGetData(pInputCol, i + 1); + + int32_t delta = v - next; // direct previous may be null + colDataAppendInt32(pOutput, pos, &delta); + + if (pTsOutput != NULL) { + colDataAppendInt64(pTsOutput, pos, &tsList[i]); + } + } else { + pDiffInfo->prev.i64 = v; + if (pTsOutput != NULL) { + pDiffInfo->prevTs = tsList[i]; + } + pDiffInfo->hasPrev = true; } + numOfElems++; } - pDiffInfo->prev.i64 = v; - pDiffInfo->hasPrev = true; - numOfElems++; } break; } case TSDB_DATA_TYPE_BIGINT: { - SColumnInfoData *pOutput = (SColumnInfoData *)pCtx->pOutput; - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; } int32_t v = 0; if (pDiffInfo->hasPrev) { - v = *(int64_t*) colDataGetData(pInputCol, i); + v = *(int64_t*)colDataGetData(pInputCol, i); int64_t delta = (int64_t)(v - pDiffInfo->prev.i64); // direct previous may be null if (pDiffInfo->ignoreNegative) { continue; } -// *(pOutput++) = delta; -// *pTimestamp = (tsList != NULL)? tsList[i]:0; -// -// pOutput += 1; -// pTimestamp += 1; + // *(pOutput++) = delta; + // *pTimestamp = (tsList != NULL)? tsList[i]:0; + // + // pOutput += 1; + // pTimestamp += 1; } pDiffInfo->prev.i64 = v; @@ -1359,11 +1433,11 @@ int32_t diffFunction(SqlFunctionCtx *pCtx) { #endif default: break; -// qError("error input type"); + // qError("error input type"); } // initial value is not set yet - if (!pDiffInfo->hasPrev || numOfElems <= 0) { + if (numOfElems <= 0) { /* * 1. current block and blocks before are full of null * 2. current block may be null value @@ -1371,48 +1445,40 @@ int32_t diffFunction(SqlFunctionCtx *pCtx) { assert(pCtx->hasNull); return 0; } else { -// for (int t = 0; t < pCtx->tagInfo.numOfTagCols; ++t) { -// SqlFunctionCtx* tagCtx = pCtx->tagInfo.pTagCtxList[t]; -// if (tagCtx->functionId == TSDB_FUNC_TAG_DUMMY) { -// aAggs[TSDB_FUNC_TAGPRJ].xFunction(tagCtx); -// } -// } - - int32_t forwardStep = (isFirstBlock) ? numOfElems - 1 : numOfElems; - return forwardStep; + return (isFirstBlock) ? numOfElems - 1 : numOfElems; } } bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { - SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1); + SValueNode* pkNode = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); pEnv->calcMemSize = sizeof(STopBotRes) + pkNode->datum.i * sizeof(STopBotResItem); return true; } -static STopBotRes *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STopBotRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); - pRes->pItems = (STopBotResItem*)((char*) pRes + sizeof(STopBotRes)); +static STopBotRes* getTopBotOutputInfo(SqlFunctionCtx* pCtx) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + STopBotRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); + pRes->pItems = (STopBotResItem*)((char*)pRes + sizeof(STopBotRes)); return pRes; } -static void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, - uint16_t type, uint64_t uid, SResultRowEntryInfo* pEntryInfo); +static void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, + uint64_t uid, SResultRowEntryInfo* pEntryInfo); static void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem); static void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem); -int32_t topFunction(SqlFunctionCtx *pCtx) { - int32_t numOfElems = 0; - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); +int32_t topFunction(SqlFunctionCtx* pCtx) { + int32_t numOfElems = 0; + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); -// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotRes) + POINTER_BYTES * pCtx->param[0].i)) { -// buildTopBotStruct(pRes, pCtx); -// } + // if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotRes) + POINTER_BYTES * pCtx->param[0].i)) { + // buildTopBotStruct(pRes, pCtx); + // } SInputColumnInfoData* pInput = &pCtx->input; - SColumnInfoData* pCol = pInput->pData[0]; + SColumnInfoData* pCol = pInput->pData[0]; int32_t type = pInput->pData[0]->info.type; @@ -1432,11 +1498,11 @@ int32_t topFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } -static int32_t topBotResComparFn(const void *p1, const void *p2, const void *param) { - uint16_t type = *(uint16_t *) param; +static int32_t topBotResComparFn(const void* p1, const void* p2, const void* param) { + uint16_t type = *(uint16_t*)param; - STopBotResItem *val1 = (STopBotResItem *) p1; - STopBotResItem *val2 = (STopBotResItem *) p2; + STopBotResItem* val1 = (STopBotResItem*)p1; + STopBotResItem* val2 = (STopBotResItem*)p2; if (IS_SIGNED_NUMERIC_TYPE(type)) { if (val1->v.i == val2->v.i) { @@ -1461,19 +1527,19 @@ static int32_t topBotResComparFn(const void *p1, const void *p2, const void *par void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, uint64_t uid, SResultRowEntryInfo* pEntryInfo) { - STopBotRes *pRes = getTopBotOutputInfo(pCtx); - int32_t maxSize = pCtx->param[1].param.i; + STopBotRes* pRes = getTopBotOutputInfo(pCtx); + int32_t maxSize = pCtx->param[1].param.i; SVariant val = {0}; taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); - STopBotResItem *pItems = pRes->pItems; + STopBotResItem* pItems = pRes->pItems; assert(pItems != NULL); // not full yet if (pEntryInfo->numOfRes < maxSize) { STopBotResItem* pItem = &pItems[pEntryInfo->numOfRes]; - pItem->v = val; + pItem->v = val; pItem->uid = uid; // save the data of this tuple @@ -1481,20 +1547,21 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData // allocate the buffer and keep the data of this row into the new allocated buffer pEntryInfo->numOfRes++; - taosheapsort((void *) pItems, sizeof(STopBotResItem), pEntryInfo->numOfRes, (const void *) &type, topBotResComparFn, false); - } else { // replace the minimum value in the result + taosheapsort((void*)pItems, sizeof(STopBotResItem), pEntryInfo->numOfRes, (const void*)&type, topBotResComparFn, + false); + } else { // replace the minimum value in the result if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pItems[0].v.i) || - (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) || - (IS_FLOAT_TYPE(type) && val.d > pItems[0].v.d)) { + (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) || (IS_FLOAT_TYPE(type) && val.d > pItems[0].v.d)) { // replace the old data and the coresponding tuple data STopBotResItem* pItem = &pItems[0]; - pItem->v = val; + pItem->v = val; pItem->uid = uid; // save the data of this tuple by over writing the old data copyTupleData(pCtx, rowIndex, pSrcBlock, pItem); - taosheapadjust((void *) pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void *) &type, topBotResComparFn, NULL, false); + taosheapadjust((void*)pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void*)&type, + topBotResComparFn, NULL, false); } } } @@ -1553,7 +1620,7 @@ void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS char* pStart = (char*)(nullList + pSrcBlock->info.numOfCols * sizeof(bool)); int32_t offset = 0; - for(int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) { continue; @@ -1574,17 +1641,17 @@ void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS } int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { - SResultRowEntryInfo *pEntryInfo = GET_RES_INFO(pCtx); - STopBotRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); + STopBotRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); pEntryInfo->complete = true; - int32_t type = pCtx->input.pData[0]->info.type; - int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + int32_t type = pCtx->input.pData[0]->info.type; + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); // todo assign the tag value and the corresponding row data int32_t currentRow = pBlock->info.rows; - switch(type) { + switch (type) { case TSDB_DATA_TYPE_INT: { for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { STopBotResItem* pItem = &pRes->pItems[i]; @@ -1602,12 +1669,12 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; - SFunctParam *pFuncParam = &pc->pExpr->base.pParam[0]; - int32_t srcSlotId = pFuncParam->pCol->slotId; - int32_t dstSlotId = pCtx->pExpr->base.resSchema.slotId; + SFunctParam* pFuncParam = &pc->pExpr->base.pParam[0]; + int32_t srcSlotId = pFuncParam->pCol->slotId; + int32_t dstSlotId = pCtx->pExpr->base.resSchema.slotId; int32_t ps = 0; - for(int32_t k = 0; k < srcSlotId; ++k) { + for (int32_t k = 0; k < srcSlotId; ++k) { SColumnInfoData* pSrcCol = taosArrayGet(pCtx->pSrcBlock->pDataBlock, k); ps += pSrcCol->info.bytes; } @@ -1630,3 +1697,132 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pEntryInfo->numOfRes; } + +bool getSpreadFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(SSpreadInfo); + return true; +} + +bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); + SET_DOUBLE_VAL(&pInfo->min, DBL_MAX); + SET_DOUBLE_VAL(&pInfo->max, -DBL_MAX); + pInfo->hasResult = false; + return true; +} + +int32_t spreadFunction(SqlFunctionCtx *pCtx) { + int32_t numOfElems = 0; + + // Only the pre-computing information loaded and actual data does not loaded + SInputColumnInfoData* pInput = &pCtx->input; + SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0]; + int32_t type = pInput->pData[0]->info.type; + + SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + if (pInput->colDataAggIsSet) { + numOfElems = pInput->numOfRows - pAgg->numOfNull; + if (numOfElems == 0) { + goto _spread_over; + } + double tmin = 0.0, tmax = 0.0; + if (IS_SIGNED_NUMERIC_TYPE(type)) { + tmin = (double)GET_INT64_VAL(&pAgg->min); + tmax = (double)GET_INT64_VAL(&pAgg->max); + } else if (IS_FLOAT_TYPE(type)) { + tmin = GET_DOUBLE_VAL(&pAgg->min); + tmax = GET_DOUBLE_VAL(&pAgg->max); + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + tmin = (double)GET_UINT64_VAL(&pAgg->min); + tmax = (double)GET_UINT64_VAL(&pAgg->max); + } + + if (GET_DOUBLE_VAL(&pInfo->min) > tmin) { + SET_DOUBLE_VAL(&pInfo->min, tmin); + } + + if (GET_DOUBLE_VAL(&pInfo->max) < tmax) { + SET_DOUBLE_VAL(&pInfo->max, tmax); + } + + } else { // computing based on the true data block + SColumnInfoData* pCol = pInput->pData[0]; + + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; + + // check the valid data one by one + for (int32_t i = start; i < pInput->numOfRows + start; ++i) { + if (colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + char *data = colDataGetData(pCol, i); + + double v = 0; + GET_TYPED_DATA(v, double, type, data); + if (v < GET_DOUBLE_VAL(&pInfo->min)) { + SET_DOUBLE_VAL(&pInfo->min, v); + } + + if (v > GET_DOUBLE_VAL(&pInfo->max)) { + SET_DOUBLE_VAL(&pInfo->max, v); + } + + numOfElems += 1; + } + } + +_spread_over: + // data in the check operation are all null, not output + SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); + if (numOfElems > 0) { + pInfo->hasResult = true; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + if (pInfo->hasResult == true) { + SET_DOUBLE_VAL(&pInfo->result, pInfo->max - pInfo->min); + } + return functionFinalize(pCtx, pBlock); +} + +bool getHistogramFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(SHistoFuncInfo) + HISTOGRAM_MAX_BINS_NUM * sizeof(SHistoFuncBin); + return true; +} + +bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); + char* binType = pCtx->param[1].param.pz; + char* binDesc = pCtx->param[2].param.pz; + int64_t nornalized = pCtx->param[3].param.i; + + + return true; +} + +int32_t histogramFunction(SqlFunctionCtx *pCtx) { + return TSDB_CODE_SUCCESS; +} + +int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + //if (pInfo->hasResult == true) { + // SET_DOUBLE_VAL(&pInfo->result, pInfo->max - pInfo->min); + //} + return functionFinalize(pCtx, pBlock); +} diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 0113da94eb08a419ff05d2c48768438d9acf5e0b..f8ef0f7d200e3a7479cda6afb4a06eaa8c0c6a43 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -21,11 +21,8 @@ #include "taos.h" #include "taoserror.h" #include "thash.h" -#include "builtins.h" -#include "catalog.h" #include "tudf.h" - typedef struct SFuncMgtService { SHashObj* pFuncNameHashTable; } SFuncMgtService; @@ -148,6 +145,10 @@ bool fmIsAggFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MG bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); } +bool fmIsSelectFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SELECT_FUNC); } + +bool fmIsTimelineFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_TIMELINE_FUNC); } + bool fmIsPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); } bool fmIsScanPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCAN_PC_FUNC); } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index df469993799f7f1e18b37e2116df79858f92c5dd..0a99ef61ce2c807e8be01c9d4b36ed3a1d6deaf7 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -481,8 +481,8 @@ void* decodeUdfResponse(const void* buf, SUdfResponse* rsp) { return (void*)buf; } -void freeUdfColumnData(SUdfColumnData *data) { - if (data->varLengthColumn) { +void freeUdfColumnData(SUdfColumnData *data, SUdfColumnMeta *meta) { + if (IS_VAR_DATA_TYPE(meta->type)) { taosMemoryFree(data->varLenCol.varOffsets); data->varLenCol.varOffsets = NULL; taosMemoryFree(data->varLenCol.payload); @@ -496,7 +496,7 @@ void freeUdfColumnData(SUdfColumnData *data) { } void freeUdfColumn(SUdfColumn* col) { - freeUdfColumnData(&col->colData); + freeUdfColumnData(&col->colData, &col->colMeta); } void freeUdfDataDataBlock(SUdfDataBlock *block) { @@ -528,8 +528,7 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo udfCol->colMeta.scale = col->info.scale; udfCol->colMeta.precision = col->info.precision; udfCol->colData.numOfRows = udfBlock->numOfRows; - udfCol->colData.varLengthColumn = IS_VAR_DATA_TYPE(udfCol->colMeta.type); - if (udfCol->colData.varLengthColumn) { + if (IS_VAR_DATA_TYPE(udfCol->colMeta.type)) { udfCol->colData.varLenCol.varOffsetsLen = sizeof(int32_t) * udfBlock->numOfRows; udfCol->colData.varLenCol.varOffsets = taosMemoryMalloc(udfCol->colData.varLenCol.varOffsetsLen); memcpy(udfCol->colData.varLenCol.varOffsets, col->varmeta.offset, udfCol->colData.varLenCol.varOffsetsLen); @@ -555,7 +554,7 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { block->info.numOfCols = 1; block->info.rows = udfCol->colData.numOfRows; - block->info.hasVarCol = udfCol->colData.varLengthColumn; + block->info.hasVarCol = IS_VAR_DATA_TYPE(udfCol->colMeta.type); block->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); taosArraySetSize(block->pDataBlock, 1); @@ -595,7 +594,9 @@ int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SS //TODO: free the array output->pDataBlock output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - taosArrayPush(output->pDataBlock, input->columnData); + for (int32_t i = 0; i < numOfCols; ++i) { + taosArrayPush(output->pDataBlock, (input + i)->columnData); + } return 0; } @@ -1176,7 +1177,7 @@ int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInte // input: interBuf // output: resultData int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) { - int8_t callType = TSDB_UDF_CALL_AGG_PROC; + int8_t callType = TSDB_UDF_CALL_AGG_FIN; int32_t err = callUdf(handle, callType, NULL, interBuf, NULL, NULL, resultData); return err; } @@ -1243,12 +1244,12 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult } SUdfUvSession *session = (SUdfUvSession *)handle; SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo); - udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); - udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize; memset(udfRes, 0, envSize); + udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); + udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; + udfRes->session = (SUdfUvSession *)handle; SUdfInterBuf buf = {0}; if (callUdfAggInit(handle, &buf) != 0) { @@ -1260,7 +1261,6 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult } int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { - SInputColumnInfoData* pInput = &pCtx->input; int32_t numOfCols = pInput->numOfInputCols; @@ -1320,13 +1320,15 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - SUdfInterBuf resultBuf = {.buf = udfRes->finalResBuf, - .bufLen = session->outputLen, - .numOfResult = udfRes->finalResNum}; + SUdfInterBuf resultBuf = {0}; SUdfInterBuf state = {.buf = udfRes->interResBuf, .bufLen = session->bufSize, .numOfResult = udfRes->interResNum}; callUdfAggFinalize(session, &state, &resultBuf); + + udfRes->finalResBuf = resultBuf.buf; + udfRes->finalResNum = resultBuf.numOfResult; + teardownUdf(session); if (resultBuf.numOfResult == 1) { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index ae24a832c8cea8d72d4f621c4802060aecfc96f9..f5e4a9c6e6cf059e3927eb86b9f3cf7482c0f9b3 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -75,12 +75,15 @@ typedef struct SUdf { char path[PATH_MAX]; uv_lib_t lib; + TUdfScalarProcFunc scalarProcFunc; - TUdfFreeUdfColumnFunc freeUdfColumn; TUdfAggStartFunc aggStartFunc; TUdfAggProcessFunc aggProcFunc; TUdfAggFinishFunc aggFinishFunc; + + TUdfInitFunc initFunc; + TUdfDestroyFunc destroyFunc; } SUdf; // TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix @@ -101,16 +104,23 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); return UDFC_CODE_LOAD_UDF_FAILURE; } - //TODO: init and destroy function + + char initFuncName[TSDB_FUNC_NAME_LEN+5] = {0}; + char *initSuffix = "_init"; + strcpy(initFuncName, udfName); + strncat(initFuncName, initSuffix, strlen(initSuffix)); + uv_dlsym(&udf->lib, initFuncName, (void**)(&udf->initFunc)); + + char destroyFuncName[TSDB_FUNC_NAME_LEN+5] = {0}; + char *destroySuffix = "_destroy"; + strcpy(destroyFuncName, udfName); + strncat(destroyFuncName, destroySuffix, strlen(destroySuffix)); + uv_dlsym(&udf->lib, destroyFuncName, (void**)(&udf->destroyFunc)); + if (udf->funcType == TSDB_FUNC_TYPE_SCALAR) { char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; strcpy(processFuncName, udfName); uv_dlsym(&udf->lib, processFuncName, (void **)(&udf->scalarProcFunc)); - char freeFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; - char *freeSuffix = "_free"; - strncpy(freeFuncName, processFuncName, strlen(processFuncName)); - strncat(freeFuncName, freeSuffix, strlen(freeSuffix)); - uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn)); } else if (udf->funcType == TSDB_FUNC_TYPE_AGGREGATE) { char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; strcpy(processFuncName, udfName); @@ -124,7 +134,7 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { char *finishSuffix = "_finish"; strncpy(finishFuncName, processFuncName, strlen(processFuncName)); strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); - uv_dlsym(&udf->lib, startFuncName, (void **)(&udf->aggFinishFunc)); + uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggFinishFunc)); //TODO: merge } return 0; @@ -164,6 +174,9 @@ void udfdProcessRequest(uv_work_t *req) { if (udf->state == UDF_STATE_INIT) { udf->state = UDF_STATE_LOADING; udfdLoadUdf(setup->udfName, udf); + if (udf->initFunc) { + udf->initFunc(); + } udf->state = UDF_STATE_READY; uv_cond_broadcast(&udf->condReady); uv_mutex_unlock(&udf->lock); @@ -175,7 +188,6 @@ void udfdProcessRequest(uv_work_t *req) { } SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle)); handle->udf = udf; - // TODO: allocate private structure and call init function and set it to handle SUdfResponse rsp; rsp.seqNum = request.seqNum; rsp.type = request.type; @@ -215,7 +227,7 @@ void udfdProcessRequest(uv_work_t *req) { udf->scalarProcFunc(&input, &output); convertUdfColumnToDataBlock(&output, &response.callRsp.resultData); - udf->freeUdfColumn(&output); + freeUdfColumn(&output); break; } case TSDB_UDF_CALL_AGG_INIT: { @@ -280,10 +292,12 @@ void udfdProcessRequest(uv_work_t *req) { if (unloadUdf) { uv_cond_destroy(&udf->condReady); uv_mutex_destroy(&udf->lock); + if (udf->destroyFunc) { + (udf->destroyFunc)(); + } uv_dlclose(&udf->lib); taosMemoryFree(udf); } - // TODO: call destroy and free udf private taosMemoryFree(handle); SUdfResponse response; diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c index b2367313ae2fbada9f3cc4abb30b5d1cca8a3c88..4384d326cbbdc64f9713c03f75d625707cb22809 100644 --- a/source/libs/function/test/udf1.c +++ b/source/libs/function/test/udf1.c @@ -18,52 +18,20 @@ int32_t udf1_destroy() { } int32_t udf1(SUdfDataBlock* block, SUdfColumn *resultCol) { - SUdfColumnData *resultData = &resultCol->colData; - resultData->numOfRows = block->numOfRows; - SUdfColumnData *srcData = &block->udfCols[0]->colData; - resultData->varLengthColumn = srcData->varLengthColumn; - - if (resultData->varLengthColumn) { - resultData->varLenCol.varOffsetsLen = srcData->varLenCol.varOffsetsLen; - resultData->varLenCol.varOffsets = malloc(resultData->varLenCol.varOffsetsLen); - memcpy(resultData->varLenCol.varOffsets, srcData->varLenCol.varOffsets, srcData->varLenCol.varOffsetsLen); - - resultData->varLenCol.payloadLen = srcData->varLenCol.payloadLen; - resultData->varLenCol.payload = malloc(resultData->varLenCol.payloadLen); - memcpy(resultData->varLenCol.payload, srcData->varLenCol.payload, srcData->varLenCol.payloadLen); - } else { - resultData->fixLenCol.nullBitmapLen = srcData->fixLenCol.nullBitmapLen; - resultData->fixLenCol.nullBitmap = malloc(resultData->fixLenCol.nullBitmapLen); - memcpy(resultData->fixLenCol.nullBitmap, srcData->fixLenCol.nullBitmap, srcData->fixLenCol.nullBitmapLen); - - resultData->fixLenCol.dataLen = srcData->fixLenCol.dataLen; - resultData->fixLenCol.data = malloc(resultData->fixLenCol.dataLen); - memcpy(resultData->fixLenCol.data, srcData->fixLenCol.data, srcData->fixLenCol.dataLen); - for (int32_t i = 0; i < resultData->numOfRows; ++i) { - *(resultData->fixLenCol.data + i * sizeof(int32_t)) = 88; - } - } - SUdfColumnMeta *meta = &resultCol->colMeta; meta->bytes = 4; meta->type = TSDB_DATA_TYPE_INT; meta->scale = 0; meta->precision = 0; - return 0; -} -int32_t udf1_free(SUdfColumn *col) { - SUdfColumnData *data = &col->colData; - if (data->varLengthColumn) { - free(data->varLenCol.varOffsets); - data->varLenCol.varOffsets = NULL; - free(data->varLenCol.payload); - data->varLenCol.payload = NULL; - } else { - free(data->fixLenCol.nullBitmap); - data->fixLenCol.nullBitmap = NULL; - free(data->fixLenCol.data); - data->fixLenCol.data = NULL; + SUdfColumnData *resultData = &resultCol->colData; + resultData->numOfRows = block->numOfRows; + SUdfColumnData *srcData = &block->udfCols[0]->colData; + + for (int32_t i = 0; i < resultData->numOfRows; ++i) { + int32_t luckyNum = 88; + udfColSetRow(resultCol, i, (char*)&luckyNum, false); } + return 0; } \ No newline at end of file diff --git a/source/libs/function/test/udf2.c b/source/libs/function/test/udf2.c index 83187c5855650437a215901126b22975baa6216d..69ed515d2bc4981c4e5adb6a1f71fae0b259626c 100644 --- a/source/libs/function/test/udf2.c +++ b/source/libs/function/test/udf2.c @@ -27,7 +27,7 @@ int32_t udf2_start(SUdfInterBuf *buf) { int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { int64_t sumSquares = *(int64_t*)interBuf->buf; for (int32_t i = 0; i < block->numOfCols; ++i) { - for (int32_t j = 0; j < block->numOfRows; ++i) { + for (int32_t j = 0; j < block->numOfRows; ++j) { SUdfColumn* col = block->udfCols[i]; //TODO: check the bitmap for null value int32_t* rows = (int32_t*)col->colData.fixLenCol.data; @@ -35,7 +35,7 @@ int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInte } } - *(int64_t*)newInterBuf = sumSquares; + *(int64_t*)(newInterBuf->buf) = sumSquares; newInterBuf->bufLen = sizeof(int64_t); //TODO: if all null value, numOfResult = 0; newInterBuf->numOfResult = 1; diff --git a/source/libs/index/inc/indexUtil.h b/source/libs/index/inc/indexUtil.h index 814d61afd737a2e455ff4670ee9192fe54ed3ec1..f1676ed411a5e2074667816d1746dc607dc0f44d 100644 --- a/source/libs/index/inc/indexUtil.h +++ b/source/libs/index/inc/indexUtil.h @@ -68,7 +68,7 @@ extern "C" { */ void iIntersection(SArray *interResults, SArray *finalResult); -/* multi sorted result intersection +/* multi sorted result union * input: [1, 2, 4, 5] * [2, 3, 4, 5] * [1, 4, 5] @@ -76,7 +76,7 @@ void iIntersection(SArray *interResults, SArray *finalResult); */ void iUnion(SArray *interResults, SArray *finalResult); -/* sorted array +/* see example * total: [1, 2, 4, 5, 7, 8] * except: [4, 5] * return: [1, 2, 7, 8] saved in total diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index f1fe68c55adaf74a07fa6618f7eb88ce2f42ace7..5e9e4e1d5742b5c84bb630246001c3ea076cba81 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -154,6 +154,7 @@ static SNode* logicConditionNodeCopy(const SLogicConditionNode* pSrc, SLogicCond } static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { + COPY_ALL_SCALAR_FIELDS; exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_CHAR_ARRAY_FIELD(functionName); COPY_SCALAR_FIELD(funcId); @@ -190,6 +191,7 @@ static SNode* nodeListNodeCopy(const SNodeListNode* pSrc, SNodeListNode* pDst) { static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) { COPY_SCALAR_FIELD(mode); CLONE_NODE_FIELD(pValues); + CLONE_NODE_FIELD(pWStartTs); return (SNode*)pDst; } @@ -268,11 +270,18 @@ static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pD COPY_ALL_SCALAR_FIELDS; COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pFuncs); - CLONE_NODE_FIELD(pFill); CLONE_NODE_FIELD(pTspk); return (SNode*)pDst; } +static SNode* logicFillCopy(const SFillLogicNode* pSrc, SFillLogicNode* pDst) { + COPY_ALL_SCALAR_FIELDS; + COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + CLONE_NODE_FIELD(pWStartTs); + CLONE_NODE_FIELD(pValues); + return (SNode*)pDst; +} + static SNode* logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pSortKeys); @@ -369,6 +378,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return logicExchangeCopy((const SExchangeLogicNode*)pNode, (SExchangeLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_WINDOW: return logicWindowCopy((const SWindowLogicNode*)pNode, (SWindowLogicNode*)pDst); + case QUERY_NODE_LOGIC_PLAN_FILL: + return logicFillCopy((const SFillLogicNode*)pNode, (SFillLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_SORT: return logicSortCopy((const SSortLogicNode*)pNode, (SSortLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_PARTITION: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 0e6ec4f945af8ba7a920e25e3a0a1ffeb6cf387e..5dcacc4354c4be7e4ea26ad97998d0d600a68f24 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -192,6 +192,8 @@ const char* nodesNodeName(ENodeType type) { return "LogicExchange"; case QUERY_NODE_LOGIC_PLAN_WINDOW: return "LogicWindow"; + case QUERY_NODE_LOGIC_PLAN_FILL: + return "LogicFill"; case QUERY_NODE_LOGIC_PLAN_SORT: return "LogicSort"; case QUERY_NODE_LOGIC_PLAN_PARTITION: @@ -222,6 +224,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiSort"; case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return "PhysiInterval"; + case QUERY_NODE_PHYSICAL_PLAN_FILL: + return "PhysiFill"; case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return "PhysiSessionWindow"; case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: @@ -564,6 +568,58 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkFillLogicPlanMode = "Mode"; +static const char* jkFillLogicPlanWStartTs = "WStartTs"; +static const char* jkFillLogicPlanValues = "Values"; +static const char* jkFillLogicPlanStartTime = "StartTime"; +static const char* jkFillLogicPlanEndTime = "EndTime"; + +static int32_t logicFillNodeToJson(const void* pObj, SJson* pJson) { + const SFillLogicNode* pNode = (const SFillLogicNode*)pObj; + + int32_t code = logicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillLogicPlanMode, pNode->mode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillLogicPlanWStartTs, nodeToJson, pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillLogicPlanValues, nodeToJson, pNode->pValues); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillLogicPlanStartTime, pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillLogicPlanEndTime, pNode->timeRange.ekey); + } + + return code; +} + +static int32_t jsonToLogicFillNode(const SJson* pJson, void* pObj) { + SFillLogicNode* pNode = (SFillLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkFillLogicPlanMode, pNode->mode); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillLogicPlanWStartTs, &pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillLogicPlanValues, &pNode->pValues); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillLogicPlanStartTime, &pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillLogicPlanEndTime, &pNode->timeRange.ekey); + } + + return code; +} + static const char* jkSortLogicPlanSortKeys = "SortKeys"; static int32_t logicSortNodeToJson(const void* pObj, SJson* pJson) { @@ -588,6 +644,259 @@ static int32_t jsonToLogicSortNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkPartitionLogicPlanPartitionKeys = "PartitionKeys"; + +static int32_t logicPartitionNodeToJson(const void* pObj, SJson* pJson) { + const SPartitionLogicNode* pNode = (const SPartitionLogicNode*)pObj; + + int32_t code = logicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkPartitionLogicPlanPartitionKeys, pNode->pPartitionKeys); + } + + return code; +} + +static int32_t jsonToLogicPartitionNode(const SJson* pJson, void* pObj) { + SPartitionLogicNode* pNode = (SPartitionLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkPartitionLogicPlanPartitionKeys, &pNode->pPartitionKeys); + } + + return code; +} + +static const char* jkSubplanIdQueryId = "QueryId"; +static const char* jkSubplanIdGroupId = "GroupId"; +static const char* jkSubplanIdSubplanId = "SubplanId"; + +static int32_t subplanIdToJson(const void* pObj, SJson* pJson) { + const SSubplanId* pNode = (const SSubplanId*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkSubplanIdQueryId, pNode->queryId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSubplanIdGroupId, pNode->groupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSubplanIdSubplanId, pNode->subplanId); + } + + return code; +} + +static int32_t jsonToSubplanId(const SJson* pJson, void* pObj) { + SSubplanId* pNode = (SSubplanId*)pObj; + + int32_t code = tjsonGetUBigIntValue(pJson, jkSubplanIdQueryId, &pNode->queryId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkSubplanIdGroupId, &pNode->groupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkSubplanIdSubplanId, &pNode->subplanId); + } + + return code; +} + +static const char* jkEndPointFqdn = "Fqdn"; +static const char* jkEndPointPort = "Port"; + +static int32_t epToJson(const void* pObj, SJson* pJson) { + const SEp* pNode = (const SEp*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkEndPointFqdn, pNode->fqdn); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkEndPointPort, pNode->port); + } + + return code; +} + +static int32_t jsonToEp(const SJson* pJson, void* pObj) { + SEp* pNode = (SEp*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkEndPointFqdn, pNode->fqdn); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetSmallIntValue(pJson, jkEndPointPort, &pNode->port); + } + + return code; +} + +static const char* jkEpSetInUse = "InUse"; +static const char* jkEpSetNumOfEps = "NumOfEps"; +static const char* jkEpSetEps = "Eps"; + +static int32_t epSetToJson(const void* pObj, SJson* pJson) { + const SEpSet* pNode = (const SEpSet*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkEpSetInUse, pNode->inUse); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkEpSetNumOfEps, pNode->numOfEps); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddArray(pJson, jkEpSetEps, epToJson, pNode->eps, sizeof(SEp), pNode->numOfEps); + } + + return code; +} + +static int32_t jsonToEpSet(const SJson* pJson, void* pObj) { + SEpSet* pNode = (SEpSet*)pObj; + + int32_t code = tjsonGetTinyIntValue(pJson, jkEpSetInUse, &pNode->inUse); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkEpSetNumOfEps, &pNode->numOfEps); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToArray(pJson, jkEpSetEps, jsonToEp, pNode->eps, sizeof(SEp)); + } + + return code; +} + +static const char* jkVgroupInfoVgId = "VgId"; +static const char* jkVgroupInfoHashBegin = "HashBegin"; +static const char* jkVgroupInfoHashEnd = "HashEnd"; +static const char* jkVgroupInfoEpSet = "EpSet"; +static const char* jkVgroupInfoNumOfTable = "NumOfTable"; + +static int32_t vgroupInfoToJson(const void* pObj, SJson* pJson) { + const SVgroupInfo* pNode = (const SVgroupInfo*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupInfoVgId, pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashBegin, pNode->hashBegin); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashEnd, pNode->hashEnd); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkVgroupInfoEpSet, epSetToJson, &pNode->epSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVgroupInfoNumOfTable, pNode->numOfTable); + } + + return code; +} + +static int32_t jsonToVgroupInfo(const SJson* pJson, void* pObj) { + SVgroupInfo* pNode = (SVgroupInfo*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkVgroupInfoVgId, &pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUIntValue(pJson, jkVgroupInfoHashBegin, &pNode->hashBegin); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUIntValue(pJson, jkVgroupInfoHashEnd, &pNode->hashEnd); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkVgroupInfoEpSet, jsonToEpSet, &pNode->epSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkVgroupInfoNumOfTable, &pNode->numOfTable); + } + + return code; +} + +static const char* jkVgroupsInfoNum = "Num"; +static const char* jkVgroupsInfoVgroups = "Vgroups"; + +static int32_t vgroupsInfoToJson(const void* pObj, SJson* pJson) { + const SVgroupsInfo* pNode = (const SVgroupsInfo*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupsInfoNum, pNode->numOfVgroups); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddArray(pJson, jkVgroupsInfoVgroups, vgroupInfoToJson, pNode->vgroups, sizeof(SVgroupInfo), + pNode->numOfVgroups); + } + + return code; +} + +static int32_t jsonToVgroupsInfo(const SJson* pJson, void* pObj) { + SVgroupsInfo* pNode = (SVgroupsInfo*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkVgroupsInfoNum, &pNode->numOfVgroups); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToArray(pJson, jkVgroupsInfoVgroups, jsonToVgroupInfo, pNode->vgroups, sizeof(SVgroupInfo)); + } + + return code; +} + +static const char* jkLogicSubplanId = "Id"; +static const char* jkLogicSubplanChildren = "Children"; +static const char* jkLogicSubplanRootNode = "RootNode"; +static const char* jkLogicSubplanType = "SubplanType"; +static const char* jkLogicSubplanVgroupsSize = "VgroupsSize"; +static const char* jkLogicSubplanVgroups = "Vgroups"; +static const char* jkLogicSubplanLevel = "Level"; +static const char* jkLogicSubplanSplitFlag = "SplitFlag"; + +static int32_t logicSubplanToJson(const void* pObj, SJson* pJson) { + const SLogicSubplan* pNode = (const SLogicSubplan*)pObj; + + int32_t code = tjsonAddObject(pJson, jkLogicSubplanId, subplanIdToJson, &pNode->id); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkLogicSubplanChildren, pNode->pChildren); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkLogicSubplanRootNode, nodeToJson, pNode->pNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkLogicSubplanType, pNode->subplanType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkLogicSubplanVgroupsSize, VGROUPS_INFO_SIZE(pNode->pVgroupList)); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkLogicSubplanVgroups, vgroupsInfoToJson, pNode->pVgroupList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkLogicSubplanLevel, pNode->level); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkLogicSubplanSplitFlag, pNode->splitFlag); + } + + return code; +} + +static int32_t jsonToLogicSubplan(const SJson* pJson, void* pObj) { + SLogicSubplan* pNode = (SLogicSubplan*)pObj; + + int32_t code = tjsonToObject(pJson, jkLogicSubplanId, jsonToSubplanId, &pNode->id); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkLogicSubplanChildren, &pNode->pChildren); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkLogicSubplanRootNode, (SNode**)&pNode->pNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkLogicSubplanType, pNode->subplanType); + } + int32_t objSize = 0; + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkLogicSubplanVgroupsSize, &objSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonMakeObject(pJson, jkLogicSubplanVgroups, jsonToVgroupsInfo, (void**)&pNode->pVgroupList, objSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkLogicSubplanLevel, &pNode->level); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkLogicSubplanSplitFlag, &pNode->splitFlag); + } + + return code; +} + static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; @@ -837,63 +1146,6 @@ static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) { retur static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) { return jsonToPhysiScanNode(pJson, pObj); } -static const char* jkEndPointFqdn = "Fqdn"; -static const char* jkEndPointPort = "Port"; - -static int32_t epToJson(const void* pObj, SJson* pJson) { - const SEp* pNode = (const SEp*)pObj; - - int32_t code = tjsonAddStringToObject(pJson, jkEndPointFqdn, pNode->fqdn); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkEndPointPort, pNode->port); - } - - return code; -} - -static int32_t jsonToEp(const SJson* pJson, void* pObj) { - SEp* pNode = (SEp*)pObj; - - int32_t code = tjsonGetStringValue(pJson, jkEndPointFqdn, pNode->fqdn); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetSmallIntValue(pJson, jkEndPointPort, &pNode->port); - } - - return code; -} - -static const char* jkEpSetInUse = "InUse"; -static const char* jkEpSetNumOfEps = "NumOfEps"; -static const char* jkEpSetEps = "Eps"; - -static int32_t epSetToJson(const void* pObj, SJson* pJson) { - const SEpSet* pNode = (const SEpSet*)pObj; - - int32_t code = tjsonAddIntegerToObject(pJson, jkEpSetInUse, pNode->inUse); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkEpSetNumOfEps, pNode->numOfEps); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddArray(pJson, jkEpSetEps, epToJson, pNode->eps, sizeof(SEp), pNode->numOfEps); - } - - return code; -} - -static int32_t jsonToEpSet(const SJson* pJson, void* pObj) { - SEpSet* pNode = (SEpSet*)pObj; - - int32_t code = tjsonGetTinyIntValue(pJson, jkEpSetInUse, &pNode->inUse); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetTinyIntValue(pJson, jkEpSetNumOfEps, &pNode->numOfEps); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonToArray(pJson, jkEpSetEps, jsonToEp, pNode->eps, sizeof(SEp)); - } - - return code; -} - static const char* jkSysTableScanPhysiPlanMnodeEpSet = "MnodeEpSet"; static const char* jkSysTableScanPhysiPlanShowRewrite = "ShowRewrite"; static const char* jkSysTableScanPhysiPlanAccountId = "AccountId"; @@ -1186,7 +1438,6 @@ static const char* jkIntervalPhysiPlanOffset = "Offset"; static const char* jkIntervalPhysiPlanSliding = "Sliding"; static const char* jkIntervalPhysiPlanIntervalUnit = "intervalUnit"; static const char* jkIntervalPhysiPlanSlidingUnit = "slidingUnit"; -static const char* jkIntervalPhysiPlanFill = "Fill"; static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { const SIntervalPhysiNode* pNode = (const SIntervalPhysiNode*)pObj; @@ -1207,9 +1458,6 @@ static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkIntervalPhysiPlanSlidingUnit, pNode->slidingUnit); } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkIntervalPhysiPlanFill, nodeToJson, pNode->pFill); - } return code; } @@ -1233,8 +1481,64 @@ static int32_t jsonToPhysiIntervalNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkIntervalPhysiPlanSlidingUnit, &pNode->slidingUnit); } + + return code; +} + +static const char* jkFillPhysiPlanMode = "Mode"; +static const char* jkFillPhysiPlanWStartTs = "WStartTs"; +static const char* jkFillPhysiPlanValues = "Values"; +static const char* jkFillPhysiPlanTargets = "Targets"; +static const char* jkFillPhysiPlanStartTime = "StartTime"; +static const char* jkFillPhysiPlanEndTime = "EndTime"; + +static int32_t physiFillNodeToJson(const void* pObj, SJson* pJson) { + const SFillPhysiNode* pNode = (const SFillPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillPhysiPlanMode, pNode->mode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillPhysiPlanWStartTs, nodeToJson, pNode->pWStartTs); + } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkIntervalPhysiPlanFill, (SNode**)&pNode->pFill); + code = tjsonAddObject(pJson, jkFillPhysiPlanValues, nodeToJson, pNode->pValues); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkFillPhysiPlanTargets, pNode->pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillPhysiPlanStartTime, pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillPhysiPlanEndTime, pNode->timeRange.ekey); + } + + return code; +} + +static int32_t jsonToPhysiFillNode(const SJson* pJson, void* pObj) { + SFillPhysiNode* pNode = (SFillPhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkFillPhysiPlanMode, pNode->mode); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillPhysiPlanWStartTs, &pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillPhysiPlanValues, &pNode->pValues); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkFillPhysiPlanTargets, &pNode->pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillPhysiPlanStartTime, &pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillPhysiPlanEndTime, &pNode->timeRange.ekey); } return code; @@ -1342,38 +1646,6 @@ static int32_t physiDispatchNodeToJson(const void* pObj, SJson* pJson) { return static int32_t jsonToPhysiDispatchNode(const SJson* pJson, void* pObj) { return jsonToPhysicDataSinkNode(pJson, pObj); } -static const char* jkSubplanIdQueryId = "QueryId"; -static const char* jkSubplanIdGroupId = "GroupId"; -static const char* jkSubplanIdSubplanId = "SubplanId"; - -static int32_t subplanIdToJson(const void* pObj, SJson* pJson) { - const SSubplanId* pNode = (const SSubplanId*)pObj; - - int32_t code = tjsonAddIntegerToObject(pJson, jkSubplanIdQueryId, pNode->queryId); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkSubplanIdGroupId, pNode->groupId); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkSubplanIdSubplanId, pNode->subplanId); - } - - return code; -} - -static int32_t jsonToSubplanId(const SJson* pJson, void* pObj) { - SSubplanId* pNode = (SSubplanId*)pObj; - - int32_t code = tjsonGetUBigIntValue(pJson, jkSubplanIdQueryId, &pNode->queryId); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkSubplanIdGroupId, &pNode->groupId); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkSubplanIdSubplanId, &pNode->subplanId); - } - - return code; -} - static const char* jkQueryNodeAddrId = "Id"; static const char* jkQueryNodeAddrInUse = "InUse"; static const char* jkQueryNodeAddrNumOfEps = "NumOfEps"; @@ -1699,7 +1971,18 @@ static int32_t datumToJson(const void* pObj, SJson* pJson) { case TSDB_DATA_TYPE_DOUBLE: code = tjsonAddDoubleToObject(pJson, jkValueDatum, pNode->datum.d); break; - case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_NCHAR: { + //cJSON only support utf-8 encoding. Convert memory content to hex string. + char *buf = taosMemoryCalloc(varDataLen(pNode->datum.p) * 2 + 1, sizeof(char)); + code = taosHexEncode(varDataVal(pNode->datum.p), buf, varDataLen(pNode->datum.p)); + if(code != TSDB_CODE_SUCCESS) { + taosMemoryFree(buf); + return TSDB_CODE_TSC_INVALID_VALUE; + } + code = tjsonAddStringToObject(pJson, jkValueDatum, buf); + taosMemoryFree(buf); + break; + } case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: code = tjsonAddStringToObject(pJson, jkValueDatum, varDataVal(pNode->datum.p)); @@ -1747,23 +2030,51 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) { break; case TSDB_DATA_TYPE_BOOL: code = tjsonGetBoolValue(pJson, jkValueDatum, &pNode->datum.b); + *(bool*)&pNode->typeData = pNode->datum.b; break; case TSDB_DATA_TYPE_TINYINT: + code = tjsonGetBigIntValue(pJson, jkValueDatum, &pNode->datum.i); + *(int8_t*)&pNode->typeData = pNode->datum.i; + break; case TSDB_DATA_TYPE_SMALLINT: + code = tjsonGetBigIntValue(pJson, jkValueDatum, &pNode->datum.i); + *(int16_t*)&pNode->typeData = pNode->datum.i; + break; case TSDB_DATA_TYPE_INT: + code = tjsonGetBigIntValue(pJson, jkValueDatum, &pNode->datum.i); + *(int32_t*)&pNode->typeData = pNode->datum.i; + break; case TSDB_DATA_TYPE_BIGINT: + code = tjsonGetBigIntValue(pJson, jkValueDatum, &pNode->datum.i); + *(int64_t*)&pNode->typeData = pNode->datum.i; + break; case TSDB_DATA_TYPE_TIMESTAMP: code = tjsonGetBigIntValue(pJson, jkValueDatum, &pNode->datum.i); + *(int64_t*)&pNode->typeData = pNode->datum.i; break; case TSDB_DATA_TYPE_UTINYINT: + code = tjsonGetUBigIntValue(pJson, jkValueDatum, &pNode->datum.u); + *(uint8_t*)&pNode->typeData = pNode->datum.u; + break; case TSDB_DATA_TYPE_USMALLINT: + code = tjsonGetUBigIntValue(pJson, jkValueDatum, &pNode->datum.u); + *(uint16_t*)&pNode->typeData = pNode->datum.u; + break; case TSDB_DATA_TYPE_UINT: + code = tjsonGetUBigIntValue(pJson, jkValueDatum, &pNode->datum.u); + *(uint32_t*)&pNode->typeData = pNode->datum.u; + break; case TSDB_DATA_TYPE_UBIGINT: code = tjsonGetUBigIntValue(pJson, jkValueDatum, &pNode->datum.u); + *(uint64_t*)&pNode->typeData = pNode->datum.u; break; case TSDB_DATA_TYPE_FLOAT: + code = tjsonGetDoubleValue(pJson, jkValueDatum, &pNode->datum.d); + *(float*)&pNode->typeData = pNode->datum.d; + break; case TSDB_DATA_TYPE_DOUBLE: code = tjsonGetDoubleValue(pJson, jkValueDatum, &pNode->datum.d); + *(double*)&pNode->typeData = pNode->datum.d; break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: @@ -1774,7 +2085,26 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) { break; } varDataSetLen(pNode->datum.p, pNode->node.resType.bytes); - code = tjsonGetStringValue(pJson, jkValueDatum, varDataVal(pNode->datum.p)); + if (TSDB_DATA_TYPE_NCHAR == pNode->node.resType.type) { + char *buf = taosMemoryCalloc(1, pNode->node.resType.bytes * 2 + VARSTR_HEADER_SIZE + 1); + if (NULL == buf) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } + code = tjsonGetStringValue(pJson, jkValueDatum, buf); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(buf); + break; + } + code = taosHexDecode(buf, varDataVal(pNode->datum.p), pNode->node.resType.bytes - VARSTR_HEADER_SIZE); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(buf); + break; + } + taosMemoryFree(buf); + } else { + code = tjsonGetStringValue(pJson, jkValueDatum, varDataVal(pNode->datum.p)); + } break; } case TSDB_DATA_TYPE_JSON: @@ -1885,6 +2215,7 @@ static const char* jkFunctionName = "Name"; static const char* jkFunctionId = "Id"; static const char* jkFunctionType = "Type"; static const char* jkFunctionParameter = "Parameters"; +static const char* jkFunctionUdfBufSize = "UdfBufSize"; static int32_t functionNodeToJson(const void* pObj, SJson* pJson) { const SFunctionNode* pNode = (const SFunctionNode*)pObj; @@ -1902,6 +2233,9 @@ static int32_t functionNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkFunctionParameter, pNode->pParameterList); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFunctionUdfBufSize, pNode->udfBufSize); + } return code; } @@ -1922,6 +2256,9 @@ static int32_t jsonToFunctionNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkFunctionParameter, &pNode->pParameterList); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkFunctionUdfBufSize, &pNode->udfBufSize); + } return code; } @@ -1964,78 +2301,6 @@ static int32_t jsonToTableNode(const SJson* pJson, void* pObj) { return code; } -static const char* jkVgroupInfoVgId = "VgId"; -static const char* jkVgroupInfoHashBegin = "HashBegin"; -static const char* jkVgroupInfoHashEnd = "HashEnd"; -static const char* jkVgroupInfoEpSet = "EpSet"; -static const char* jkVgroupInfoNumOfTable = "NumOfTable"; - -static int32_t vgroupInfoToJson(const void* pObj, SJson* pJson) { - const SVgroupInfo* pNode = (const SVgroupInfo*)pObj; - - int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupInfoVgId, pNode->vgId); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashBegin, pNode->hashBegin); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashEnd, pNode->hashEnd); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkVgroupInfoEpSet, epSetToJson, &pNode->epSet); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkVgroupInfoNumOfTable, pNode->numOfTable); - } - - return code; -} - -static int32_t jsonToVgroupInfo(const SJson* pJson, void* pObj) { - SVgroupInfo* pNode = (SVgroupInfo*)pObj; - - int32_t code = tjsonGetIntValue(pJson, jkVgroupInfoVgId, &pNode->vgId); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetUIntValue(pJson, jkVgroupInfoHashBegin, &pNode->hashBegin); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetUIntValue(pJson, jkVgroupInfoHashEnd, &pNode->hashEnd); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonToObject(pJson, jkVgroupInfoEpSet, jsonToEpSet, &pNode->epSet); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkVgroupInfoNumOfTable, &pNode->numOfTable); - } - - return code; -} - -static const char* jkVgroupsInfoNum = "Num"; -static const char* jkVgroupsInfoVgroups = "Vgroups"; - -static int32_t vgroupsInfoToJson(const void* pObj, SJson* pJson) { - const SVgroupsInfo* pNode = (const SVgroupsInfo*)pObj; - - int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupsInfoNum, pNode->numOfVgroups); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddArray(pJson, jkVgroupsInfoVgroups, vgroupInfoToJson, pNode->vgroups, sizeof(SVgroupInfo), - pNode->numOfVgroups); - } - - return code; -} - -static int32_t jsonToVgroupsInfo(const SJson* pJson, void* pObj) { - SVgroupsInfo* pNode = (SVgroupsInfo*)pObj; - - int32_t code = tjsonGetIntValue(pJson, jkVgroupsInfoNum, &pNode->numOfVgroups); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonToArray(pJson, jkVgroupsInfoVgroups, jsonToVgroupInfo, pNode->vgroups, sizeof(SVgroupInfo)); - } - - return code; -} - static const char* jkRealTableMetaSize = "MetaSize"; static const char* jkRealTableMeta = "Meta"; static const char* jkRealTableVgroupsInfoSize = "VgroupsInfoSize"; @@ -2201,6 +2466,9 @@ static int32_t jsonToNodeListNode(const SJson* pJson, void* pObj) { static const char* jkFillMode = "Mode"; static const char* jkFillValues = "Values"; +static const char* jkFillWStartTs = "WStartTs"; +static const char* jkFillStartTime = "StartTime"; +static const char* jkFillEndTime = "EndTime"; static int32_t fillNodeToJson(const void* pObj, SJson* pJson) { const SFillNode* pNode = (const SFillNode*)pObj; @@ -2209,6 +2477,15 @@ static int32_t fillNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkFillValues, nodeToJson, pNode->pValues); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillWStartTs, nodeToJson, pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillStartTime, pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillEndTime, pNode->timeRange.ekey); + } return code; } @@ -2220,6 +2497,15 @@ static int32_t jsonToFillNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillWStartTs, &pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillStartTime, &pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillEndTime, &pNode->timeRange.ekey); + } return code; } @@ -2580,9 +2866,14 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return logicProjectNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: break; + case QUERY_NODE_LOGIC_PLAN_FILL: + return logicFillNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_SORT: return logicSortNodeToJson(pObj, pJson); + case QUERY_NODE_LOGIC_PLAN_PARTITION: + return logicPartitionNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_SUBPLAN: + return logicSubplanToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN: break; case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: @@ -2605,6 +2896,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiSortNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return physiIntervalNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_FILL: + return physiFillNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return physiSessionWindowNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: @@ -2665,8 +2958,14 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicScanNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_PROJECT: return jsonToLogicProjectNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_FILL: + return jsonToLogicFillNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_SORT: return jsonToLogicSortNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_PARTITION: + return jsonToLogicPartitionNode(pJson, pObj); + case QUERY_NODE_LOGIC_SUBPLAN: + return jsonToLogicSubplan(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: return jsonToPhysiTagScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: @@ -2687,6 +2986,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiSortNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return jsonToPhysiIntervalNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_FILL: + return jsonToPhysiFillNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return jsonToPhysiSessionWindowNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 88f5d914e4998234367ebd03803aa5b75bf47850..f2c043b9ea7b27e46748fa4893a26aa03da9ca47 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -132,9 +132,14 @@ static EDealRes dispatchExpr(SNode* pNode, ETraversalOrder order, FNodeWalker wa case QUERY_NODE_NODE_LIST: res = walkExprs(((SNodeListNode*)pNode)->pNodeList, order, walker, pContext); break; - case QUERY_NODE_FILL: - res = walkExpr(((SFillNode*)pNode)->pValues, order, walker, pContext); + case QUERY_NODE_FILL: { + SFillNode* pFill = (SFillNode*)pNode; + res = walkExpr(pFill->pValues, order, walker, pContext); + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = walkExpr(pFill->pWStartTs, order, walker, pContext); + } break; + } case QUERY_NODE_RAW_EXPR: res = walkExpr(((SRawExprNode*)pNode)->pNode, order, walker, pContext); break; @@ -272,9 +277,14 @@ static EDealRes rewriteExpr(SNode** pRawNode, ETraversalOrder order, FNodeRewrit case QUERY_NODE_NODE_LIST: res = rewriteExprs(((SNodeListNode*)pNode)->pNodeList, order, rewriter, pContext); break; - case QUERY_NODE_FILL: - res = rewriteExpr(&(((SFillNode*)pNode)->pValues), order, rewriter, pContext); + case QUERY_NODE_FILL: { + SFillNode* pFill = (SFillNode*)pNode; + res = rewriteExpr(&pFill->pValues, order, rewriter, pContext); + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = rewriteExpr(&(pFill->pWStartTs), order, rewriter, pContext); + } break; + } case QUERY_NODE_RAW_EXPR: res = rewriteExpr(&(((SRawExprNode*)pNode)->pNode), order, rewriter, pContext); break; @@ -333,6 +343,9 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa case SQL_CLAUSE_PARTITION_BY: nodesWalkExpr(pSelect->pWindow, walker, pContext); case SQL_CLAUSE_WINDOW: + if (NULL != pSelect->pWindow && QUERY_NODE_INTERVAL_WINDOW == nodeType(pSelect->pWindow)) { + nodesWalkExpr(((SIntervalWindowNode*)pSelect->pWindow)->pFill, walker, pContext); + } nodesWalkExprs(pSelect->pGroupByList, walker, pContext); case SQL_CLAUSE_GROUP_BY: nodesWalkExpr(pSelect->pHaving, walker, pContext); @@ -362,6 +375,9 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit case SQL_CLAUSE_PARTITION_BY: nodesRewriteExpr(&(pSelect->pWindow), rewriter, pContext); case SQL_CLAUSE_WINDOW: + if (NULL != pSelect->pWindow && QUERY_NODE_INTERVAL_WINDOW == nodeType(pSelect->pWindow)) { + nodesRewriteExpr(&(((SIntervalWindowNode*)pSelect->pWindow)->pFill), rewriter, pContext); + } nodesRewriteExprs(pSelect->pGroupByList, rewriter, pContext); case SQL_CLAUSE_GROUP_BY: nodesRewriteExpr(&(pSelect->pHaving), rewriter, pContext); @@ -496,14 +512,9 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk } break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { - SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)pNode; + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext); - if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { - res = walkPhysiPlan((SNode*)pInterval->pFill, order, walker, pContext); - } break; - } case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext); break; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 8e48df284ffa80101b149568b0838ab15b839378..0bceb84f4a2807406b1877d5a938316934d009b1 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -213,6 +213,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SExchangeLogicNode)); case QUERY_NODE_LOGIC_PLAN_WINDOW: return makeNode(type, sizeof(SWindowLogicNode)); + case QUERY_NODE_LOGIC_PLAN_FILL: + return makeNode(type, sizeof(SFillLogicNode)); case QUERY_NODE_LOGIC_PLAN_SORT: return makeNode(type, sizeof(SSortLogicNode)); case QUERY_NODE_LOGIC_PLAN_PARTITION: @@ -243,6 +245,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SSortPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_FILL: + return makeNode(type, sizeof(SFillPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return makeNode(type, sizeof(SSessionWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: @@ -373,9 +377,12 @@ void nodesDestroyNode(SNodeptr pNode) { case QUERY_NODE_NODE_LIST: nodesDestroyList(((SNodeListNode*)pNode)->pNodeList); break; - case QUERY_NODE_FILL: - nodesDestroyNode(((SFillNode*)pNode)->pValues); + case QUERY_NODE_FILL: { + SFillNode* pFill = (SFillNode*)pNode; + nodesDestroyNode(pFill->pValues); + nodesDestroyNode(pFill->pWStartTs); break; + } case QUERY_NODE_RAW_EXPR: nodesDestroyNode(((SRawExprNode*)pNode)->pNode); break; @@ -554,7 +561,6 @@ void nodesDestroyNode(SNodeptr pNode) { SWindowLogicNode* pLogicNode = (SWindowLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyList(pLogicNode->pFuncs); - nodesDestroyNode(pLogicNode->pFill); nodesDestroyNode(pLogicNode->pTspk); break; } @@ -630,12 +636,9 @@ void nodesDestroyNode(SNodeptr pNode) { nodesDestroyNode(pPhyNode->pSortKeys); break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { - SIntervalPhysiNode* pPhyNode = (SIntervalPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); - nodesDestroyNode(pPhyNode->pFill); + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); break; - } case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); break; @@ -871,21 +874,18 @@ void nodesClearList(SNodeList* pList) { void* nodesGetValueFromNode(SValueNode* pNode) { switch (pNode->node.resType.type) { case TSDB_DATA_TYPE_BOOL: - return (void*)&pNode->datum.b; case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: - return (void*)&pNode->datum.i; case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: - return (void*)&pNode->datum.u; case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: - return (void*)&pNode->datum.d; + return (void*)&pNode->typeData; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: @@ -897,6 +897,68 @@ void* nodesGetValueFromNode(SValueNode* pNode) { return NULL; } +int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value) { + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_BOOL: + pNode->datum.b = *(bool*)value; + *(bool*)&pNode->typeData = pNode->datum.b; + break; + case TSDB_DATA_TYPE_TINYINT: + pNode->datum.i = *(int8_t*)value; + *(int8_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_SMALLINT: + pNode->datum.i = *(int16_t*)value; + *(int16_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_INT: + pNode->datum.i = *(int32_t*)value; + *(int32_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_BIGINT: + pNode->datum.i = *(int64_t*)value; + *(int64_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_TIMESTAMP: + pNode->datum.i = *(int64_t*)value; + *(int64_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_UTINYINT: + pNode->datum.u = *(int8_t*)value; + *(int8_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_USMALLINT: + pNode->datum.u = *(int16_t*)value; + *(int16_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_UINT: + pNode->datum.u = *(int32_t*)value; + *(int32_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_UBIGINT: + pNode->datum.u = *(uint64_t*)value; + *(uint64_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_FLOAT: + pNode->datum.d = *(float*)value; + *(float*)&pNode->typeData = pNode->datum.d; + break; + case TSDB_DATA_TYPE_DOUBLE: + pNode->datum.d = *(double*)value; + *(double*)&pNode->typeData = pNode->datum.d; + break; + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + pNode->datum.p = (char*)value; + break; + default: + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + char* nodesGetStrValueFromNode(SValueNode* pNode) { switch (pNode->node.resType.type) { case TSDB_DATA_TYPE_BOOL: { @@ -1090,30 +1152,31 @@ static EDealRes collectColumns(SNode* pNode, void* pContext) { int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type, SNodeList** pCols) { if (NULL == pSelect || NULL == pCols) { - return TSDB_CODE_SUCCESS; + return TSDB_CODE_FAILED; } SCollectColumnsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pTableAlias = pTableAlias, .collectType = type, - .pCols = nodesMakeList(), + .pCols = (NULL == *pCols ? nodesMakeList() : *pCols), .pColHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK)}; if (NULL == cxt.pCols || NULL == cxt.pColHash) { return TSDB_CODE_OUT_OF_MEMORY; } - + *pCols = NULL; nodesWalkSelectStmt(pSelect, clause, collectColumns, &cxt); taosHashCleanup(cxt.pColHash); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(cxt.pCols); return cxt.errCode; } - if (0 == LIST_LENGTH(cxt.pCols)) { + if (LIST_LENGTH(cxt.pCols) > 0) { + *pCols = cxt.pCols; + } else { nodesDestroyList(cxt.pCols); - cxt.pCols = NULL; } - *pCols = cxt.pCols; + return TSDB_CODE_SUCCESS; } @@ -1132,9 +1195,9 @@ static EDealRes collectFuncs(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs) { +int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifier classifier, SNodeList** pFuncs) { if (NULL == pSelect || NULL == pFuncs) { - return TSDB_CODE_SUCCESS; + return TSDB_CODE_FAILED; } SCollectFuncsCxt cxt = { @@ -1143,7 +1206,7 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNod return TSDB_CODE_OUT_OF_MEMORY; } *pFuncs = NULL; - nodesWalkSelectStmt(pSelect, SQL_CLAUSE_GROUP_BY, collectFuncs, &cxt); + nodesWalkSelectStmt(pSelect, clause, collectFuncs, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(cxt.pFuncs); return cxt.errCode; @@ -1157,6 +1220,46 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNod return TSDB_CODE_SUCCESS; } +typedef struct SCollectSpecialNodesCxt { + int32_t errCode; + ENodeType type; + SNodeList* pNodes; +} SCollectSpecialNodesCxt; + +static EDealRes collectSpecialNodes(SNode* pNode, void* pContext) { + SCollectSpecialNodesCxt* pCxt = (SCollectSpecialNodesCxt*)pContext; + if (pCxt->type == nodeType(pNode)) { + pCxt->errCode = nodesListStrictAppend(pCxt->pNodes, nodesCloneNode(pNode)); + return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); + } + return DEAL_RES_CONTINUE; +} + +int32_t nodesCollectSpecialNodes(SSelectStmt* pSelect, ESqlClause clause, ENodeType type, SNodeList** pNodes) { + if (NULL == pSelect || NULL == pNodes) { + return TSDB_CODE_FAILED; + } + + SCollectSpecialNodesCxt cxt = { + .errCode = TSDB_CODE_SUCCESS, .type = type, .pNodes = (NULL == *pNodes ? nodesMakeList() : *pNodes)}; + if (NULL == cxt.pNodes) { + return TSDB_CODE_OUT_OF_MEMORY; + } + *pNodes = NULL; + nodesWalkSelectStmt(pSelect, SQL_CLAUSE_GROUP_BY, collectSpecialNodes, &cxt); + if (TSDB_CODE_SUCCESS != cxt.errCode) { + nodesDestroyList(cxt.pNodes); + return cxt.errCode; + } + if (LIST_LENGTH(cxt.pNodes) > 0) { + *pNodes = cxt.pNodes; + } else { + nodesDestroyList(cxt.pNodes); + } + + return TSDB_CODE_SUCCESS; +} + char* getFillModeString(EFillMode mode) { switch (mode) { case FILL_MODE_NONE: diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 39cbf5de13a3d6bef5e6a9cdf433b8df29cae242..8ecfab72ce3c985eb14a54b0203807ef41f3a658 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -30,9 +30,9 @@ typedef struct SAstCreateContext { SParseContext* pQueryCxt; SMsgBuf msgBuf; bool notSupport; - bool valid; SNode* pRootNode; int16_t placeholderNo; + int32_t errCode; } SAstCreateContext; typedef enum EDatabaseOptionType { diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 9fdfe4ff7ece652ba4560fe69f7248e0a8855752..30f0d8ed8be55785118a1d41b5078acc9461b71a 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -23,13 +23,12 @@ } %syntax_error { - if (pCxt->valid) { + if (TSDB_CODE_SUCCESS == pCxt->errCode) { if(TOKEN.z) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); } else { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); } - pCxt->valid = false; } } @@ -42,8 +41,8 @@ %left NK_CONCAT. /************************************************ create/alter account *****************************************/ -cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } -cmd ::= ALTER ACCOUNT NK_ID alter_account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options. { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +cmd ::= ALTER ACCOUNT NK_ID alter_account_options. { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } %type account_options { int32_t } %destructor account_options { } @@ -323,7 +322,7 @@ cmd ::= SHOW QNODES. cmd ::= SHOW FUNCTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT, NULL, NULL); } cmd ::= SHOW INDEXES FROM table_name_cond(A) from_db_opt(B). { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, A, B); } cmd ::= SHOW STREAMS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT, NULL, NULL); } -cmd ::= SHOW ACCOUNTS. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +cmd ::= SHOW ACCOUNTS. { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } cmd ::= SHOW APPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT, NULL, NULL); } cmd ::= SHOW CONNECTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT, NULL, NULL); } cmd ::= SHOW LICENCE. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCE_STMT, NULL, NULL); } @@ -491,7 +490,7 @@ signed_literal(A) ::= NK_STRING(B). signed_literal(A) ::= NK_BOOL(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B); } signed_literal(A) ::= TIMESTAMP NK_STRING(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); } signed_literal(A) ::= duration_literal(B). { A = releaseRawExprNode(pCxt, B); } -signed_literal(A) ::= NULL. { A = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL); } +signed_literal(A) ::= NULL(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &B); } signed_literal(A) ::= literal_func(B). { A = releaseRawExprNode(pCxt, B); } %type literal_list { SNodeList* } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index c7438b0c3dd0f2bbfa6765dbd1227e9da1ccd710..82c5d17f10fa81180cb1df7da9bc5ec9e15e4f2b 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -21,7 +21,7 @@ #define CHECK_OUT_OF_MEM(p) \ do { \ if (NULL == (p)) { \ - pCxt->valid = false; \ + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; \ snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory"); \ return NULL; \ } \ @@ -30,7 +30,7 @@ #define CHECK_RAW_EXPR_NODE(node) \ do { \ if (NULL == (node) || QUERY_NODE_RAW_EXPR != nodeType(node)) { \ - pCxt->valid = false; \ + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; \ return NULL; \ } \ } while (0) @@ -42,9 +42,9 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) { pCxt->msgBuf.buf = pParseCxt->pMsg; pCxt->msgBuf.len = pParseCxt->msgLen; pCxt->notSupport = false; - pCxt->valid = true; pCxt->pRootNode = NULL; pCxt->placeholderNo = 0; + pCxt->errCode = TSDB_CODE_SUCCESS; } static void copyStringFormStringToken(SToken* pToken, char* pBuf, int32_t len) { @@ -63,42 +63,38 @@ static void trimEscape(SToken* pName) { static bool checkUserName(SAstCreateContext* pCxt, SToken* pUserName) { if (NULL == pUserName) { - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else { if (pUserName->n >= TSDB_USER_LEN) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); } } - if (pCxt->valid) { + if (TSDB_CODE_SUCCESS == pCxt->errCode) { trimEscape(pUserName); } - return pCxt->valid; + return TSDB_CODE_SUCCESS == pCxt->errCode; } static bool checkPassword(SAstCreateContext* pCxt, const SToken* pPasswordToken, char* pPassword) { if (NULL == pPasswordToken) { - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else if (pPasswordToken->n >= (TSDB_USET_PASSWORD_LEN - 2)) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); } else { strncpy(pPassword, pPasswordToken->z, pPasswordToken->n); strdequote(pPassword); if (strtrim(pPassword) <= 0) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_PASSWD_EMPTY); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_PASSWD_EMPTY); } } - return pCxt->valid; + return TSDB_CODE_SUCCESS == pCxt->errCode; } static bool checkAndSplitEndpoint(SAstCreateContext* pCxt, const SToken* pEp, char* pFqdn, int32_t* pPort) { if (NULL == pEp) { - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else if (pEp->n >= TSDB_FQDN_LEN + 2 + 6) { // format 'fqdn:port' - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); } else { char ep[TSDB_FQDN_LEN + 2 + 6]; strncpy(ep, pEp->z, pEp->n); @@ -106,66 +102,59 @@ static bool checkAndSplitEndpoint(SAstCreateContext* pCxt, const SToken* pEp, ch strtrim(ep); char* pColon = strchr(ep, ':'); if (NULL == pColon) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENDPOINT); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENDPOINT); } else { strncpy(pFqdn, ep, pColon - ep); *pPort = strtol(pColon + 1, NULL, 10); if (*pPort >= UINT16_MAX || *pPort <= 0) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); } } } - return pCxt->valid; + return TSDB_CODE_SUCCESS == pCxt->errCode; } static bool checkFqdn(SAstCreateContext* pCxt, const SToken* pFqdn) { if (NULL == pFqdn) { - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else { if (pFqdn->n >= TSDB_FQDN_LEN) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); } } - return pCxt->valid; + return TSDB_CODE_SUCCESS == pCxt->errCode; } static bool checkPort(SAstCreateContext* pCxt, const SToken* pPortToken, int32_t* pPort) { if (NULL == pPortToken) { - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else { *pPort = strtol(pPortToken->z, NULL, 10); if (*pPort >= UINT16_MAX || *pPort <= 0) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); } } - return pCxt->valid; + return TSDB_CODE_SUCCESS == pCxt->errCode; } static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool query) { if (NULL == pDbName) { if (query && NULL == pCxt->pQueryCxt->db) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DB_NOT_SPECIFIED); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DB_NOT_SPECIFIED); } } else { trimEscape(pDbName); if (pDbName->n >= TSDB_DB_NAME_LEN) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pDbName->z); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pDbName->z); } } - return pCxt->valid; + return TSDB_CODE_SUCCESS == pCxt->errCode; } static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) { trimEscape(pTableName); if (NULL != pTableName && pTableName->n >= TSDB_TABLE_NAME_LEN) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTableName->z); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTableName->z); return false; } return true; @@ -174,8 +163,7 @@ static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) { static bool checkColumnName(SAstCreateContext* pCxt, SToken* pColumnName) { trimEscape(pColumnName); if (NULL != pColumnName && pColumnName->n >= TSDB_COL_NAME_LEN) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pColumnName->z); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pColumnName->z); return false; } return true; @@ -184,8 +172,7 @@ static bool checkColumnName(SAstCreateContext* pCxt, SToken* pColumnName) { static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) { trimEscape(pIndexName); if (NULL != pIndexName && pIndexName->n >= TSDB_INDEX_NAME_LEN) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pIndexName->z); - pCxt->valid = false; + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pIndexName->z); return false; } return true; @@ -224,7 +211,7 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { if (NULL == pNode || QUERY_NODE_RAW_EXPR != nodeType(pNode)) { - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; return nil_token; } SRawExprNode* target = (SRawExprNode*)pNode; @@ -235,16 +222,12 @@ SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode) { SNodeList* list = nodesMakeList(); CHECK_OUT_OF_MEM(list); - if (TSDB_CODE_SUCCESS != nodesListAppend(list, pNode)) { - pCxt->valid = false; - } + pCxt->errCode = nodesListAppend(list, pNode); return list; } SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode) { - if (TSDB_CODE_SUCCESS != nodesListAppend(pList, pNode)) { - pCxt->valid = false; - } + pCxt->errCode = nodesListAppend(pList, pNode); return pList; } @@ -380,11 +363,9 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d CHECK_OUT_OF_MEM(func); strcpy(func->functionName, "cast"); func->node.resType = dt; - if (TSDB_DATA_TYPE_BINARY == dt.type) { - func->node.resType.bytes += 2; - } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { - func->node.resType.bytes = func->node.resType.bytes * TSDB_NCHAR_SIZE + 2; - } + if (TSDB_DATA_TYPE_NCHAR == dt.type) { + func->node.resType.bytes = func->node.resType.bytes * TSDB_NCHAR_SIZE; + } nodesListMakeAppend(&func->pParameterList, pExpr); return (SNode*)func; } @@ -518,6 +499,12 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { CHECK_OUT_OF_MEM(fill); fill->mode = mode; fill->pValues = pValues; + fill->pWStartTs = nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == fill->pWStartTs) { + nodesDestroyNode(fill); + CHECK_OUT_OF_MEM(fill->pWStartTs); + } + strcpy(((SFunctionNode*)fill->pWStartTs)->functionName, "_wstartts"); return (SNode*)fill; } @@ -531,7 +518,7 @@ SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode) { } SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) { - if (NULL == pNode || !pCxt->valid) { + if (NULL == pNode || TSDB_CODE_SUCCESS != pCxt->errCode) { return pNode; } int32_t len = TMIN(sizeof(((SExprNode*)pNode)->aliasName) - 1, pAlias->n); @@ -603,6 +590,7 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr select->pProjectionList = pProjectionList; select->pFromTable = pTable; sprintf(select->stmtName, "%p", select); + select->isTimeOrderQuery = true; return (SNode*)select; } @@ -995,7 +983,7 @@ static bool needDbShowStmt(ENodeType type) { SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbNamePattern) { if (needDbShowStmt(type) && NULL == pDbName && NULL == pCxt->pQueryCxt->db) { snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "db not specified"); - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; return NULL; } SShowStmt* pStmt = nodesMakeNode(type); @@ -1256,7 +1244,7 @@ SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups) { SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize) { if (pLibPath->n <= 2) { - pCxt->valid = false; + pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; return NULL; } SCreateFunctionStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_FUNCTION_STMT); diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 270cd30231e6a83cc4424157c0a03be8dbd55fb7..ee1a92d8b35248cd315a432dc70f10f21f936f17 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -53,20 +53,20 @@ int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) { } case TK_NK_ILLEGAL: { snprintf(cxt.pQueryCxt->pMsg, cxt.pQueryCxt->msgLen, "unrecognized token: \"%s\"", t0.z); - cxt.valid = false; + cxt.errCode = TSDB_CODE_PAR_SYNTAX_ERROR; goto abort_parse; } case TK_NK_HEX: case TK_NK_OCT: case TK_NK_BIN: { snprintf(cxt.pQueryCxt->pMsg, cxt.pQueryCxt->msgLen, "unsupported token: \"%s\"", t0.z); - cxt.valid = false; + cxt.errCode = TSDB_CODE_PAR_SYNTAX_ERROR; goto abort_parse; } default: Parse(pParser, t0.type, t0, &cxt); // ParseTrace(stdout, ""); - if (!cxt.valid) { + if (TSDB_CODE_SUCCESS != cxt.errCode) { goto abort_parse; } } @@ -74,7 +74,7 @@ int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) { abort_parse: ParseFree(pParser, (FFree)taosMemoryFree); - if (cxt.valid) { + if (TSDB_CODE_SUCCESS == cxt.errCode) { *pQuery = taosMemoryCalloc(1, sizeof(SQuery)); if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; @@ -82,5 +82,5 @@ abort_parse: (*pQuery)->pRoot = cxt.pRootNode; (*pQuery)->placeholderNum = cxt.placeholderNo; } - return cxt.valid ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; + return cxt.errCode; } diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index d0e670df6e71cffa8136040fd4e9af81d0e45bc9..9c2bd106863af1d2ac21a2d26aac60fd150f69e2 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -208,6 +208,23 @@ static int32_t calcConstProjections(SCalcConstContext* pCxt, SNodeList* pProject return TSDB_CODE_SUCCESS; } +static int32_t calcConstGroupBy(SCalcConstContext* pCxt, SSelectStmt* pSelect) { + int32_t code = calcConstList(pSelect->pGroupByList); + if (TSDB_CODE_SUCCESS == code) { + SNode* pNode = NULL; + FOREACH(pNode, pSelect->pGroupByList) { + SNode* pGroupPara = NULL; + FOREACH(pGroupPara, ((SGroupingSetNode*)pNode)->pParameterList) { + if (QUERY_NODE_VALUE != nodeType(pGroupPara)) { + return code; + } + } + } + DESTORY_LIST(pSelect->pGroupByList); + } + return code; +} + static int32_t calcConstSelect(SCalcConstContext* pCxt, SSelectStmt* pSelect, bool subquery) { int32_t code = calcConstProjections(pCxt, pSelect->pProjectionList, subquery); if (TSDB_CODE_SUCCESS == code) { @@ -223,7 +240,7 @@ static int32_t calcConstSelect(SCalcConstContext* pCxt, SSelectStmt* pSelect, bo code = calcConstNode(&pSelect->pWindow); } if (TSDB_CODE_SUCCESS == code) { - code = calcConstList(pSelect->pGroupByList); + code = calcConstGroupBy(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { code = calcConstSelectCondition(pCxt, pSelect, &pSelect->pHaving); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index f452838921d629bdbb86dd600adbf52c3ae7d0a6..e82873b92305dee0c023d971c4fa1561e9a05696 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -22,7 +22,6 @@ #include "ttime.h" #include "ttypes.h" -// clang-format off #define NEXT_TOKEN(pSql, sToken) \ do { \ int32_t index = 0; \ @@ -248,12 +247,11 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SToken* pTname, bool } else { CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); + SVgroupInfo vg; + CHECK_CODE( + catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); + CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); } - SVgroupInfo vg; - CHECK_CODE( - catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); - CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); - return TSDB_CODE_SUCCESS; } @@ -828,12 +826,21 @@ static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } -static int32_t storeTableMeta(SHashObj* pHash, const char* pName, int32_t len, STableMeta* pMeta) { +static int32_t storeTableMeta(SInsertParseContext* pCxt, SHashObj* pHash, SName* pTableName, const char* pName, + int32_t len, STableMeta* pMeta) { + SVgroupInfo vg; + SParseContext* pBasicCtx = pCxt->pComCxt; + CHECK_CODE( + catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pTableName, &vg)); + CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); + + pMeta->uid = tGenIdPI64(); + pMeta->vgId = vg.vgId; + STableMeta* pBackup = NULL; if (TSDB_CODE_SUCCESS != cloneTableMeta(pMeta, &pBackup)) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pBackup->uid = tGenIdPI64(); return taosHashPut(pHash, pName, len, &pBackup, POINTER_BYTES); } @@ -856,7 +863,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); } - CHECK_CODE(storeTableMeta(pCxt->pSubTableHashObj, tbFName, len, pCxt->pTableMeta)); + CHECK_CODE(storeTableMeta(pCxt, pCxt->pSubTableHashObj, &name, tbFName, len, pCxt->pTableMeta)); SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); setBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); @@ -1043,6 +1050,7 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { static void destroyInsertParseContext(SInsertParseContext* pCxt) { destroyInsertParseContextForTable(pCxt); taosHashCleanup(pCxt->pVgroupsHashObj); + taosHashCleanup(pCxt->pSubTableHashObj); destroyBlockHashmap(pCxt->pTableBlockHashObj); destroyBlockArrayList(pCxt->pVgDataBlocks); @@ -1261,9 +1269,10 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash return TSDB_CODE_SUCCESS; } -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen){ - STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; +int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_MULTI_BIND* bind, + char* msgBuf, int32_t msgBufLen) { + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_QRY_APP_ERROR; @@ -1311,11 +1320,10 @@ int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *p return TSDB_CODE_SUCCESS; } - -int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen) { - STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); +int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlock->rowBuilder; SMemParam param = {.rb = pBuilder}; @@ -1390,10 +1398,11 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, in return TSDB_CODE_SUCCESS; } -int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum) { - STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); +int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, + int32_t rowNum) { + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlock->rowBuilder; SMemParam param = {.rb = pBuilder}; @@ -1458,7 +1467,7 @@ int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBu } #ifdef TD_DEBUG_PRINT_ROW - if(rowEnd) { + if (rowEnd) { STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&pSchema, spd->numOfCols); tdSRowPrint(row, pSTSchema, __func__); taosMemoryFree(pSTSchema); diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index ae05d2293eb7f7d9d9cdb1f38950eae3a5455824..4a96301d95cce6041b0c5785c0d6fe2f8fcfedd2 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -498,14 +498,9 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); } - int32_t len = pBlocks->numOfRows * - (isRawPayload ? (pOneTableBlock->rowSize + expandSize) : getExtendedRowSize(pOneTableBlock)) + - sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); - // erase the empty space reserved for binary data int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, isRawPayload); - assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); assert(dataBuf->size <= dataBuf->nAllocSize); diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index abd3af8e1214bb29ecefdaeac138a930ee0c5a90..44d68a78865fc482ab3dc544d4fdba05f57376fe 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -26,190 +26,192 @@ typedef struct SKeyword { uint8_t len; // length } SKeyword; +// clang-format off // keywords in sql string static SKeyword keywordTable[] = { - {"ACCOUNT", TK_ACCOUNT}, - {"ACCOUNTS", TK_ACCOUNTS}, - {"ADD", TK_ADD}, - {"AGGREGATE", TK_AGGREGATE}, - {"ALL", TK_ALL}, - {"ALTER", TK_ALTER}, - {"ANALYZE", TK_ANALYZE}, - {"AND", TK_AND}, - {"APPS", TK_APPS}, - {"AS", TK_AS}, - {"ASC", TK_ASC}, - {"AT_ONCE", TK_AT_ONCE}, - {"BETWEEN", TK_BETWEEN}, - {"BINARY", TK_BINARY}, - {"BIGINT", TK_BIGINT}, - // {"BLOCKS", TK_BLOCKS}, - {"BNODE", TK_BNODE}, - {"BNODES", TK_BNODES}, - {"BOOL", TK_BOOL}, - {"BUFSIZE", TK_BUFSIZE}, - {"BY", TK_BY}, - {"CACHE", TK_CACHE}, - {"CACHELAST", TK_CACHELAST}, - {"CAST", TK_CAST}, - {"CLUSTER", TK_CLUSTER}, - {"COLUMN", TK_COLUMN}, - {"COMMENT", TK_COMMENT}, - {"COMP", TK_COMP}, - {"COMPACT", TK_COMPACT}, - {"CONNS", TK_CONNS}, - {"CONNECTION", TK_CONNECTION}, - {"CONNECTIONS", TK_CONNECTIONS}, - {"COUNT", TK_COUNT}, - {"CREATE", TK_CREATE}, - {"DATABASE", TK_DATABASE}, - {"DATABASES", TK_DATABASES}, - {"DAYS", TK_DAYS}, - {"DBS", TK_DBS}, - {"DELAY", TK_DELAY}, - {"DESC", TK_DESC}, - {"DESCRIBE", TK_DESCRIBE}, - {"DISTINCT", TK_DISTINCT}, - {"DNODE", TK_DNODE}, - {"DNODES", TK_DNODES}, - {"DOUBLE", TK_DOUBLE}, - {"DROP", TK_DROP}, - {"EXISTS", TK_EXISTS}, - {"EXPLAIN", TK_EXPLAIN}, - {"FILE_FACTOR", TK_FILE_FACTOR}, - {"FILL", TK_FILL}, - {"FIRST", TK_FIRST}, - {"FLOAT", TK_FLOAT}, - {"FROM", TK_FROM}, - {"FSYNC", TK_FSYNC}, - {"FUNCTION", TK_FUNCTION}, - {"FUNCTIONS", TK_FUNCTIONS}, - {"GRANTS", TK_GRANTS}, - {"GROUP", TK_GROUP}, - {"HAVING", TK_HAVING}, - {"IF", TK_IF}, - {"IMPORT", TK_IMPORT}, - {"IN", TK_IN}, - {"INDEX", TK_INDEX}, - {"INDEXES", TK_INDEXES}, - {"INNER", TK_INNER}, - {"INT", TK_INT}, - {"INSERT", TK_INSERT}, - {"INTEGER", TK_INTEGER}, - {"INTERVAL", TK_INTERVAL}, - {"INTO", TK_INTO}, - {"IS", TK_IS}, - {"JOIN", TK_JOIN}, - {"JSON", TK_JSON}, - {"KEEP", TK_KEEP}, - {"KILL", TK_KILL}, - {"LAST", TK_LAST}, - {"LAST_ROW", TK_LAST_ROW}, - {"LICENCE", TK_LICENCE}, - {"LIKE", TK_LIKE}, - {"LIMIT", TK_LIMIT}, - {"LINEAR", TK_LINEAR}, - {"LOCAL", TK_LOCAL}, - {"MATCH", TK_MATCH}, - {"MAXROWS", TK_MAXROWS}, - {"MINROWS", TK_MINROWS}, - {"MINUS", TK_MINUS}, - {"MNODE", TK_MNODE}, - {"MNODES", TK_MNODES}, - {"MODIFY", TK_MODIFY}, - {"MODULES", TK_MODULES}, - {"NCHAR", TK_NCHAR}, - {"NMATCH", TK_NMATCH}, - {"NONE", TK_NONE}, - {"NOT", TK_NOT}, - {"NOW", TK_NOW}, - {"NULL", TK_NULL}, - {"NULLS", TK_NULLS}, - {"OFFSET", TK_OFFSET}, - {"ON", TK_ON}, - {"OR", TK_OR}, - {"ORDER", TK_ORDER}, - {"OUTPUTTYPE", TK_OUTPUTTYPE}, - {"PARTITION", TK_PARTITION}, - {"PASS", TK_PASS}, - {"PORT", TK_PORT}, - {"PPS", TK_PPS}, - {"PRECISION", TK_PRECISION}, - {"PRIVILEGE", TK_PRIVILEGE}, - {"PREV", TK_PREV}, - {"QNODE", TK_QNODE}, - {"QNODES", TK_QNODES}, - {"QTIME", TK_QTIME}, - {"QUERIES", TK_QUERIES}, - {"QUERY", TK_QUERY}, - // {"QUORUM", TK_QUORUM}, - {"RATIO", TK_RATIO}, - {"REPLICA", TK_REPLICA}, - {"RESET", TK_RESET}, - {"RETENTIONS", TK_RETENTIONS}, - {"ROLLUP", TK_ROLLUP}, - {"SCHEMA", TK_SCHEMA}, - {"SCORES", TK_SCORES}, - {"SELECT", TK_SELECT}, - {"SESSION", TK_SESSION}, - {"SET", TK_SET}, - {"SHOW", TK_SHOW}, + {"ACCOUNT", TK_ACCOUNT}, + {"ACCOUNTS", TK_ACCOUNTS}, + {"ADD", TK_ADD}, + {"AGGREGATE", TK_AGGREGATE}, + {"ALL", TK_ALL}, + {"ALTER", TK_ALTER}, + {"ANALYZE", TK_ANALYZE}, + {"AND", TK_AND}, + {"APPS", TK_APPS}, + {"AS", TK_AS}, + {"ASC", TK_ASC}, + {"AT_ONCE", TK_AT_ONCE}, + {"BETWEEN", TK_BETWEEN}, + {"BINARY", TK_BINARY}, + {"BIGINT", TK_BIGINT}, + {"BNODE", TK_BNODE}, + {"BNODES", TK_BNODES}, + {"BOOL", TK_BOOL}, + {"BUFFER", TK_BUFFER}, + {"BUFSIZE", TK_BUFSIZE}, + {"BY", TK_BY}, + {"CACHE", TK_CACHE}, + {"CACHELAST", TK_CACHELAST}, + {"CAST", TK_CAST}, + {"CLUSTER", TK_CLUSTER}, + {"COLUMN", TK_COLUMN}, + {"COMMENT", TK_COMMENT}, + {"COMP", TK_COMP}, + {"COMPACT", TK_COMPACT}, + {"CONNS", TK_CONNS}, + {"CONNECTION", TK_CONNECTION}, + {"CONNECTIONS", TK_CONNECTIONS}, + {"COUNT", TK_COUNT}, + {"CREATE", TK_CREATE}, + {"DATABASE", TK_DATABASE}, + {"DATABASES", TK_DATABASES}, + {"DAYS", TK_DAYS}, + {"DBS", TK_DBS}, + {"DELAY", TK_DELAY}, + {"DESC", TK_DESC}, + {"DESCRIBE", TK_DESCRIBE}, + {"DISTINCT", TK_DISTINCT}, + {"DNODE", TK_DNODE}, + {"DNODES", TK_DNODES}, + {"DOUBLE", TK_DOUBLE}, + {"DROP", TK_DROP}, + {"EXISTS", TK_EXISTS}, + {"EXPLAIN", TK_EXPLAIN}, + {"FILE_FACTOR", TK_FILE_FACTOR}, + {"FILL", TK_FILL}, + {"FIRST", TK_FIRST}, + {"FLOAT", TK_FLOAT}, + {"FROM", TK_FROM}, + {"FSYNC", TK_FSYNC}, + {"FUNCTION", TK_FUNCTION}, + {"FUNCTIONS", TK_FUNCTIONS}, + {"GRANTS", TK_GRANTS}, + {"GROUP", TK_GROUP}, + {"HAVING", TK_HAVING}, + {"IF", TK_IF}, + {"IMPORT", TK_IMPORT}, + {"IN", TK_IN}, + {"INDEX", TK_INDEX}, + {"INDEXES", TK_INDEXES}, + {"INNER", TK_INNER}, + {"INT", TK_INT}, + {"INSERT", TK_INSERT}, + {"INTEGER", TK_INTEGER}, + {"INTERVAL", TK_INTERVAL}, + {"INTO", TK_INTO}, + {"IS", TK_IS}, + {"JOIN", TK_JOIN}, + {"JSON", TK_JSON}, + {"KEEP", TK_KEEP}, + {"KILL", TK_KILL}, + {"LAST", TK_LAST}, + {"LAST_ROW", TK_LAST_ROW}, + {"LICENCE", TK_LICENCE}, + {"LIKE", TK_LIKE}, + {"LIMIT", TK_LIMIT}, + {"LINEAR", TK_LINEAR}, + {"LOCAL", TK_LOCAL}, + {"MATCH", TK_MATCH}, + {"MAXROWS", TK_MAXROWS}, + {"MINROWS", TK_MINROWS}, + {"MINUS", TK_MINUS}, + {"MNODE", TK_MNODE}, + {"MNODES", TK_MNODES}, + {"MODIFY", TK_MODIFY}, + {"MODULES", TK_MODULES}, + {"NCHAR", TK_NCHAR}, + {"NEXT", TK_NEXT}, + {"NMATCH", TK_NMATCH}, + {"NONE", TK_NONE}, + {"NOT", TK_NOT}, + {"NOW", TK_NOW}, + {"NULL", TK_NULL}, + {"NULLS", TK_NULLS}, + {"OFFSET", TK_OFFSET}, + {"ON", TK_ON}, + {"OR", TK_OR}, + {"ORDER", TK_ORDER}, + {"OUTPUTTYPE", TK_OUTPUTTYPE}, + {"PARTITION", TK_PARTITION}, + {"PASS", TK_PASS}, + {"PAGES", TK_PAGES}, + {"PAGESIZE", TK_PAGESIZE}, + {"PORT", TK_PORT}, + {"PPS", TK_PPS}, + {"PRECISION", TK_PRECISION}, + {"PRIVILEGE", TK_PRIVILEGE}, + {"PREV", TK_PREV}, + {"QNODE", TK_QNODE}, + {"QNODES", TK_QNODES}, + {"QTIME", TK_QTIME}, + {"QUERIES", TK_QUERIES}, + {"QUERY", TK_QUERY}, + {"RATIO", TK_RATIO}, + {"REPLICA", TK_REPLICA}, + {"RESET", TK_RESET}, + {"RETENTIONS", TK_RETENTIONS}, + {"ROLLUP", TK_ROLLUP}, + {"SCHEMA", TK_SCHEMA}, + {"SCORES", TK_SCORES}, + {"SELECT", TK_SELECT}, + {"SESSION", TK_SESSION}, + {"SET", TK_SET}, + {"SHOW", TK_SHOW}, {"SINGLE_STABLE", TK_SINGLE_STABLE}, - {"SLIDING", TK_SLIDING}, - {"SLIMIT", TK_SLIMIT}, - {"SMA", TK_SMA}, - {"SMALLINT", TK_SMALLINT}, - {"SNODE", TK_SNODE}, - {"SNODES", TK_SNODES}, - {"SOFFSET", TK_SOFFSET}, - {"STABLE", TK_STABLE}, - {"STABLES", TK_STABLES}, - {"STATE", TK_STATE}, - {"STATE_WINDOW", TK_STATE_WINDOW}, - {"STORAGE", TK_STORAGE}, - {"STREAM", TK_STREAM}, - {"STREAMS", TK_STREAMS}, - // {"STREAM_MODE", TK_STREAM_MODE}, - {"STRICT", TK_STRICT}, - {"SYNCDB", TK_SYNCDB}, - {"TABLE", TK_TABLE}, - {"TABLES", TK_TABLES}, - {"TAG", TK_TAG}, - {"TAGS", TK_TAGS}, - {"TBNAME", TK_TBNAME}, - {"TIMESTAMP", TK_TIMESTAMP}, - {"TIMEZONE", TK_TIMEZONE}, - {"TINYINT", TK_TINYINT}, - {"TODAY", TK_TODAY}, - {"TOPIC", TK_TOPIC}, - {"TOPICS", TK_TOPICS}, - {"TRIGGER", TK_TRIGGER}, - {"TSERIES", TK_TSERIES}, - {"TTL", TK_TTL}, - {"UNION", TK_UNION}, - {"UNSIGNED", TK_UNSIGNED}, - {"USE", TK_USE}, - {"USER", TK_USER}, - {"USERS", TK_USERS}, - {"USING", TK_USING}, - {"VALUE", TK_VALUE}, - {"VALUES", TK_VALUES}, - {"VARCHAR", TK_VARCHAR}, - {"VARIABLES", TK_VARIABLES}, - {"VERBOSE", TK_VERBOSE}, - {"VGROUPS", TK_VGROUPS}, - {"VNODES", TK_VNODES}, - {"WAL", TK_WAL}, - {"WATERMARK", TK_WATERMARK}, - {"WHERE", TK_WHERE}, - {"WINDOW_CLOSE", TK_WINDOW_CLOSE}, - {"WITH", TK_WITH}, - {"_QENDTS", TK_QENDTS}, - {"_QSTARTTS", TK_QSTARTTS}, - {"_ROWTS", TK_ROWTS}, - {"_WDURATION", TK_WDURATION}, - {"_WENDTS", TK_WENDTS}, - {"_WSTARTTS", TK_WSTARTTS}, + {"SLIDING", TK_SLIDING}, + {"SLIMIT", TK_SLIMIT}, + {"SMA", TK_SMA}, + {"SMALLINT", TK_SMALLINT}, + {"SNODE", TK_SNODE}, + {"SNODES", TK_SNODES}, + {"SOFFSET", TK_SOFFSET}, + {"STABLE", TK_STABLE}, + {"STABLES", TK_STABLES}, + {"STATE", TK_STATE}, + {"STATE_WINDOW", TK_STATE_WINDOW}, + {"STORAGE", TK_STORAGE}, + {"STREAM", TK_STREAM}, + {"STREAMS", TK_STREAMS}, + {"STRICT", TK_STRICT}, + {"SYNCDB", TK_SYNCDB}, + {"TABLE", TK_TABLE}, + {"TABLES", TK_TABLES}, + {"TAG", TK_TAG}, + {"TAGS", TK_TAGS}, + {"TBNAME", TK_TBNAME}, + {"TIMESTAMP", TK_TIMESTAMP}, + {"TIMEZONE", TK_TIMEZONE}, + {"TINYINT", TK_TINYINT}, + {"TODAY", TK_TODAY}, + {"TOPIC", TK_TOPIC}, + {"TOPICS", TK_TOPICS}, + {"TRIGGER", TK_TRIGGER}, + {"TSERIES", TK_TSERIES}, + {"TTL", TK_TTL}, + {"UNION", TK_UNION}, + {"UNSIGNED", TK_UNSIGNED}, + {"USE", TK_USE}, + {"USER", TK_USER}, + {"USERS", TK_USERS}, + {"USING", TK_USING}, + {"VALUE", TK_VALUE}, + {"VALUES", TK_VALUES}, + {"VARCHAR", TK_VARCHAR}, + {"VARIABLES", TK_VARIABLES}, + {"VERBOSE", TK_VERBOSE}, + {"VGROUPS", TK_VGROUPS}, + {"VNODES", TK_VNODES}, + {"WAL", TK_WAL}, + {"WATERMARK", TK_WATERMARK}, + {"WHERE", TK_WHERE}, + {"WINDOW_CLOSE", TK_WINDOW_CLOSE}, + {"WITH", TK_WITH}, + {"_QENDTS", TK_QENDTS}, + {"_QSTARTTS", TK_QSTARTTS}, + {"_ROWTS", TK_ROWTS}, + {"_WDURATION", TK_WDURATION}, + {"_WENDTS", TK_WENDTS}, + {"_WSTARTTS", TK_WSTARTTS}, // {"ID", TK_ID}, // {"STRING", TK_STRING}, // {"EQ", TK_EQ}, @@ -276,6 +278,7 @@ static SKeyword keywordTable[] = { // {"PARTITIONS", TK_PARTITIONS}, // {"MODE", TK_MODE}, }; +// clang-format on static const char isIdChar[] = { /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5f7dc5de9015a2b9aad7a54688afc04e38e0bcda..6874b5b7d40cb879e940ef7eb7465c267111a60b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -17,14 +17,16 @@ #include "catalog.h" #include "cmdnodes.h" +#include "filter.h" #include "functionMgt.h" #include "parUtil.h" #include "scalar.h" +#include "systable.h" #include "tglobal.h" #include "ttime.h" #define generateDealNodeErrMsg(pCxt, code, ...) \ - (pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__) ? DEAL_RES_ERROR : DEAL_RES_ERROR) + (pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__), DEAL_RES_ERROR) typedef struct STranslateContext { SParseContext* pParseCxt; @@ -254,6 +256,26 @@ static void destroyTranslateContext(STranslateContext* pCxt) { taosHashCleanup(pCxt->pTables); } +static bool isAliasColumn(const SNode* pNode) { + return (QUERY_NODE_COLUMN == nodeType(pNode) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0])); +} + +static bool isAggFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)); +} + +static bool isSelectFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsSelectFunc(((SFunctionNode*)pNode)->funcId)); +} + +static bool isTimelineFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsTimelineFunc(((SFunctionNode*)pNode)->funcId)); +} + +static bool isDistinctOrderBy(STranslateContext* pCxt) { + return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct); +} + static bool belongTable(const char* currentDb, const SColumnNode* pCol, const STableNode* pTable) { int cmp = 0; if ('\0' != pCol->dbName[0]) { @@ -318,7 +340,7 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol pCol->node.resType = pExpr->resType; } -static int32_t createColumnNodeByTable(STranslateContext* pCxt, const STableNode* pTable, SNodeList* pList) { +static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, SNodeList* pList) { if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; int32_t nums = @@ -465,27 +487,66 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { break; case TSDB_DATA_TYPE_BOOL: pVal->datum.b = (0 == strcasecmp(pVal->literal, "true")); + *(bool*)&pVal->typeData = pVal->datum.b; + break; + case TSDB_DATA_TYPE_TINYINT: { + char* endPtr = NULL; + pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + *(int8_t*)&pVal->typeData = pVal->datum.i; break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: + } + case TSDB_DATA_TYPE_SMALLINT: { + char* endPtr = NULL; + pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + *(int16_t*)&pVal->typeData = pVal->datum.i; + break; + } + case TSDB_DATA_TYPE_INT: { + char* endPtr = NULL; + pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + *(int32_t*)&pVal->typeData = pVal->datum.i; + break; + } case TSDB_DATA_TYPE_BIGINT: { char* endPtr = NULL; pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + *(int64_t*)&pVal->typeData = pVal->datum.i; + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + char* endPtr = NULL; + pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + *(uint8_t*)&pVal->typeData = pVal->datum.u; + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + char* endPtr = NULL; + pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + *(uint16_t*)&pVal->typeData = pVal->datum.u; + break; + } + case TSDB_DATA_TYPE_UINT: { + char* endPtr = NULL; + pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + *(uint32_t*)&pVal->typeData = pVal->datum.u; break; } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: { char* endPtr = NULL; pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + *(uint64_t*)&pVal->typeData = pVal->datum.u; + break; + } + case TSDB_DATA_TYPE_FLOAT: { + char* endPtr = NULL; + pVal->datum.d = strtold(pVal->literal, &endPtr); + *(float*)&pVal->typeData = pVal->datum.d; break; } - case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: { char* endPtr = NULL; pVal->datum.d = strtold(pVal->literal, &endPtr); + *(double*)&pVal->typeData = pVal->datum.d; break; } case TSDB_DATA_TYPE_VARCHAR: @@ -503,6 +564,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } + *(int64_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_NCHAR: @@ -575,13 +637,45 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { } static EDealRes haveAggFunction(SNode* pNode, void* pContext) { - if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)) { + if (isAggFunc(pNode)) { *((bool*)pContext) = true; return DEAL_RES_END; } return DEAL_RES_CONTINUE; } +static int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput) { + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t nums = taosArrayGetSize(pTables); + for (size_t i = 0; i < nums; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + if (NULL == pTableAlias || 0 == strcmp(pTable->tableAlias, pTableAlias)) { + *pOutput = pTable; + return TSDB_CODE_SUCCESS; + } + } + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pTableAlias); +} + +static bool isCountStar(SFunctionNode* pFunc) { + if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) { + return false; + } + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + return (QUERY_NODE_COLUMN == nodeType(pPara) && 0 == strcmp(((SColumnNode*)pPara)->colName, "*")); +} + +// count(*) is rewritten as count(ts) for scannning optimization +static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) { + SColumnNode* pCol = nodesListGetNode(pCount->pParameterList, 0); + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable); + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); + } + return code; +} + static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, @@ -589,10 +683,7 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) .pErrBuf = pCxt->msgBuf.buf, .errBufLen = pCxt->msgBuf.len}; pCxt->errCode = fmGetFuncInfo(¶m, pFunc); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - return DEAL_RES_ERROR; - } - if (fmIsAggFunc(pFunc->funcId)) { + if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsAggFunc(pFunc->funcId)) { if (beforeHaving(pCxt->currClause)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION); } @@ -601,10 +692,14 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) if (haveAggFunc) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_AGG_FUNC_NESTING); } + pCxt->pCurrStmt->hasAggFuncs = true; + pCxt->pCurrStmt->isTimeOrderQuery = false; + if (isCountStar(pFunc)) { + pCxt->errCode = rewriteCountStar(pCxt, pFunc); + } } - - return DEAL_RES_CONTINUE; + return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } static EDealRes translateExprSubquery(STranslateContext* pCxt, SNode* pNode) { @@ -648,12 +743,6 @@ static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) { return pCxt->errCode; } -static bool isAliasColumn(SColumnNode* pCol) { return ('\0' == pCol->tableAlias[0]); } - -static bool isDistinctOrderBy(STranslateContext* pCxt) { - return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct); -} - static SNodeList* getGroupByList(STranslateContext* pCxt) { if (isDistinctOrderBy(pCxt)) { return pCxt->pCurrStmt->pProjectionList; @@ -675,31 +764,71 @@ static int32_t getGroupByErrorCode(STranslateContext* pCxt) { return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION; } -static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) { - STranslateContext* pCxt = (STranslateContext*)pContext; - if (!nodesIsExprNode(pNode) || (QUERY_NODE_COLUMN == nodeType(pNode) && isAliasColumn((SColumnNode*)pNode))) { +typedef struct SCheckExprForGroupByCxt { + STranslateContext* pTranslateCxt; + int32_t selectFuncNum; + bool hasSelectValFunc; +} SCheckExprForGroupByCxt; + +static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, bool* pHasSelectValFunc, SNode** pNode) { + SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + strcpy(pFunc->functionName, "_select_value"); + pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + translateFunction(pCxt, pFunc); + } + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + *pNode = (SNode*)pFunc; + if (NULL != pHasSelectValFunc) { + *pHasSelectValFunc = true; + } + } else { + nodesDestroyNode(pFunc); + } + return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR; +} + +static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { + SCheckExprForGroupByCxt* pCxt = (SCheckExprForGroupByCxt*)pContext; + if (!nodesIsExprNode(*pNode) || isAliasColumn(*pNode)) { return DEAL_RES_CONTINUE; } - if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId) && - !isDistinctOrderBy(pCxt)) { + pCxt->selectFuncNum += isSelectFunc(*pNode) ? 1 : 0; + if (pCxt->selectFuncNum > 1 && pCxt->hasSelectValFunc) { + return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); + } + if (isAggFunc(*pNode) && !isDistinctOrderBy(pCxt->pTranslateCxt)) { return DEAL_RES_IGNORE_CHILD; } SNode* pGroupNode; - FOREACH(pGroupNode, getGroupByList(pCxt)) { - if (nodesEqualNode(getGroupByNode(pGroupNode), pNode)) { + FOREACH(pGroupNode, getGroupByList(pCxt->pTranslateCxt)) { + if (nodesEqualNode(getGroupByNode(pGroupNode), *pNode)) { return DEAL_RES_IGNORE_CHILD; } } - if (QUERY_NODE_COLUMN == nodeType(pNode) || - (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId) && - isDistinctOrderBy(pCxt))) { - return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt)); + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + if (pCxt->selectFuncNum > 1) { + return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); + } else { + return rewriteColToSelectValFunc(pCxt->pTranslateCxt, &pCxt->hasSelectValFunc, pNode); + } + } + if (isAggFunc(*pNode) && isDistinctOrderBy(pCxt->pTranslateCxt)) { + return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); } return DEAL_RES_CONTINUE; } -static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode* pNode) { - nodesWalkExpr(pNode, doCheckExprForGroupBy, pCxt); +static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode** pNode) { + SCheckExprForGroupByCxt cxt = {.pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false}; + nodesRewriteExpr(pNode, doCheckExprForGroupBy, &cxt); + if (cxt.selectFuncNum != 1 && cxt.hasSelectValFunc) { + return generateSyntaxErrMsg(&pCxt->msgBuf, getGroupByErrorCode(pCxt)); + } return pCxt->errCode; } @@ -707,7 +836,29 @@ static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList if (NULL == getGroupByList(pCxt)) { return TSDB_CODE_SUCCESS; } - nodesWalkExprs(pList, doCheckExprForGroupBy, pCxt); + SCheckExprForGroupByCxt cxt = {.pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false}; + nodesRewriteExprs(pList, doCheckExprForGroupBy, &cxt); + if (cxt.selectFuncNum != 1 && cxt.hasSelectValFunc) { + return generateSyntaxErrMsg(&pCxt->msgBuf, getGroupByErrorCode(pCxt)); + } + return pCxt->errCode; +} + +static EDealRes rewriteColsToSelectValFuncImpl(SNode** pNode, void* pContext) { + if (isAggFunc(*pNode)) { + return DEAL_RES_IGNORE_CHILD; + } + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + return rewriteColToSelectValFunc((STranslateContext*)pContext, NULL, pNode); + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteColsToSelectValFunc(STranslateContext* pCxt, SSelectStmt* pSelect) { + nodesRewriteExprs(pSelect->pProjectionList, rewriteColsToSelectValFuncImpl, pCxt); + if (TSDB_CODE_SUCCESS == pCxt->errCode && !pSelect->isDistinct) { + nodesRewriteExprs(pSelect->pOrderByList, rewriteColsToSelectValFuncImpl, pCxt); + } return pCxt->errCode; } @@ -715,11 +866,13 @@ typedef struct CheckAggColCoexistCxt { STranslateContext* pTranslateCxt; bool existAggFunc; bool existCol; + int32_t selectFuncNum; } CheckAggColCoexistCxt; static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; - if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)) { + pCxt->selectFuncNum += isSelectFunc(pNode) ? 1 : 0; + if (isAggFunc(pNode)) { pCxt->existAggFunc = true; return DEAL_RES_IGNORE_CHILD; } @@ -738,7 +891,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) if (!pSelect->isDistinct) { nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); } - if ((cxt.existAggFunc || NULL != pSelect->pWindow) && cxt.existCol) { + if (1 == cxt.selectFuncNum) { + return rewriteColsToSelectValFunc(pCxt, pSelect); + } else if ((cxt.selectFuncNum > 1 || cxt.existAggFunc || NULL != pSelect->pWindow) && cxt.existCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SINGLE_GROUP); } return TSDB_CODE_SUCCESS; @@ -780,17 +935,17 @@ static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRea SArray* vgroupList = NULL; if ('\0' != pRealTable->qualDbName[0]) { // todo release after mnode can be processed - // if (0 != strcmp(pRealTable->qualDbName, TSDB_INFORMATION_SCHEMA_DB)) { - code = getDBVgInfo(pCxt, pRealTable->qualDbName, &vgroupList); - // } + if (0 != strcmp(pRealTable->qualDbName, TSDB_INFORMATION_SCHEMA_DB)) { + code = getDBVgInfo(pCxt, pRealTable->qualDbName, &vgroupList); + } } else { code = getDBVgInfoImpl(pCxt, pName, &vgroupList); } // todo release after mnode can be processed - // if (TSDB_CODE_SUCCESS == code) { - // code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList); - // } + if (TSDB_CODE_SUCCESS == code) { + code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList); + } if (TSDB_CODE_SUCCESS == code) { code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList); @@ -898,7 +1053,7 @@ static int32_t createAllColumns(STranslateContext* pCxt, SNodeList** pCols) { size_t nums = taosArrayGetSize(pTables); for (size_t i = 0; i < nums; ++i) { STableNode* pTable = taosArrayGetP(pTables, i); - int32_t code = createColumnNodeByTable(pCxt, pTable, *pCols); + int32_t code = createColumnsByTable(pCxt, pTable, *pCols); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -947,19 +1102,6 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { return (SNode*)pFunc; } -static int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput) { - SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); - size_t nums = taosArrayGetSize(pTables); - for (size_t i = 0; i < nums; ++i) { - STableNode* pTable = taosArrayGetP(pTables, i); - if (NULL == pTableAlias || 0 == strcmp(pTable->tableAlias, pTableAlias)) { - *pOutput = pTable; - return TSDB_CODE_SUCCESS; - } - } - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pTableAlias); -} - static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, SNodeList** pOutput) { STableNode* pTable = NULL; int32_t code = findTable(pCxt, pCol->tableAlias, &pTable); @@ -970,7 +1112,7 @@ static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, SN } } if (TSDB_CODE_SUCCESS == code) { - code = createColumnNodeByTable(pCxt, pTable, *pOutput); + code = createColumnsByTable(pCxt, pTable, *pOutput); } return code; } @@ -1052,52 +1194,33 @@ static int32_t createMultiResFuncsFromStar(STranslateContext* pCxt, SFunctionNod return code; } -static bool isCountStar(SNode* pNode) { - if (QUERY_NODE_FUNCTION != nodeType(pNode) || 1 != LIST_LENGTH(((SFunctionNode*)pNode)->pParameterList)) { - return false; - } - SNode* pPara = nodesListGetNode(((SFunctionNode*)pNode)->pParameterList, 0); - return (QUERY_NODE_COLUMN == nodeType(pPara) && 0 == strcmp(((SColumnNode*)pPara)->colName, "*")); -} - -static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) { - SColumnNode* pCol = nodesListGetNode(pCount->pParameterList, 0); - STableNode* pTable = NULL; - int32_t code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable); - if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { - setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); - } - return code; -} - static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL == pSelect->pProjectionList) { // select * ... return createAllColumns(pCxt, &pSelect->pProjectionList); } else { SNode* pNode = NULL; WHERE_EACH(pNode, pSelect->pProjectionList) { + int32_t code = TSDB_CODE_SUCCESS; if (isMultiResFunc(pNode)) { SNodeList* pFuncs = NULL; - if (TSDB_CODE_SUCCESS != createMultiResFuncsFromStar(pCxt, (SFunctionNode*)pNode, &pFuncs)) { - return TSDB_CODE_OUT_OF_MEMORY; + code = createMultiResFuncsFromStar(pCxt, (SFunctionNode*)pNode, &pFuncs); + if (TSDB_CODE_SUCCESS == code) { + INSERT_LIST(pSelect->pProjectionList, pFuncs); + ERASE_NODE(pSelect->pProjectionList); + continue; } - INSERT_LIST(pSelect->pProjectionList, pFuncs); - ERASE_NODE(pSelect->pProjectionList); - continue; } else if (isTableStar(pNode)) { SNodeList* pCols = NULL; - if (TSDB_CODE_SUCCESS != createTableAllCols(pCxt, (SColumnNode*)pNode, &pCols)) { - return TSDB_CODE_OUT_OF_MEMORY; - } - INSERT_LIST(pSelect->pProjectionList, pCols); - ERASE_NODE(pSelect->pProjectionList); - continue; - } else if (isCountStar(pNode)) { - int32_t code = rewriteCountStar(pCxt, (SFunctionNode*)pNode); - if (TSDB_CODE_SUCCESS != code) { - return code; + code = createTableAllCols(pCxt, (SColumnNode*)pNode, &pCols); + if (TSDB_CODE_SUCCESS == code) { + INSERT_LIST(pSelect->pProjectionList, pCols); + ERASE_NODE(pSelect->pProjectionList); + continue; } } + if (TSDB_CODE_SUCCESS != code) { + return code; + } WHERE_NEXT; } } @@ -1204,7 +1327,7 @@ static int32_t translateHaving(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->currClause = SQL_CLAUSE_HAVING; int32_t code = translateExpr(pCxt, pSelect->pHaving); if (TSDB_CODE_SUCCESS == code) { - code = checkExprForGroupBy(pCxt, pSelect->pHaving); + code = checkExprForGroupBy(pCxt, &pSelect->pHaving); } return code; } @@ -1213,11 +1336,120 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL != pSelect->pGroupByList && NULL != pSelect->pWindow) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST); } - pCxt->currClause = SQL_CLAUSE_GROUP_BY; - return translateExprList(pCxt, pSelect->pGroupByList); + if (NULL != pSelect->pGroupByList) { + pCxt->currClause = SQL_CLAUSE_GROUP_BY; + pSelect->isTimeOrderQuery = false; + return translateExprList(pCxt, pSelect->pGroupByList); + } + return TSDB_CODE_SUCCESS; +} + +static EDealRes isPrimaryKeyCondImpl(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + *((bool*)pContext) = ((PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) ? true : false); + return *((bool*)pContext) ? DEAL_RES_CONTINUE : DEAL_RES_END; + } + return DEAL_RES_CONTINUE; +} + +static bool isPrimaryKeyCond(SNode* pNode) { + bool isPrimaryKeyCond = false; + nodesWalkExpr(pNode, isPrimaryKeyCondImpl, &isPrimaryKeyCond); + return isPrimaryKeyCond; +} + +static int32_t getTimeRangeFromLogicCond(STranslateContext* pCxt, SLogicConditionNode* pLogicCond, + STimeWindow* pTimeRange) { + SNodeList* pPrimaryKeyConds = NULL; + SNode* pCond = NULL; + FOREACH(pCond, pLogicCond->pParameterList) { + if (isPrimaryKeyCond(pCond)) { + if (TSDB_CODE_SUCCESS != nodesListMakeAppend(&pPrimaryKeyConds, pCond)) { + nodesClearList(pPrimaryKeyConds); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + } + + if (NULL == pPrimaryKeyConds) { + *pTimeRange = TSWINDOW_INITIALIZER; + return TSDB_CODE_SUCCESS; + } + + SLogicConditionNode* pPrimaryKeyLogicCond = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + if (NULL == pPrimaryKeyLogicCond) { + nodesClearList(pPrimaryKeyConds); + return TSDB_CODE_OUT_OF_MEMORY; + } + pPrimaryKeyLogicCond->condType = LOGIC_COND_TYPE_AND; + pPrimaryKeyLogicCond->pParameterList = pPrimaryKeyConds; + bool isStrict = false; + int32_t code = filterGetTimeRange((SNode*)pPrimaryKeyLogicCond, pTimeRange, &isStrict); + nodesClearList(pPrimaryKeyConds); + pPrimaryKeyLogicCond->pParameterList = NULL; + nodesDestroyNode(pPrimaryKeyLogicCond); + return code; +} + +static int32_t getTimeRange(STranslateContext* pCxt, SNode* pWhere, STimeWindow* pTimeRange) { + if (NULL == pWhere) { + *pTimeRange = TSWINDOW_INITIALIZER; + return TSDB_CODE_SUCCESS; + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere) && + LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)pWhere)->condType) { + return getTimeRangeFromLogicCond(pCxt, (SLogicConditionNode*)pWhere, pTimeRange); + } + + if (isPrimaryKeyCond(pWhere)) { + bool isStrict = false; + return filterGetTimeRange(pWhere, pTimeRange, &isStrict); + } else { + *pTimeRange = TSWINDOW_INITIALIZER; + } + return TSDB_CODE_SUCCESS; } -static bool isValTimeUnit(char unit) { return ('n' == unit || 'y' == unit); } +static int32_t checkFill(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { + SFillNode* pFill = (SFillNode*)pInterval->pFill; + if (TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_INITIALIZER) || + TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_DESC_INITIALIZER)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); + } + + int64_t timeRange = TABS(pFill->timeRange.skey - pFill->timeRange.ekey); + int64_t intervalRange = 0; + SValueNode* pInter = (SValueNode*)pInterval->pInterval; + if (TIME_IS_VAR_DURATION(pInter->unit)) { + int64_t f = 1; + if (pInter->unit == 'n') { + f = 30L * MILLISECOND_PER_DAY; + } else if (pInter->unit == 'y') { + f = 365L * MILLISECOND_PER_DAY; + } + intervalRange = pInter->datum.i * f; + } else { + intervalRange = pInter->datum.i; + } + if ((timeRange == 0) || (timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateFill(STranslateContext* pCxt, SNode* pWhere, SIntervalWindowNode* pInterval) { + if (NULL == pInterval->pFill) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = getTimeRange(pCxt, pWhere, &(((SFillNode*)pInterval->pFill)->timeRange)); + if (TSDB_CODE_SUCCESS == code) { + code = checkFill(pCxt, pInterval); + } + return code; +} static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char unit) { int64_t days = convertTimeFromPrecisionToUnit(val, fromPrecision, 'd'); @@ -1241,11 +1473,11 @@ static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char uni return -1; } -static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { +static int32_t checkIntervalWindow(STranslateContext* pCxt, SNode* pWhere, SIntervalWindowNode* pInterval) { uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; SValueNode* pInter = (SValueNode*)pInterval->pInterval; - bool valInter = isValTimeUnit(pInter->unit); + bool valInter = TIME_IS_VAR_DURATION(pInter->unit); if (pInter->datum.i <= 0 || (!valInter && convertTimePrecision(pInter->datum.i, precision, TSDB_TIME_PRECISION_MICRO) < tsMinIntervalTime)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime); @@ -1259,7 +1491,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* if (pInter->unit == 'n' && pOffset->unit == 'y') { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT); } - bool fixed = !isValTimeUnit(pOffset->unit) && !valInter; + bool fixed = !TIME_IS_VAR_DURATION(pOffset->unit) && !valInter; if ((fixed && pOffset->datum.i >= pInter->datum.i) || (!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >= getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { @@ -1271,7 +1503,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* const static int32_t INTERVAL_SLIDING_FACTOR = 100; SValueNode* pSliding = (SValueNode*)pInterval->pSliding; - if (pInter->unit == 'n' || pInter->unit == 'y') { + if (TIME_IS_VAR_DURATION(pSliding->unit)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT); } if ((pSliding->datum.i < convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, precision)) || @@ -1283,7 +1515,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* } } - return TSDB_CODE_SUCCESS; + return translateFill(pCxt, pWhere, pInterval); } static EDealRes checkStateExpr(SNode* pNode, void* pContext) { @@ -1320,28 +1552,28 @@ static int32_t checkSessionWindow(STranslateContext* pCxt, SSessionWindowNode* p return TSDB_CODE_SUCCESS; } -static int32_t checkWindow(STranslateContext* pCxt, SNode* pWindow) { - switch (nodeType(pWindow)) { +static int32_t checkWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + switch (nodeType(pSelect->pWindow)) { case QUERY_NODE_STATE_WINDOW: - return checkStateWindow(pCxt, (SStateWindowNode*)pWindow); + return checkStateWindow(pCxt, (SStateWindowNode*)pSelect->pWindow); case QUERY_NODE_SESSION_WINDOW: - return checkSessionWindow(pCxt, (SSessionWindowNode*)pWindow); + return checkSessionWindow(pCxt, (SSessionWindowNode*)pSelect->pWindow); case QUERY_NODE_INTERVAL_WINDOW: - return checkIntervalWindow(pCxt, (SIntervalWindowNode*)pWindow); + return checkIntervalWindow(pCxt, pSelect->pWhere, (SIntervalWindowNode*)pSelect->pWindow); default: break; } return TSDB_CODE_SUCCESS; } -static int32_t translateWindow(STranslateContext* pCxt, SNode* pWindow) { - if (NULL == pWindow) { +static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; } pCxt->currClause = SQL_CLAUSE_WINDOW; - int32_t code = translateExpr(pCxt, pWindow); + int32_t code = translateExpr(pCxt, pSelect->pWindow); if (TSDB_CODE_SUCCESS == code) { - code = checkWindow(pCxt, pWindow); + code = checkWindow(pCxt, pSelect); } return code; } @@ -1378,6 +1610,48 @@ static int32_t checkLimit(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } +static int32_t createPrimaryKeyColByTable(STranslateContext* pCxt, STableNode* pTable, SNode** pPrimaryKey) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); + } else { + // todo + } + *pPrimaryKey = (SNode*)pCol; + return TSDB_CODE_SUCCESS; +} + +static int32_t createPrimaryKeyCol(STranslateContext* pCxt, SNode** pPrimaryKey) { + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, NULL, &pTable); + if (TSDB_CODE_SUCCESS == code) { + code = createPrimaryKeyColByTable(pCxt, pTable, pPrimaryKey); + } + return code; +} + +static EDealRes rewriteTimelineFuncImpl(SNode* pNode, void* pContext) { + STranslateContext* pCxt = pContext; + if (isTimelineFunc(pNode)) { + SFunctionNode* pFunc = (SFunctionNode*)pNode; + SNode* pPrimaryKey = NULL; + pCxt->errCode = createPrimaryKeyCol(pCxt, &pPrimaryKey); + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + pCxt->errCode = nodesListMakeStrictAppend(&pFunc->pParameterList, pPrimaryKey); + } + return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR; + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SSelectStmt* pSelect) { + nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, rewriteTimelineFuncImpl, pCxt); + return pCxt->errCode; +} + static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrStmt = pSelect; int32_t code = translateFrom(pCxt, pSelect); @@ -1388,7 +1662,7 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { code = translatePartitionBy(pCxt, pSelect->pPartitionByList); } if (TSDB_CODE_SUCCESS == code) { - code = translateWindow(pCxt, pSelect->pWindow); + code = translateWindow(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { code = translateGroupBy(pCxt, pSelect); @@ -1408,6 +1682,9 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) { code = checkLimit(pCxt, pSelect); } + if (TSDB_CODE_SUCCESS == code) { + code = rewriteTimelineFunc(pCxt, pSelect); + } return code; } @@ -1539,26 +1816,24 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameGetFullDbName(&name, pReq->db); pReq->numOfVgroups = pStmt->pOptions->numOfVgroups; + pReq->numOfStables = pStmt->pOptions->singleStable; + pReq->buffer = pStmt->pOptions->buffer; + pReq->pageSize = pStmt->pOptions->pagesize; + pReq->pages = pStmt->pOptions->pages; pReq->daysPerFile = pStmt->pOptions->daysPerFile; pReq->daysToKeep0 = pStmt->pOptions->keep[0]; pReq->daysToKeep1 = pStmt->pOptions->keep[1]; pReq->daysToKeep2 = pStmt->pOptions->keep[2]; pReq->minRows = pStmt->pOptions->minRowsPerBlock; pReq->maxRows = pStmt->pOptions->maxRowsPerBlock; - pReq->commitTime = -1; pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod; pReq->walLevel = pStmt->pOptions->walLevel; pReq->precision = pStmt->pOptions->precision; pReq->compression = pStmt->pOptions->compressionLevel; pReq->replications = pStmt->pOptions->replica; - pReq->update = -1; + pReq->strict = pStmt->pOptions->strict; pReq->cacheLastRow = pStmt->pOptions->cachelast; pReq->ignoreExist = pStmt->ignoreExists; - pReq->singleSTable = pStmt->pOptions->singleStable; - pReq->strict = pStmt->pOptions->strict; - // pStmt->pOptions->buffer; - // pStmt->pOptions->pages; - // pStmt->pOptions->pagesize; return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq); } @@ -1664,11 +1939,12 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION); } - SNode* pNode = NULL; - FOREACH(pNode, pRetentions) { - SNode* pVal = NULL; - FOREACH(pVal, ((SNodeListNode*)pNode)->pNodeList) { - if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pVal)) { + SNode* pRetention = NULL; + FOREACH(pRetention, pRetentions) { + SNode* pNode = NULL; + FOREACH(pNode, ((SNodeListNode*)pRetention)->pNodeList) { + SValueNode* pVal = (SValueNode*)pNode; + if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { return pCxt->errCode; } } @@ -1677,11 +1953,12 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete return TSDB_CODE_SUCCESS; } -static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, - bool alter) { +static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { int32_t daysPerFile = pOptions->daysPerFile; int32_t daysToKeep0 = pOptions->keep[0]; - if (alter && (-1 == daysPerFile || -1 == daysToKeep0)) { + if (-1 == daysPerFile && -1 == daysToKeep0) { + return TSDB_CODE_SUCCESS; + } else if (-1 == daysPerFile || -1 == daysToKeep0) { SDbCfgInfo dbCfg; int32_t code = getDBCfg(pCxt, pDbName, &dbCfg); if (TSDB_CODE_SUCCESS != code) { @@ -1696,9 +1973,9 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa return TSDB_CODE_SUCCESS; } -static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, - bool alter) { - int32_t code = checkRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, INT32_MAX); +static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { + int32_t code = + checkRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE); if (TSDB_CODE_SUCCESS == code) { code = checkRangeOption(pCxt, "cacheLast", pOptions->cachelast, TSDB_MIN_DB_CACHE_LAST_ROW, TSDB_MAX_DB_CACHE_LAST_ROW); @@ -1724,7 +2001,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbKeepOption(pCxt, pOptions); } if (TSDB_CODE_SUCCESS == code) { - code = checkRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, INT32_MAX); + code = checkRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, TSDB_MAX_PAGES_PER_VNODE); } if (TSDB_CODE_SUCCESS == code) { code = checkRangeOption(pCxt, "pagesize", pOptions->pagesize, TSDB_MIN_PAGESIZE_PER_VNODE, @@ -1753,13 +2030,13 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbRetentionsOption(pCxt, pOptions->pRetentions); } if (TSDB_CODE_SUCCESS == code) { - code = checkOptionsDependency(pCxt, pDbName, pOptions, alter); + code = checkOptionsDependency(pCxt, pDbName, pOptions); } return code; } static int32_t checkCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt) { - return checkDatabaseOptions(pCxt, pStmt->dbName, pStmt->pOptions, false); + return checkDatabaseOptions(pCxt, pStmt->dbName, pStmt->pOptions); } typedef int32_t (*FSerializeFunc)(void* pBuf, int32_t bufLen, void* pReq); @@ -1810,21 +2087,23 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, SName name = {0}; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameGetFullDbName(&name, pReq->db); - // pStmt->pOptions->buffer - pReq->cacheLastRow = pStmt->pOptions->cachelast; - pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod; + pReq->buffer = pStmt->pOptions->buffer; + pReq->pageSize = -1; + pReq->pages = pStmt->pOptions->pages; + pReq->daysPerFile = -1; pReq->daysToKeep0 = pStmt->pOptions->keep[0]; pReq->daysToKeep1 = pStmt->pOptions->keep[1]; pReq->daysToKeep2 = pStmt->pOptions->keep[2]; - // pStmt->pOptions->pages - pReq->replications = pStmt->pOptions->replica; - pReq->strict = pStmt->pOptions->strict; + pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod; pReq->walLevel = pStmt->pOptions->walLevel; + pReq->strict = pStmt->pOptions->strict; + pReq->cacheLastRow = pStmt->pOptions->cachelast; + pReq->replications = pStmt->pOptions->replica; return; } static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt) { - int32_t code = checkDatabaseOptions(pCxt, pStmt->dbName, pStmt->pOptions, true); + int32_t code = checkDatabaseOptions(pCxt, pStmt->dbName, pStmt->pOptions); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -1895,17 +2174,6 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS return TSDB_CODE_SUCCESS; } -static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt) { - SNode* pNode; - FOREACH(pNode, pStmt->pTags) { - SColumnDefNode* pCol = (SColumnDefNode*)pNode; - if (pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); - } - } - return TSDB_CODE_SUCCESS; -} - static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) { if (NULL == pFuncs) { return TSDB_CODE_SUCCESS; @@ -1917,6 +2185,113 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs return TSDB_CODE_SUCCESS; } +static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SNodeList* pTags) { + int32_t ntags = LIST_LENGTH(pTags); + if (0 == ntags) { + return TSDB_CODE_SUCCESS; + } else if (ntags > TSDB_MAX_TAGS) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_NUM); + } + + int32_t code = TSDB_CODE_SUCCESS; + int32_t tagsSize = 0; + SNode* pNode = NULL; + FOREACH(pNode, pTags) { + SColumnDefNode* pTag = (SColumnDefNode*)pNode; + int32_t len = strlen(pTag->colName); + if (NULL != taosHashGet(pHash, pTag->colName, len)) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN); + } + if (TSDB_CODE_SUCCESS == code && pTag->dataType.type == TSDB_DATA_TYPE_JSON && ntags > 1) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); + } + if (TSDB_CODE_SUCCESS == code) { + if ((TSDB_DATA_TYPE_VARCHAR == pTag->dataType.type && pTag->dataType.bytes > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_NCHAR == pTag->dataType.type && pTag->dataType.bytes > TSDB_MAX_NCHAR_LEN)) { + code = code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); + } + } + if (TSDB_CODE_SUCCESS == code) { + code = taosHashPut(pHash, pTag->colName, len, &pTag, POINTER_BYTES); + } + if (TSDB_CODE_SUCCESS == code) { + tagsSize += pTag->dataType.bytes; + } else { + break; + } + } + + if (TSDB_CODE_SUCCESS == code && tagsSize > TSDB_MAX_TAGS_LEN) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_LENGTH, TSDB_MAX_TAGS_LEN); + } + + return code; +} + +static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, SNodeList* pCols) { + int32_t ncols = LIST_LENGTH(pCols); + if (ncols < TSDB_MIN_COLUMNS) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); + } else if (ncols > TSDB_MAX_COLUMNS) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TOO_MANY_COLUMNS); + } + + int32_t code = TSDB_CODE_SUCCESS; + + bool first = true; + int32_t rowSize = 0; + SNode* pNode = NULL; + FOREACH(pNode, pCols) { + SColumnDefNode* pCol = (SColumnDefNode*)pNode; + if (first) { + first = false; + if (TSDB_DATA_TYPE_TIMESTAMP != pCol->dataType.type) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FIRST_COLUMN); + } + } + int32_t len = strlen(pCol->colName); + if (TSDB_CODE_SUCCESS == code && NULL != taosHashGet(pHash, pCol->colName, len)) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN); + } + if (TSDB_CODE_SUCCESS == code) { + if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && pCol->dataType.bytes > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && pCol->dataType.bytes > TSDB_MAX_NCHAR_LEN)) { + code = code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); + } + } + if (TSDB_CODE_SUCCESS == code) { + code = taosHashPut(pHash, pCol->colName, len, &pCol, POINTER_BYTES); + } + if (TSDB_CODE_SUCCESS == code) { + rowSize += pCol->dataType.bytes; + } else { + break; + } + } + + if (TSDB_CODE_SUCCESS == code && rowSize > TSDB_MAX_BYTES_PER_ROW) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROW_LENGTH, TSDB_MAX_BYTES_PER_ROW); + } + + return code; +} + +static int32_t checkTableSchema(STranslateContext* pCxt, SCreateTableStmt* pStmt) { + SHashObj* pHash = taosHashInit(LIST_LENGTH(pStmt->pTags) + LIST_LENGTH(pStmt->pCols), + taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == pHash) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = checkTableTagsSchema(pCxt, pHash, pStmt->pTags); + if (TSDB_CODE_SUCCESS == code) { + code = checkTableColsSchema(pCxt, pHash, pStmt->pCols); + } + + taosHashCleanup(pHash); + return code; +} + static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { int32_t code = checkRangeOption(pCxt, "delay", pStmt->pOptions->delay, TSDB_MIN_ROLLUP_DELAY, TSDB_MAX_ROLLUP_DELAY); if (TSDB_CODE_SUCCESS == code) { @@ -1932,7 +2307,7 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt code = checkTableSmaOption(pCxt, pStmt); } if (TSDB_CODE_SUCCESS == code) { - code = checkTableTags(pCxt, pStmt); + code = checkTableSchema(pCxt, pStmt); } return code; } @@ -3134,11 +3509,11 @@ static int32_t rewriteShow(STranslateContext* pCxt, SQuery* pQuery) { return code; } -typedef struct SVgroupTablesBatch { +typedef struct SVgroupCreateTableBatch { SVCreateTbBatchReq req; SVgroupInfo info; char dbName[TSDB_DB_NAME_LEN]; -} SVgroupTablesBatch; +} SVgroupCreateTableBatch; static void destroyCreateTbReq(SVCreateTbReq* pReq) { taosMemoryFreeClear(pReq->name); @@ -3146,7 +3521,7 @@ static void destroyCreateTbReq(SVCreateTbReq* pReq) { } static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* pStmt, const SVgroupInfo* pVgroupInfo, - SVgroupTablesBatch* pBatch) { + SVgroupCreateTableBatch* pBatch) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; strcpy(name.dbname, pStmt->dbName); @@ -3180,13 +3555,13 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* return TSDB_CODE_SUCCESS; } -static int32_t serializeVgroupTablesBatch(SVgroupTablesBatch* pTbBatch, SArray* pBufArray) { +static int32_t serializeVgroupCreateTableBatch(SVgroupCreateTableBatch* pTbBatch, SArray* pBufArray) { int tlen; SCoder coder = {0}; int32_t ret = 0; tEncodeSize(tEncodeSVCreateTbBatchReq, &pTbBatch->req, tlen, ret); - tlen += sizeof(SMsgHead); //+ tSerializeSVCreateTbBatchReq(NULL, &(pTbBatch->req)); + tlen += sizeof(SMsgHead); void* buf = taosMemoryMalloc(tlen); if (NULL == buf) { return TSDB_CODE_OUT_OF_MEMORY; @@ -3212,7 +3587,7 @@ static int32_t serializeVgroupTablesBatch(SVgroupTablesBatch* pTbBatch, SArray* return TSDB_CODE_SUCCESS; } -static void destroyCreateTbReqBatch(SVgroupTablesBatch* pTbBatch) { +static void destroyCreateTbReqBatch(SVgroupCreateTableBatch* pTbBatch) { size_t size = taosArrayGetSize(pTbBatch->req.pArray); for (int32_t i = 0; i < size; ++i) { SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i); @@ -3257,10 +3632,10 @@ static int32_t buildCreateTableDataBlock(int32_t acctId, const SCreateTableStmt* return TSDB_CODE_OUT_OF_MEMORY; } - SVgroupTablesBatch tbatch = {0}; - int32_t code = buildNormalTableBatchReq(acctId, pStmt, pInfo, &tbatch); + SVgroupCreateTableBatch tbatch = {0}; + int32_t code = buildNormalTableBatchReq(acctId, pStmt, pInfo, &tbatch); if (TSDB_CODE_SUCCESS == code) { - code = serializeVgroupTablesBatch(&tbatch, *pBufArray); + code = serializeVgroupCreateTableBatch(&tbatch, *pBufArray); } destroyCreateTbReqBatch(&tbatch); @@ -3305,9 +3680,9 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c req.ctb.suid = suid; req.ctb.pTag = row; - SVgroupTablesBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); + SVgroupCreateTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); if (pTableBatch == NULL) { - SVgroupTablesBatch tBatch = {0}; + SVgroupCreateTableBatch tBatch = {0}; tBatch.info = *pVgInfo; strcpy(tBatch.dbName, pDbName); @@ -3480,21 +3855,21 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla return code; } -static SArray* serializeVgroupsTablesBatch(int32_t acctId, SHashObj* pVgroupHashmap) { +static SArray* serializeVgroupsCreateTableBatch(int32_t acctId, SHashObj* pVgroupHashmap) { SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*)); if (NULL == pBufArray) { return NULL; } - int32_t code = TSDB_CODE_SUCCESS; - SVgroupTablesBatch* pTbBatch = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SVgroupCreateTableBatch* pTbBatch = NULL; do { pTbBatch = taosHashIterate(pVgroupHashmap, pTbBatch); if (pTbBatch == NULL) { break; } - serializeVgroupTablesBatch(pTbBatch, pBufArray); + serializeVgroupCreateTableBatch(pTbBatch, pBufArray); destroyCreateTbReqBatch(pTbBatch); } while (true); @@ -3519,7 +3894,147 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) } } - SArray* pBufArray = serializeVgroupsTablesBatch(pCxt->pParseCxt->acctId, pVgroupHashmap); + SArray* pBufArray = serializeVgroupsCreateTableBatch(pCxt->pParseCxt->acctId, pVgroupHashmap); + taosHashCleanup(pVgroupHashmap); + if (NULL == pBufArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return rewriteToVnodeModifOpStmt(pQuery, pBufArray); +} + +typedef struct SVgroupDropTableBatch { + SVDropTbBatchReq req; + SVgroupInfo info; + char dbName[TSDB_DB_NAME_LEN]; +} SVgroupDropTableBatch; + +static void addDropTbReqIntoVgroup(SHashObj* pVgroupHashmap, SDropTableClause* pClause, SVgroupInfo* pVgInfo) { + SVDropTbReq req = {.name = pClause->tableName, .igNotExists = pClause->ignoreNotExists}; + SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); + if (NULL == pTableBatch) { + SVgroupDropTableBatch tBatch = {0}; + tBatch.info = *pVgInfo; + tBatch.req.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq)); + taosArrayPush(tBatch.req.pArray, &req); + + taosHashPut(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &tBatch, sizeof(tBatch)); + } else { // add to the correct vgroup + taosArrayPush(pTableBatch->req.pArray, &req); + } +} + +static int32_t buildDropTableVgroupHashmap(STranslateContext* pCxt, SDropTableClause* pClause, bool* pIsSuperTable, + SHashObj* pVgroupHashmap) { + STableMeta* pTableMeta = NULL; + int32_t code = getTableMeta(pCxt, pClause->dbName, pClause->tableName, &pTableMeta); + + if (TSDB_CODE_SUCCESS == code && TSDB_SUPER_TABLE == pTableMeta->tableType) { + *pIsSuperTable = true; + goto over; + } + + if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code && pClause->ignoreNotExists) { + code = TSDB_CODE_SUCCESS; + } + + *pIsSuperTable = false; + + SVgroupInfo info = {0}; + if (TSDB_CODE_SUCCESS == code) { + code = getTableHashVgroup(pCxt, pClause->dbName, pClause->tableName, &info); + } + if (TSDB_CODE_SUCCESS == code) { + addDropTbReqIntoVgroup(pVgroupHashmap, pClause, &info); + } + +over: + taosMemoryFreeClear(pTableMeta); + return code; +} + +static void destroyDropTbReqBatch(SVgroupDropTableBatch* pTbBatch) { taosArrayDestroy(pTbBatch->req.pArray); } + +static int32_t serializeVgroupDropTableBatch(SVgroupDropTableBatch* pTbBatch, SArray* pBufArray) { + int tlen; + SCoder coder = {0}; + + int32_t ret = 0; + tEncodeSize(tEncodeSVDropTbBatchReq, &pTbBatch->req, tlen, ret); + tlen += sizeof(SMsgHead); + void* buf = taosMemoryMalloc(tlen); + if (NULL == buf) { + return TSDB_CODE_OUT_OF_MEMORY; + } + ((SMsgHead*)buf)->vgId = htonl(pTbBatch->info.vgId); + ((SMsgHead*)buf)->contLen = htonl(tlen); + void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + tCoderInit(&coder, TD_LITTLE_ENDIAN, pBuf, tlen - sizeof(SMsgHead), TD_ENCODER); + tEncodeSVDropTbBatchReq(&coder, &pTbBatch->req); + tCoderClear(&coder); + + SVgDataBlocks* pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == pVgData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pVgData->vg = pTbBatch->info; + pVgData->pData = buf; + pVgData->size = tlen; + pVgData->numOfTables = (int32_t)taosArrayGetSize(pTbBatch->req.pArray); + taosArrayPush(pBufArray, &pVgData); + + return TSDB_CODE_SUCCESS; +} + +static SArray* serializeVgroupsDropTableBatch(int32_t acctId, SHashObj* pVgroupHashmap) { + SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*)); + if (NULL == pBufArray) { + return NULL; + } + + int32_t code = TSDB_CODE_SUCCESS; + SVgroupDropTableBatch* pTbBatch = NULL; + do { + pTbBatch = taosHashIterate(pVgroupHashmap, pTbBatch); + if (pTbBatch == NULL) { + break; + } + + serializeVgroupDropTableBatch(pTbBatch, pBufArray); + destroyDropTbReqBatch(pTbBatch); + } while (true); + + return pBufArray; +} + +static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { + SDropTableStmt* pStmt = (SDropTableStmt*)pQuery->pRoot; + + SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == pVgroupHashmap) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + bool isSuperTable = false; + SNode* pNode; + FOREACH(pNode, pStmt->pTables) { + int32_t code = buildDropTableVgroupHashmap(pCxt, (SDropTableClause*)pNode, &isSuperTable, pVgroupHashmap); + if (TSDB_CODE_SUCCESS != code) { + taosHashCleanup(pVgroupHashmap); + return code; + } + if (isSuperTable && LIST_LENGTH(pStmt->pTables) > 1) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DROP_STABLE); + } + } + + if (isSuperTable) { + taosHashCleanup(pVgroupHashmap); + return TSDB_CODE_SUCCESS; + } + + SArray* pBufArray = serializeVgroupsDropTableBatch(pCxt->pParseCxt->acctId, pVgroupHashmap); taosHashCleanup(pVgroupHashmap); if (NULL == pBufArray) { return TSDB_CODE_OUT_OF_MEMORY; @@ -3565,6 +4080,9 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_CREATE_MULTI_TABLE_STMT: code = rewriteCreateMultiTable(pCxt, pQuery); break; + case QUERY_NODE_DROP_TABLE_STMT: + code = rewriteDropTable(pCxt, pQuery); + break; case QUERY_NODE_ALTER_TABLE_STMT: if (TSDB_ALTER_TABLE_UPDATE_TAG_VAL == ((SAlterTableStmt*)pQuery->pRoot)->alterType) { code = rewriteAlterTable(pCxt, pQuery); @@ -3644,7 +4162,8 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { int32_t translate(SParseContext* pParseCxt, SQuery* pQuery) { STranslateContext cxt = {0}; - int32_t code = initTranslateContext(pParseCxt, &cxt); + + int32_t code = initTranslateContext(pParseCxt, &cxt); if (TSDB_CODE_SUCCESS == code) { code = fmFuncMgtInit(); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index d75328300899d4d0bf9d817a0717a6cdabced70e..e37dbd1eddac0ffc7a4ac64357b4a2a35a9d0676 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -126,6 +126,26 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "slimit/soffset only available for PARTITION BY query"; case TSDB_CODE_PAR_INVALID_TOPIC_QUERY: return "Invalid topic query"; + case TSDB_CODE_PAR_INVALID_DROP_STABLE: + return "Cannot drop super table in batch"; + case TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE: + return "Start(end) time of query range required or time range too large"; + case TSDB_CODE_PAR_DUPLICATED_COLUMN: + return "Duplicated column names"; + case TSDB_CODE_PAR_INVALID_TAGS_LENGTH: + return "Tags length exceeds max length %d"; + case TSDB_CODE_PAR_INVALID_ROW_LENGTH: + return "Row length exceeds max length %d"; + case TSDB_CODE_PAR_INVALID_COLUMNS_NUM: + return "Illegal number of columns"; + case TSDB_CODE_PAR_TOO_MANY_COLUMNS: + return "Too many columns"; + case TSDB_CODE_PAR_INVALID_FIRST_COLUMN: + return "First column must be timestamp"; + case TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN: + return "Invalid binary/nchar column length"; + case TSDB_CODE_PAR_INVALID_TAGS_NUM: + return "Invalid number of tag columns"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: @@ -138,7 +158,6 @@ int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...) { va_start(vArgList, errCode); vsnprintf(pBuf->buf, pBuf->len, getSyntaxErrFormat(errCode), vArgList); va_end(vArgList); - terrno = errCode; return errCode; } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 8b60001172266934eb6ef8d9498e6925095850a8..4d06024c68440dd6fa451997ad72720d66dbf0d1 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -3021,11 +3021,11 @@ static YYACTIONTYPE yy_reduce( /********** Begin reduce actions **********************************************/ YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ -{ pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +{ pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } yy_destructor(yypParser,232,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ -{ pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +{ pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } yy_destructor(yypParser,233,&yymsp[0].minor); break; case 2: /* account_options ::= */ @@ -3591,7 +3591,7 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT, NULL, NULL); } break; case 178: /* cmd ::= SHOW ACCOUNTS */ -{ pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +{ pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } break; case 179: /* cmd ::= SHOW APPS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT, NULL, NULL); } @@ -3902,7 +3902,8 @@ static YYACTIONTYPE yy_reduce( yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 272: /* signed_literal ::= NULL */ -{ yymsp[0].minor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL); } +{ yylhsminor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 291: /* expression ::= NK_LP expression NK_RP */ case 355: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==355); @@ -4352,13 +4353,12 @@ static void yy_syntax_error( #define TOKEN yyminor /************ Begin %syntax_error code ****************************************/ - if (pCxt->valid) { + if (TSDB_CODE_SUCCESS == pCxt->errCode) { if(TOKEN.z) { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); } else { - generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); } - pCxt->valid = false; } /************ End %syntax_error code ******************************************/ ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index 96de3fb7356a6cf3acb4df828e3284c472f4ef07..0e8adb978dd0e9fca5a67e9999ce7c5faa877cc0 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -24,4 +24,9 @@ if(${BUILD_WINGETOPT}) PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" ) target_link_libraries(parserTest PUBLIC wingetopt) -endif() \ No newline at end of file +endif() + +add_test( + NAME parserTest + COMMAND parserTest +) diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 289888c71b3d6ff8063215e13d12e859f0d9e9fe..2d9a907d9bf64199ffbca73d1f2f5a3ac92d6504 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -100,6 +100,17 @@ void generateInformationSchema(MockCatalogService* mcs) { } } +/* + * Table:t1 + * Field | Type | DataType | Bytes | + * ========================================================================== + * ts | column | TIMESTAMP | 8 | + * c1 | column | INT | 4 | + * c2 | column | VARCHAR | 20 | + * c3 | column | BIGINT | 8 | + * c4 | column | DOUBLE | 8 | + * c5 | column | DOUBLE | 8 | + */ void generateTestT1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 6) .setPrecision(TSDB_TIME_PRECISION_MILLI) @@ -113,6 +124,17 @@ void generateTestT1(MockCatalogService* mcs) { builder.done(); } +/* + * Super Table: st1 + * Field | Type | DataType | Bytes | + * ========================================================================== + * ts | column | TIMESTAMP | 8 | + * c1 | column | INT | 4 | + * c2 | column | VARCHAR | 20 | + * tag1 | tag | INT | 4 | + * tag2 | tag | VARCHAR | 20 | + * Child Table: st1s1, st1s2 + */ void generateTestST1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 2) .setPrecision(TSDB_TIME_PRECISION_MILLI) @@ -155,16 +177,22 @@ int32_t __catalogGetDBVgInfo(SCatalog* pCtg, void* pRpc, const SEpSet* pMgmtEps, return 0; } +int32_t __catalogGetDBCfg(SCatalog* pCtg, void* pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { + return 0; +} + void initMetaDataEnv() { mockCatalogService.reset(new MockCatalogService()); static Stub stub; stub.set(catalogGetHandle, __catalogGetHandle); stub.set(catalogGetTableMeta, __catalogGetTableMeta); + stub.set(catalogGetSTableMeta, __catalogGetTableMeta); stub.set(catalogGetTableHashVgroup, __catalogGetTableHashVgroup); stub.set(catalogGetTableDistVgInfo, __catalogGetTableDistVgInfo); stub.set(catalogGetDBVgVersion, __catalogGetDBVgVersion); stub.set(catalogGetDBVgInfo, __catalogGetDBVgInfo); + stub.set(catalogGetDBCfg, __catalogGetDBCfg); // { // AddrAny any("libcatalog.so"); // std::map result; diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index ef989e919051c4e9f5565e7963796d8d8d46a0e5..f86cecb9e3399bf6b5b55c59adcc6b99e1950468 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -103,7 +103,6 @@ class MockCatalogServiceImpl { const char* tname = tNameGetTableName(pTableName); int32_t code = copyTableSchemaMeta(db, tname, &table); if (TSDB_CODE_SUCCESS != code) { - std::cout << "db : " << db << ", table :" << tname << std::endl; return code; } *pTableMeta = table.release(); @@ -111,9 +110,8 @@ class MockCatalogServiceImpl { } int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { - char db[TSDB_DB_NAME_LEN] = {0}; - tNameGetDbName(pTableName, db); - return copyTableVgroup(db, tNameGetTableName(pTableName), vgInfo); + vgInfo->vgId = 1; + return TSDB_CODE_SUCCESS; } int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** vgList) const { diff --git a/source/libs/parser/test/parExplainToSyncdbTest.cpp b/source/libs/parser/test/parExplainToSyncdbTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c12912ef337519585524a2d3f5c2d485bc661885 --- /dev/null +++ b/source/libs/parser/test/parExplainToSyncdbTest.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parTestUtil.h" + +using namespace std; + +namespace ParserTest { + +class ParserExplainToSyncdbTest : public ParserTestBase {}; + +TEST_F(ParserExplainToSyncdbTest, explain) { + useDb("root", "test"); + + run("explain SELECT * FROM t1"); + + run("explain analyze SELECT * FROM t1"); + + run("explain analyze verbose true ratio 0.01 SELECT * FROM t1"); +} + +// todo kill connection +// todo kill query +// todo kill stream +// todo merge vgroup +// todo redistribute vgroup +// todo reset query cache +// todo syncdb + +} // namespace ParserTest diff --git a/source/libs/parser/test/parInitialATest.cpp b/source/libs/parser/test/parInitialATest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e58f50f52aeedfe6767b02b44d0c0d0d84037b0a --- /dev/null +++ b/source/libs/parser/test/parInitialATest.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parTestUtil.h" + +using namespace std; + +namespace ParserTest { + +class ParserInitialATest : public ParserTestBase {}; + +TEST_F(ParserInitialATest, alterAccount) { + useDb("root", "test"); + + run("alter account ac_wxy pass '123456'", TSDB_CODE_PAR_EXPRIE_STATEMENT); +} + +TEST_F(ParserInitialATest, alterDnode) { + useDb("root", "test"); + + run("alter dnode 1 'resetLog'"); + + run("alter dnode 1 'debugFlag' '134'"); +} + +TEST_F(ParserInitialATest, alterDatabase) { + useDb("root", "test"); + + run("alter database wxy_db cachelast 1 fsync 200 wal 1"); +} + +// todo alter local +// todo alter stable +// todo alter table + +TEST_F(ParserInitialATest, alterUser) { + useDb("root", "test"); + + run("alter user wxy pass '123456'"); + + run("alter user wxy privilege 'write'"); +} + +TEST_F(ParserInitialATest, bug001) { + useDb("root", "test"); + + run("alter database db wal 0 # td-14436", TSDB_CODE_PAR_SYNTAX_ERROR); +} + +} // namespace ParserTest \ No newline at end of file diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..540560fcd72a3c4fa0e579dfab69fa72937a63c2 --- /dev/null +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parTestUtil.h" + +using namespace std; + +namespace ParserTest { + +class ParserInitialCTest : public ParserTestBase {}; + +// todo compact + +TEST_F(ParserInitialCTest, createAccount) { + useDb("root", "test"); + + run("create account ac_wxy pass '123456'", TSDB_CODE_PAR_EXPRIE_STATEMENT); +} + +TEST_F(ParserInitialCTest, createBnode) { + useDb("root", "test"); + + run("create bnode on dnode 1"); +} + +TEST_F(ParserInitialCTest, createDatabase) { + useDb("root", "test"); + + run("create database wxy_db"); + + run("create database if not exists wxy_db " + "cachelast 2 " + "comp 1 " + "days 100 " + "fsync 100 " + "maxrows 1000 " + "minrows 100 " + "keep 1440 " + "precision 'ms' " + "replica 3 " + "wal 2 " + "vgroups 100 " + "single_stable 0 " + "retentions 15s:7d,1m:21d,15m:5y"); + + run("create database if not exists wxy_db " + "days 100m " + "keep 1440m,300h,400d "); +} + +TEST_F(ParserInitialCTest, createDnode) { + useDb("root", "test"); + + run("create dnode abc1 port 7000"); + + run("create dnode 1.1.1.1 port 9000"); +} + +// todo create function + +TEST_F(ParserInitialCTest, createIndexSma) { + useDb("root", "test"); + + run("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) INTERVAL(10s)"); +} + +TEST_F(ParserInitialCTest, createMnode) { + useDb("root", "test"); + + run("create mnode on dnode 1"); +} + +TEST_F(ParserInitialCTest, createQnode) { + useDb("root", "test"); + + run("create qnode on dnode 1"); +} + +TEST_F(ParserInitialCTest, createSnode) { + useDb("root", "test"); + + run("create snode on dnode 1"); +} + +TEST_F(ParserInitialCTest, createStable) { + useDb("root", "test"); + + run("create stable t1(ts timestamp, c1 int) TAGS(id int)"); + + run("create stable if not exists test.t1(" + "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " + "SMALLINT, " + "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 NCHAR(30), " + "c15 VARCHAR(50)) " + "TAGS (tsa TIMESTAMP, a1 INT, a2 INT UNSIGNED, a3 BIGINT, a4 BIGINT UNSIGNED, a5 FLOAT, a6 DOUBLE, a7 " + "BINARY(20), a8 SMALLINT, " + "a9 SMALLINT UNSIGNED COMMENT 'test column comment', a10 TINYINT, a11 TINYINT UNSIGNED, a12 BOOL, a13 NCHAR(30), " + "a15 VARCHAR(50)) " + "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (min) FILE_FACTOR 0.1 DELAY 2"); +} + +TEST_F(ParserInitialCTest, createStream) { + useDb("root", "test"); + + run("create stream s1 as select * from t1"); + + run("create stream if not exists s1 as select * from t1"); + + run("create stream s1 into st1 as select * from t1"); + + run("create stream if not exists s1 trigger window_close watermark 10s into st1 as select * from t1"); +} + +TEST_F(ParserInitialCTest, createTable) { + useDb("root", "test"); + + run("create table t1(ts timestamp, c1 int)"); + + run("create table if not exists test.t1(" + "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " + "SMALLINT, " + "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 " + "NCHAR(30), " + "c15 VARCHAR(50)) " + "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3)"); + + run("create table if not exists test.t1(" + "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " + "SMALLINT, " + "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 " + "NCHAR(30), " + "c15 VARCHAR(50)) " + "TAGS (tsa TIMESTAMP, a1 INT, a2 INT UNSIGNED, a3 BIGINT, a4 BIGINT UNSIGNED, " + "a5 FLOAT, a6 DOUBLE, a7 " + "BINARY(20), a8 SMALLINT, " + "a9 SMALLINT UNSIGNED COMMENT 'test column comment', a10 " + "TINYINT, a11 TINYINT UNSIGNED, a12 BOOL, a13 NCHAR(30), " + "a15 VARCHAR(50)) " + "TTL 100 COMMENT 'test create " + "table' SMA(c1, c2, c3) ROLLUP (min) FILE_FACTOR 0.1 DELAY 2"); + + run("create table if not exists t1 using st1 tags(1, 'wxy')"); + + run("create table " + "if not exists test.t1 using test.st1 (tag1, tag2) tags(1, 'abc') " + "if not exists test.t2 using test.st1 (tag1, tag2) tags(2, 'abc') " + "if not exists test.t3 using test.st1 (tag1, tag2) tags(3, 'abc') "); +} + +TEST_F(ParserInitialCTest, createTopic) { + useDb("root", "test"); + + run("create topic tp1 as select * from t1"); + + run("create topic if not exists tp1 as select * from t1"); + + run("create topic tp1 as test"); + + run("create topic if not exists tp1 as test"); +} + +TEST_F(ParserInitialCTest, createUser) { + useDb("root", "test"); + + run("create user wxy pass '123456'"); +} + +} // namespace ParserTest diff --git a/source/libs/parser/test/parInitialDTest.cpp b/source/libs/parser/test/parInitialDTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1153b238b1feb1be8167c91acb0bf7f7267a391f --- /dev/null +++ b/source/libs/parser/test/parInitialDTest.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parTestUtil.h" + +using namespace std; + +namespace ParserTest { + +class ParserInitialDTest : public ParserTestBase {}; + +// todo delete +// todo desc +// todo describe +// todo drop account + +TEST_F(ParserInitialDTest, dropBnode) { + useDb("root", "test"); + + run("drop bnode on dnode 1"); +} + +// todo drop database +// todo drop dnode +// todo drop function + +TEST_F(ParserInitialDTest, dropIndex) { + useDb("root", "test"); + + run("drop index index1 on t1"); +} + +TEST_F(ParserInitialDTest, dropMnode) { + useDb("root", "test"); + + run("drop mnode on dnode 1"); +} + +TEST_F(ParserInitialDTest, dropQnode) { + useDb("root", "test"); + + run("drop qnode on dnode 1"); +} + +TEST_F(ParserInitialDTest, dropSnode) { + useDb("root", "test"); + + run("drop snode on dnode 1"); +} + +// todo drop stable +// todo drop stream +// todo drop table + +TEST_F(ParserInitialDTest, dropTopic) { + useDb("root", "test"); + + run("drop topic tp1"); + + run("drop topic if exists tp1"); +} + +TEST_F(ParserInitialDTest, dropUser) { + useDb("root", "test"); + + run("drop user wxy"); +} + +} // namespace ParserTest diff --git a/source/libs/parser/test/parInsertTest.cpp b/source/libs/parser/test/parInsertTest.cpp index 308152bf3ea12255ac41640b367a20c96dd50fa7..7fafec88824111ef8b170ba25f3b092fd7ba1f1a 100644 --- a/source/libs/parser/test/parInsertTest.cpp +++ b/source/libs/parser/test/parInsertTest.cpp @@ -111,6 +111,7 @@ class InsertTest : public Test { cxt_.pMsg = errMagBuf_; cxt_.msgLen = max_err_len; code_ = TSDB_CODE_SUCCESS; + res_ = nullptr; } SVnodeModifOpStmt* getVnodeModifStmt(SQuery* pQuery) { return (SVnodeModifOpStmt*)pQuery->pRoot; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5a385ba25e10677f77d88fb74cbd440b2cb21a8a --- /dev/null +++ b/source/libs/parser/test/parSelectTest.cpp @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parTestUtil.h" + +using namespace std; + +namespace ParserTest { + +class ParserSelectTest : public ParserTestBase {}; + +TEST_F(ParserSelectTest, basic) { + useDb("root", "test"); + + run("SELECT * FROM t1"); + + run("SELECT * FROM test.t1"); + + run("SELECT ts, c1 FROM t1"); + + run("SELECT ts, t.c1 FROM (SELECT * FROM t1) t"); + + run("SELECT * FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); +} + +TEST_F(ParserSelectTest, constant) { + useDb("root", "test"); + + run("SELECT 123, 20.4, 'abc', \"wxy\", timestamp '2022-02-09 17:30:20', true, false, 10s FROM t1"); + + run("SELECT 1234567890123456789012345678901234567890, 20.1234567890123456789012345678901234567890, 'abc', \"wxy\", " + "timestamp '2022-02-09 17:30:20', true, false, 15s FROM t1"); + + run("SELECT 123 + 45 FROM t1 WHERE 2 - 1"); +} + +TEST_F(ParserSelectTest, expression) { + useDb("root", "test"); + + run("SELECT ts + 10s, c1 + 10, concat(c2, 'abc') FROM t1"); + + run("SELECT ts > 0, c1 < 20 and c2 = 'qaz' FROM t1"); + + run("SELECT ts > 0, c1 between 10 and 20 and c2 = 'qaz' FROM t1"); +} + +TEST_F(ParserSelectTest, condition) { + useDb("root", "test"); + + run("SELECT c1 FROM t1 WHERE ts in (true, false)"); + + run("SELECT * FROM t1 WHERE c1 > 10 and c1 is not null"); +} + +TEST_F(ParserSelectTest, pseudoColumn) { + useDb("root", "test"); + + run("SELECT _WSTARTTS, _WENDTS, COUNT(*) FROM t1 INTERVAL(10s)"); +} + +TEST_F(ParserSelectTest, multiResFunc) { + useDb("root", "test"); + + run("SELECT LAST(*), FIRST(*), LAST_ROW(*) FROM t1"); + + run("SELECT LAST(c1, c2), FIRST(t1.*), LAST_ROW(c3) FROM t1"); + + run("SELECT LAST(t2.*), FIRST(t1.c1, t2.*), LAST_ROW(t1.*, t2.*) FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts"); +} + +TEST_F(ParserSelectTest, timelineFunc) { + useDb("root", "test"); + + run("SELECT LAST(*), FIRST(*) FROM t1"); + + run("SELECT FIRST(ts), FIRST(c1), FIRST(c2), FIRST(c3) FROM t1"); + + run("SELECT LAST(*), FIRST(*) FROM t1 GROUP BY c1"); + + run("SELECT LAST(*), FIRST(*) FROM t1 INTERVAL(10s)"); + + run("SELECT diff(c1) FROM t1"); +} + +TEST_F(ParserSelectTest, selectFunc) { + useDb("root", "test"); + + // select function + run("SELECT MAX(c1), MIN(c1) FROM t1"); + // select function for GROUP BY clause + run("SELECT MAX(c1), MIN(c1) FROM t1 GROUP BY c1"); + // select function for INTERVAL clause + run("SELECT MAX(c1), MIN(c1) FROM t1 INTERVAL(10s)"); + // select function along with the columns of select row + run("SELECT MAX(c1), c2 FROM t1"); + run("SELECT MAX(c1), t1.* FROM t1"); + // select function along with the columns of select row, and with GROUP BY clause + run("SELECT MAX(c1), c2 FROM t1 GROUP BY c3"); + run("SELECT MAX(c1), t1.* FROM t1 GROUP BY c3"); + // select function along with the columns of select row, and with window clause + run("SELECT MAX(c1), c2 FROM t1 INTERVAL(10s)"); + run("SELECT MAX(c1), c2 FROM t1 SESSION(ts, 10s)"); + run("SELECT MAX(c1), c2 FROM t1 STATE_WINDOW(c3)"); +} + +TEST_F(ParserSelectTest, clause) { + useDb("root", "test"); + + // GROUP BY clause + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0"); + + run("SELECT COUNT(*), c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2"); + + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 having COUNT(c1) > 10"); + + run("SELECT COUNT(*), c1, c2 + 10, c1 + c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2, c1"); + + run("SELECT COUNT(*), c1 + 10, c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c1 + 10, c2"); + + // order by clause + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 order by cnt"); + + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 order by 1"); + + // distinct clause + // run("SELECT distinct c1, c2 FROM t1 WHERE c1 > 0 order by c1"); + + // run("SELECT distinct c1 + 10, c2 FROM t1 WHERE c1 > 0 order by c1 + 10, c2"); + + // run("SELECT distinct c1 + 10 cc1, c2 cc2 FROM t1 WHERE c1 > 0 order by cc1, c2"); + + // run("SELECT distinct COUNT(c2) FROM t1 WHERE c1 > 0 GROUP BY c1 order by COUNT(c2)"); +} + +// INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [FILL(fill_mod_and_val)] +// fill_mod_and_val = { NONE | PREV | NULL | LINEAR | NEXT | value_mod } +// value_mod = VALUE , val ... +TEST_F(ParserSelectTest, interval) { + useDb("root", "test"); + // INTERVAL(interval_val) + run("SELECT COUNT(*) FROM t1 INTERVAL(10s)"); + // INTERVAL(interval_val, interval_offset) + run("SELECT COUNT(*) FROM t1 INTERVAL(10s, 5s)"); + // INTERVAL(interval_val, interval_offset) SLIDING (sliding_val) + run("SELECT COUNT(*) FROM t1 INTERVAL(10s, 5s) SLIDING(7s)"); + // INTERVAL(interval_val) FILL(NONE) + run("SELECT COUNT(*) FROM t1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " + "INTERVAL(10s) FILL(NONE)"); +} + +TEST_F(ParserSelectTest, intervalSemanticCheck) { + useDb("root", "test"); + + run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + run("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 3 INTERVAL(1d) FILL(NEXT)", TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, + PARSER_STAGE_TRANSLATE); +} + +TEST_F(ParserSelectTest, semanticError) { + useDb("root", "test"); + + // TSDB_CODE_PAR_INVALID_COLUMN + run("SELECT c1, cc1 FROM t1", TSDB_CODE_PAR_INVALID_COLUMN, PARSER_STAGE_TRANSLATE); + + run("SELECT t1.c1, t1.cc1 FROM t1", TSDB_CODE_PAR_INVALID_COLUMN, PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_TABLE_NOT_EXIST + run("SELECT * FROM t10", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); + + run("SELECT * FROM test.t10", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); + + run("SELECT t2.c1 FROM t1", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_AMBIGUOUS_COLUMN + run("SELECT c2 FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1", TSDB_CODE_PAR_AMBIGUOUS_COLUMN, PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_WRONG_VALUE_TYPE + run("SELECT timestamp '2010' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION + run("SELECT c2 FROM t1 tt1 join t1 tt2 on COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, + PARSER_STAGE_TRANSLATE); + + run("SELECT c2 FROM t1 WHERE COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); + + run("SELECT c2 FROM t1 GROUP BY COUNT(*)", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT + run("SELECT c2 FROM t1 order by 0", TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT, PARSER_STAGE_TRANSLATE); + + run("SELECT c2 FROM t1 order by 2", TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT, PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION + run("SELECT COUNT(*) cnt FROM t1 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); + + run("SELECT COUNT(*) cnt FROM t1 GROUP BY c2 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + PARSER_STAGE_TRANSLATE); + + run("SELECT COUNT(*), c1 cnt FROM t1 GROUP BY c2 having c2 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + PARSER_STAGE_TRANSLATE); + + run("SELECT COUNT(*) cnt FROM t1 GROUP BY c2 having c2 > 0 order by c1", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_NOT_SINGLE_GROUP + run("SELECT COUNT(*), c1 FROM t1", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + + run("SELECT COUNT(*) FROM t1 order by c1", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + + run("SELECT c1 FROM t1 order by COUNT(*)", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + + // TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION + run("SELECT distinct c1, c2 FROM t1 WHERE c1 > 0 order by ts", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + PARSER_STAGE_TRANSLATE); + + run("SELECT distinct c1 FROM t1 WHERE c1 > 0 order by COUNT(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + PARSER_STAGE_TRANSLATE); + + run("SELECT distinct c2 FROM t1 WHERE c1 > 0 order by COUNT(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + PARSER_STAGE_TRANSLATE); +} + +} // namespace ParserTest diff --git a/source/libs/parser/test/parShowToUse.cpp b/source/libs/parser/test/parShowToUse.cpp new file mode 100644 index 0000000000000000000000000000000000000000..37d5d87a88d2e91ecfddf70cd8bc23e17d891cba --- /dev/null +++ b/source/libs/parser/test/parShowToUse.cpp @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parTestUtil.h" + +using namespace std; + +namespace ParserTest { + +class ParserShowToUseTest : public ParserTestBase {}; + +// todo show accounts +// todo show apps +// todo show connections +// todo show create database +// todo show create stable +// todo show create table + +TEST_F(ParserShowToUseTest, showDatabases) { + useDb("root", "test"); + + run("show databases"); +} + +TEST_F(ParserShowToUseTest, showDnodes) { + useDb("root", "test"); + + run("show dnodes"); +} + +TEST_F(ParserShowToUseTest, showFunctions) { + useDb("root", "test"); + + run("show functions"); +} + +// todo show licence + +TEST_F(ParserShowToUseTest, showIndexes) { + useDb("root", "test"); + + run("show indexes from t1"); + + run("show indexes from t1 from test"); +} + +TEST_F(ParserShowToUseTest, showMnodes) { + useDb("root", "test"); + + run("show mnodes"); +} + +TEST_F(ParserShowToUseTest, showModules) { + useDb("root", "test"); + + run("show modules"); +} + +TEST_F(ParserShowToUseTest, showQnodes) { + useDb("root", "test"); + + run("show qnodes"); +} + +// todo show queries +// todo show scores + +TEST_F(ParserShowToUseTest, showStables) { + useDb("root", "test"); + + run("show stables"); + + run("show test.stables"); + + run("show stables like 'c%'"); + + run("show test.stables like 'c%'"); +} + +TEST_F(ParserShowToUseTest, showStreams) { + useDb("root", "test"); + + run("show streams"); +} + +TEST_F(ParserShowToUseTest, showTables) { + useDb("root", "test"); + + run("show tables"); + + run("show test.tables"); + + run("show tables like 'c%'"); + + run("show test.tables like 'c%'"); +} + +// todo show topics + +TEST_F(ParserShowToUseTest, showUsers) { + useDb("root", "test"); + + run("show users"); +} + +// todo show variables + +TEST_F(ParserShowToUseTest, showVgroups) { + useDb("root", "test"); + + run("show vgroups"); + + run("show test.vgroups"); +} + +// todo show vnodes + +// todo split vgroup + +TEST_F(ParserShowToUseTest, useDatabase) { + useDb("root", "test"); + + run("use wxy_db"); +} + +} // namespace ParserTest diff --git a/source/libs/parser/test/parTestMain.cpp b/source/libs/parser/test/parTestMain.cpp index 28312d80a8fb22bc00f29b100ae19d8cf7ac7966..ebc83fb21981e56666b82ec6a5a08a63cd7f0c87 100644 --- a/source/libs/parser/test/parTestMain.cpp +++ b/source/libs/parser/test/parTestMain.cpp @@ -30,6 +30,8 @@ #include "parTestUtil.h" #include "parToken.h" +namespace ParserTest { + class ParserEnv : public testing::Environment { public: virtual void SetUp() { @@ -62,9 +64,11 @@ static void parseArg(int argc, char* argv[]) { } } +} // namespace ParserTest + int main(int argc, char* argv[]) { - testing::AddGlobalTestEnvironment(new ParserEnv()); + testing::AddGlobalTestEnvironment(new ParserTest::ParserEnv()); testing::InitGoogleTest(&argc, argv); - parseArg(argc, argv); + ParserTest::parseArg(argc, argv); return RUN_ALL_TESTS(); } diff --git a/source/libs/parser/test/parTestUtil.cpp b/source/libs/parser/test/parTestUtil.cpp index 8e7123ee95f102beeadc7c9b0a6958b94cbf2b8a..a9ff756a95699e8e8f4fb89c0bf734976fa75bf3 100644 --- a/source/libs/parser/test/parTestUtil.cpp +++ b/source/libs/parser/test/parTestUtil.cpp @@ -23,17 +23,30 @@ using namespace std; using namespace testing; -#define DO_WITH_THROW(func, ...) \ - do { \ - int32_t code__ = func(__VA_ARGS__); \ - if (TSDB_CODE_SUCCESS != code__) { \ - throw runtime_error("sql:[" + stmtEnv_.sql_ + "] " #func " code:" + to_string(code__) + \ - ", strerror:" + string(tstrerror(code__)) + ", msg:" + string(stmtEnv_.msgBuf_.data())); \ - } \ +namespace ParserTest { + +#define DO_WITH_THROW(func, ...) \ + do { \ + int32_t code__ = func(__VA_ARGS__); \ + if (!checkResultCode(#func, code__)) { \ + if (TSDB_CODE_SUCCESS != code__) { \ + throw runtime_error("sql:[" + stmtEnv_.sql_ + "] " #func " code:" + to_string(code__) + \ + ", strerror:" + string(tstrerror(code__)) + ", msg:" + string(stmtEnv_.msgBuf_.data())); \ + } else { \ + throw runtime_error("sql:[" + stmtEnv_.sql_ + "] " #func " expect " + to_string(stmtEnv_.expect_) + \ + " actual " + to_string(code__)); \ + } \ + } else if (TSDB_CODE_SUCCESS != code__) { \ + throw TerminateFlag(); \ + } \ } while (0); bool g_isDump = false; +struct TerminateFlag : public exception { + const char* what() const throw() { return "success and terminate"; } +}; + class ParserTestBaseImpl { public: void useDb(const string& acctId, const string& db) { @@ -41,8 +54,8 @@ class ParserTestBaseImpl { caseEnv_.db_ = db; } - void run(const string& sql) { - reset(); + void run(const string& sql, int32_t expect, ParserStage checkStage) { + reset(expect, checkStage); try { SParseContext cxt = {0}; setParseContext(sql, &cxt); @@ -57,6 +70,9 @@ class ParserTestBaseImpl { if (g_isDump) { dump(); } + } catch (const TerminateFlag& e) { + // success and terminate + return; } catch (...) { dump(); throw; @@ -72,6 +88,8 @@ class ParserTestBaseImpl { struct stmtEnv { string sql_; array msgBuf_; + int32_t expect_; + string checkFunc_; }; struct stmtRes { @@ -80,9 +98,34 @@ class ParserTestBaseImpl { string calcConstAst_; }; - void reset() { + bool checkResultCode(const string& pFunc, int32_t resultCode) { + return !(stmtEnv_.checkFunc_.empty()) + ? (("*" == stmtEnv_.checkFunc_ || stmtEnv_.checkFunc_ == pFunc) ? stmtEnv_.expect_ == resultCode + : TSDB_CODE_SUCCESS == resultCode) + : true; + } + + string stageFunc(ParserStage stage) { + switch (stage) { + case PARSER_STAGE_PARSE: + return "parse"; + case PARSER_STAGE_TRANSLATE: + return "translate"; + case PARSER_STAGE_CALC_CONST: + return "calculateConstant"; + case PARSER_STAGE_ALL: + return "*"; + default: + break; + } + return "unknown"; + } + + void reset(int32_t expect, ParserStage checkStage) { stmtEnv_.sql_.clear(); stmtEnv_.msgBuf_.fill(0); + stmtEnv_.expect_ = expect; + stmtEnv_.checkFunc_ = stageFunc(checkStage); res_.parsedAst_.clear(); res_.translatedAst_.clear(); @@ -146,4 +189,8 @@ ParserTestBase::~ParserTestBase() {} void ParserTestBase::useDb(const std::string& acctId, const std::string& db) { impl_->useDb(acctId, db); } -void ParserTestBase::run(const std::string& sql) { return impl_->run(sql); } +void ParserTestBase::run(const std::string& sql, int32_t expect, ParserStage checkStage) { + return impl_->run(sql, expect, checkStage); +} + +} // namespace ParserTest diff --git a/source/libs/parser/test/parTestUtil.h b/source/libs/parser/test/parTestUtil.h index de3b16aabc6f020b131abf02bcb00cb788fc7a2a..0e8703b2c82fa85772e5bf8650ef1b3be78dd3f3 100644 --- a/source/libs/parser/test/parTestUtil.h +++ b/source/libs/parser/test/parTestUtil.h @@ -18,15 +18,21 @@ #include +#include "taoserror.h" + +namespace ParserTest { + class ParserTestBaseImpl; +enum ParserStage { PARSER_STAGE_PARSE, PARSER_STAGE_TRANSLATE, PARSER_STAGE_CALC_CONST, PARSER_STAGE_ALL }; + class ParserTestBase : public testing::Test { public: ParserTestBase(); virtual ~ParserTestBase(); void useDb(const std::string& acctId, const std::string& db); - void run(const std::string& sql); + void run(const std::string& sql, int32_t expect = TSDB_CODE_SUCCESS, ParserStage checkStage = PARSER_STAGE_ALL); private: std::unique_ptr impl_; @@ -34,4 +40,6 @@ class ParserTestBase : public testing::Test { extern bool g_isDump; +} // namespace ParserTest + #endif // PARSER_TEST_UTIL_H diff --git a/source/libs/parser/test/parserAstTest.cpp b/source/libs/parser/test/parserAstTest.cpp deleted file mode 100644 index 3b00e696ccdb4ebd982d7d61ebd102e426b10547..0000000000000000000000000000000000000000 --- a/source/libs/parser/test/parserAstTest.cpp +++ /dev/null @@ -1,782 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include - -#include - -#include "parInt.h" -#include "parTestUtil.h" - -using namespace std; -using namespace testing; - -class ParserTest : public Test { - protected: - void setDatabase(const string& acctId, const string& db) { - acctId_ = acctId; - db_ = db; - } - - void bind(const char* sql) { - reset(); - cxt_.acctId = atoi(acctId_.c_str()); - cxt_.db = db_.c_str(); - sqlBuf_ = string(sql); - transform(sqlBuf_.begin(), sqlBuf_.end(), sqlBuf_.begin(), ::tolower); - cxt_.sqlLen = strlen(sql); - cxt_.pSql = sqlBuf_.c_str(); - } - - bool run(int32_t parseCode = TSDB_CODE_SUCCESS, int32_t translateCode = TSDB_CODE_SUCCESS) { - query_ = nullptr; - bool res = runImpl(parseCode, translateCode); - qDestroyQuery(query_); - if (!res || g_isDump) { - dump(); - } - return res; - } - - private: - static const int max_err_len = 1024; - - bool runImpl(int32_t parseCode, int32_t translateCode) { - int32_t code = parse(&cxt_, &query_); - if (code != TSDB_CODE_SUCCESS) { - parseErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; - return (code == parseCode); - } - if (TSDB_CODE_SUCCESS != parseCode) { - return false; - } - parsedAstStr_ = toString(query_->pRoot); - code = translate(&cxt_, query_); - if (code != TSDB_CODE_SUCCESS) { - translateErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; - return (code == translateCode); - } - translatedAstStr_ = toString(query_->pRoot); - code = calculateConstant(&cxt_, query_); - if (code != TSDB_CODE_SUCCESS) { - calcConstErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; - return false; - } - calcConstAstStr_ = toString(query_->pRoot); - return (TSDB_CODE_SUCCESS == translateCode); - } - - void dump() { - cout << "input sql : [" << cxt_.pSql << "]" << endl; - if (!parseErrStr_.empty()) { - cout << "parse error: " << parseErrStr_ << endl; - } - if (!parsedAstStr_.empty()) { - cout << "parse output: " << endl; - cout << parsedAstStr_ << endl; - } - if (!translateErrStr_.empty()) { - cout << "translate error: " << translateErrStr_ << endl; - } - if (!translatedAstStr_.empty()) { - cout << "translate output: " << endl; - cout << translatedAstStr_ << endl; - } - if (!calcConstErrStr_.empty()) { - cout << "calculateConstant error: " << calcConstErrStr_ << endl; - } - if (!calcConstAstStr_.empty()) { - cout << "calculateConstant output: " << endl; - cout << calcConstAstStr_ << endl; - } - } - - string toString(const SNode* pRoot, bool format = false) { - char* pStr = NULL; - int32_t len = 0; - int32_t code = nodesNodeToString(pRoot, format, &pStr, &len); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] toString code:" << code << ", strerror:" << tstrerror(code) << endl; - throw "nodesNodeToString failed!"; - } - string str(pStr); - taosMemoryFreeClear(pStr); - return str; - } - - void reset() { - memset(&cxt_, 0, sizeof(cxt_)); - memset(errMagBuf_, 0, max_err_len); - cxt_.pMsg = errMagBuf_; - cxt_.msgLen = max_err_len; - parseErrStr_.clear(); - parsedAstStr_.clear(); - translateErrStr_.clear(); - translatedAstStr_.clear(); - calcConstErrStr_.clear(); - calcConstAstStr_.clear(); - } - - string acctId_; - string db_; - char errMagBuf_[max_err_len]; - string sqlBuf_; - SParseContext cxt_; - SQuery* query_; - string parseErrStr_; - string parsedAstStr_; - string translateErrStr_; - string translatedAstStr_; - string calcConstErrStr_; - string calcConstAstStr_; -}; - -TEST_F(ParserTest, createAccount) { - setDatabase("root", "test"); - - bind("create account ac_wxy pass '123456'"); - ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT)); -} - -TEST_F(ParserTest, alterAccount) { - setDatabase("root", "test"); - - bind("alter account ac_wxy pass '123456'"); - ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT)); -} - -TEST_F(ParserTest, createUser) { - setDatabase("root", "test"); - - bind("create user wxy pass '123456'"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, alterUser) { - setDatabase("root", "test"); - - bind("alter user wxy pass '123456'"); - ASSERT_TRUE(run()); - - bind("alter user wxy privilege 'write'"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, dropUser) { - setDatabase("root", "test"); - - bind("drop user wxy"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectSimple) { - setDatabase("root", "test"); - - bind("SELECT * FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM test.t1"); - ASSERT_TRUE(run()); - - bind("SELECT ts, c1 FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT ts, t.c1 FROM (SELECT * FROM t1) t"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectConstant) { - setDatabase("root", "test"); - - bind("SELECT 123, 20.4, 'abc', \"wxy\", TIMESTAMP '2022-02-09 17:30:20', true, false, 10s FROM t1"); - ASSERT_TRUE(run()); - - bind( - "SELECT 1234567890123456789012345678901234567890, 20.1234567890123456789012345678901234567890, 'abc', \"wxy\", " - "TIMESTAMP '2022-02-09 17:30:20', true, false, 15s FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT 123 + 45 FROM t1 where 2 - 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectExpression) { - setDatabase("root", "test"); - - bind("SELECT ts + 10s, c1 + 10, concat(c2, 'abc') FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT ts > 0, c1 < 20 AND c2 = 'qaz' FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT ts > 0, c1 BETWEEN 10 AND 20 AND c2 = 'qaz' FROM t1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectCondition) { - setDatabase("root", "test"); - - bind("SELECT c1 FROM t1 where ts in (true, false)"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 where c1 > 10 and c1 is not null"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectPseudoColumn) { - setDatabase("root", "test"); - - bind("SELECT _wstartts, _wendts, count(*) FROM t1 interval(10s)"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectMultiResFunc) { - setDatabase("root", "test"); - - // bind("SELECT last(*), first(*), last_row(*) FROM t1"); - // ASSERT_TRUE(run()); - - bind("SELECT last(c1, c2), first(t1.*), last_row(c3) FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT last(t2.*), first(t1.c1, t2.*), last_row(t1.*, t2.*) FROM st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectClause) { - setDatabase("root", "test"); - - // GROUP BY clause - bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0"); - ASSERT_TRUE(run()); - - bind("SELECT count(*), c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2"); - ASSERT_TRUE(run()); - - bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 HAVING count(c1) > 10"); - ASSERT_TRUE(run()); - - bind("SELECT count(*), c1, c2 + 10, c1 + c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2, c1"); - ASSERT_TRUE(run()); - - bind("SELECT count(*), c1 + 10, c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c1 + 10, c2"); - ASSERT_TRUE(run()); - - // ORDER BY clause - bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 ORDER BY cnt"); - ASSERT_TRUE(run()); - - bind("SELECT count(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 ORDER BY 1"); - ASSERT_TRUE(run()); - - // DISTINCT clause - bind("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 0 ORDER BY c1"); - ASSERT_TRUE(run()); - - bind("SELECT DISTINCT c1 + 10, c2 FROM t1 WHERE c1 > 0 ORDER BY c1 + 10, c2"); - ASSERT_TRUE(run()); - - bind("SELECT DISTINCT c1 + 10 cc1, c2 cc2 FROM t1 WHERE c1 > 0 ORDER BY cc1, c2"); - ASSERT_TRUE(run()); - - bind("SELECT DISTINCT count(c2) FROM t1 WHERE c1 > 0 GROUP BY c1 ORDER BY count(c2)"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectWindow) { - setDatabase("root", "test"); - - bind("SELECT count(*) FROM t1 interval(10s)"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, selectSyntaxError) { - setDatabase("root", "test"); - - bind("SELECTT * FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_FAILED)); - - bind("SELECT *"); - ASSERT_TRUE(run(TSDB_CODE_FAILED)); - - bind("SELECT *, * FROM test.t1"); - ASSERT_TRUE(run(TSDB_CODE_FAILED)); - - bind("SELECT * FROM test.t1 t WHER"); - ASSERT_TRUE(run(TSDB_CODE_FAILED)); -} - -TEST_F(ParserTest, selectSemanticError) { - setDatabase("root", "test"); - - // TSDB_CODE_PAR_INVALID_COLUMN - bind("SELECT c1, cc1 FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_INVALID_COLUMN)); - - bind("SELECT t1.c1, t1.cc1 FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_INVALID_COLUMN)); - - // TSDB_CODE_PAR_TABLE_NOT_EXIST - bind("SELECT * FROM t10"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST)); - - bind("SELECT * FROM test.t10"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST)); - - bind("SELECT t2.c1 FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_TABLE_NOT_EXIST)); - - // TSDB_CODE_PAR_AMBIGUOUS_COLUMN - bind("SELECT c2 FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_AMBIGUOUS_COLUMN)); - - // TSDB_CODE_PAR_WRONG_VALUE_TYPE - bind("SELECT 10n FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_VALUE_TYPE)); - - bind("SELECT TIMESTAMP '2010' FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_VALUE_TYPE)); - - // TSDB_CODE_PAR_INVALID_FUNTION - bind("SELECT cnt(*) FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FUNC_INVALID_FUNTION)); - - // TSDB_CODE_PAR_FUNTION_PARA_NUM - // TSDB_CODE_PAR_FUNTION_PARA_TYPE - - // TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION - bind("SELECT c2 FROM t1 tt1 JOIN t1 tt2 ON count(*) > 0"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION)); - - bind("SELECT c2 FROM t1 where count(*) > 0"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION)); - - bind("SELECT c2 FROM t1 GROUP BY count(*)"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION)); - - // TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT - bind("SELECT c2 FROM t1 ORDER BY 0"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT)); - - bind("SELECT c2 FROM t1 ORDER BY 2"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT)); - - // TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION - bind("SELECT count(*) cnt FROM t1 HAVING c1 > 0"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); - - bind("SELECT count(*) cnt FROM t1 GROUP BY c2 HAVING c1 > 0"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); - - bind("SELECT count(*), c1 cnt FROM t1 GROUP BY c2 HAVING c2 > 0"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); - - bind("SELECT count(*) cnt FROM t1 GROUP BY c2 HAVING c2 > 0 ORDER BY c1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION)); - - // TSDB_CODE_PAR_NOT_SINGLE_GROUP - bind("SELECT count(*), c1 FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); - - bind("SELECT count(*) FROM t1 ORDER BY c1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); - - bind("SELECT c1 FROM t1 ORDER BY count(*)"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SINGLE_GROUP)); - - // TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION - bind("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 0 ORDER BY ts"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); - - bind("SELECT DISTINCT c1 FROM t1 WHERE c1 > 0 ORDER BY count(c2)"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); - - bind("SELECT DISTINCT c2 FROM t1 WHERE c1 > 0 ORDER BY count(c2)"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); -} - -TEST_F(ParserTest, showUsers) { - setDatabase("root", "test"); - - bind("show users"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createDnode) { - setDatabase("root", "test"); - - bind("create dnode abc1 port 7000"); - ASSERT_TRUE(run()); - - bind("create dnode 1.1.1.1 port 9000"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showDnodes) { - setDatabase("root", "test"); - - bind("show dnodes"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, alterDnode) { - setDatabase("root", "test"); - - bind("alter dnode 1 'resetLog'"); - ASSERT_TRUE(run()); - - bind("alter dnode 1 'debugFlag' '134'"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createDatabase) { - setDatabase("root", "test"); - - bind("create database wxy_db"); - ASSERT_TRUE(run()); - - bind( - "create database if not exists wxy_db " - "BLOCKS 100 " - "CACHE 100 " - "CACHELAST 2 " - "COMP 1 " - "DAYS 100 " - "FSYNC 100 " - "MAXROWS 1000 " - "MINROWS 100 " - "KEEP 100 " - "PRECISION 'ms' " - "QUORUM 1 " - "REPLICA 3 " - "TTL 100 " - "WAL 2 " - "VGROUPS 100 " - "SINGLE_STABLE 0 " - "STREAM_MODE 1 " - "RETENTIONS '15s:7d,1m:21d,15m:5y'"); - ASSERT_TRUE(run()); - - bind( - "create database if not exists wxy_db " - "DAYS 100m " - "KEEP 200m,300h,400d "); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, alterDatabase) { - setDatabase("root", "test"); - - bind("alter database wxy_db BLOCKS 200"); - ASSERT_TRUE(run()); - - bind( - "alter database wxy_db " - "BLOCKS 200 " - "CACHELAST 1 " - "FSYNC 200 " - "KEEP 200 " - "QUORUM 2 " - "WAL 1 "); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showDatabase) { - setDatabase("root", "test"); - - bind("show databases"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, useDatabase) { - setDatabase("root", "test"); - - bind("use wxy_db"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createTable) { - setDatabase("root", "test"); - - bind("create table t1(ts timestamp, c1 int)"); - ASSERT_TRUE(run()); - - bind( - "create table if not exists test.t1(" - "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " - "SMALLINT, " - "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 NCHAR(30), " - "c14 JSON, c15 VARCHAR(50)) " - "KEEP 100 TTL 100 COMMENT 'test create table' SMA(c1, c2, c3)"); - ASSERT_TRUE(run()); - - bind( - "create table if not exists test.t1(" - "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " - "SMALLINT, " - "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 NCHAR(30), " - "c14 JSON, c15 VARCHAR(50)) " - "TAGS (tsa TIMESTAMP, a1 INT, a2 INT UNSIGNED, a3 BIGINT, a4 BIGINT UNSIGNED, a5 FLOAT, a6 DOUBLE, a7 " - "BINARY(20), a8 SMALLINT, " - "a9 SMALLINT UNSIGNED COMMENT 'test column comment', a10 TINYINT, a11 TINYINT UNSIGNED, a12 BOOL, a13 NCHAR(30), " - "a14 JSON, a15 VARCHAR(50)) " - "KEEP 100 TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (min) FILE_FACTOR 0.1 DELAY 2"); - ASSERT_TRUE(run()); - - bind("create table if not exists t1 using st1 tags(1, 'wxy')"); - ASSERT_TRUE(run()); - - bind( - "create table " - "if not exists test.t1 using test.st1 (tag1, tag2) tags(1, 'abc') " - "if not exists test.t2 using test.st1 (tag1, tag2) tags(2, 'abc') " - "if not exists test.t3 using test.st1 (tag1, tag2) tags(3, 'abc') " - "if not exists test.t4 using test.st1 (tag1, tag2) tags(3, null) " - "if not exists test.t5 using test.st1 (tag1, tag2) tags(null, 'abc') " - "if not exists test.t6 using test.st1 (tag1, tag2) tags(null, null)"); - ASSERT_TRUE(run()); - - bind("create stable t1(ts timestamp, c1 int) TAGS(id int)"); - ASSERT_TRUE(run()); - - bind( - "create stable if not exists test.t1(" - "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " - "SMALLINT, " - "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 NCHAR(30), " - "c14 JSON, c15 VARCHAR(50)) " - "TAGS (tsa TIMESTAMP, a1 INT, a2 INT UNSIGNED, a3 BIGINT, a4 BIGINT UNSIGNED, a5 FLOAT, a6 DOUBLE, a7 " - "BINARY(20), a8 SMALLINT, " - "a9 SMALLINT UNSIGNED COMMENT 'test column comment', a10 TINYINT, a11 TINYINT UNSIGNED, a12 BOOL, a13 NCHAR(30), " - "a14 JSON, a15 VARCHAR(50)) " - "KEEP 100 TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (min) FILE_FACTOR 0.1 DELAY 2"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showTables) { - setDatabase("root", "test"); - - bind("show tables"); - ASSERT_TRUE(run()); - - bind("show test.tables"); - ASSERT_TRUE(run()); - - bind("show tables like 'c%'"); - ASSERT_TRUE(run()); - - bind("show test.tables like 'c%'"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showStables) { - setDatabase("root", "test"); - - bind("show stables"); - ASSERT_TRUE(run()); - - bind("show test.stables"); - ASSERT_TRUE(run()); - - bind("show stables like 'c%'"); - ASSERT_TRUE(run()); - - bind("show test.stables like 'c%'"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showVgroups) { - setDatabase("root", "test"); - - bind("show vgroups"); - ASSERT_TRUE(run()); - - bind("show test.vgroups"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showMnodes) { - setDatabase("root", "test"); - - bind("show mnodes"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showModules) { - setDatabase("root", "test"); - - bind("show modules"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showQnodes) { - setDatabase("root", "test"); - - bind("show qnodes"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showFunctions) { - setDatabase("root", "test"); - - bind("show functions"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showIndexes) { - setDatabase("root", "test"); - - bind("show indexes from t1"); - ASSERT_TRUE(run()); - - bind("show indexes from t1 from test"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, showStreams) { - setDatabase("root", "test"); - - bind("show streams"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createSmaIndex) { - setDatabase("root", "test"); - - bind("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) INTERVAL(10s)"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, dropIndex) { - setDatabase("root", "test"); - - bind("drop index index1 on t1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createQnode) { - setDatabase("root", "test"); - - bind("create qnode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, dropQnode) { - setDatabase("root", "test"); - - bind("drop qnode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createBnode) { - setDatabase("root", "test"); - - bind("create bnode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, dropBnode) { - setDatabase("root", "test"); - - bind("drop bnode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createSnode) { - setDatabase("root", "test"); - - bind("create snode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, dropSnode) { - setDatabase("root", "test"); - - bind("drop snode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createMnode) { - setDatabase("root", "test"); - - bind("create mnode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, dropMnode) { - setDatabase("root", "test"); - - bind("drop mnode on dnode 1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createTopic) { - setDatabase("root", "test"); - - bind("create topic tp1 as select * from t1"); - ASSERT_TRUE(run()); - - bind("create topic if not exists tp1 as select * from t1"); - ASSERT_TRUE(run()); - - bind("create topic tp1 as test"); - ASSERT_TRUE(run()); - - bind("create topic if not exists tp1 as test"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, dropTopic) { - setDatabase("root", "test"); - - bind("drop topic tp1"); - ASSERT_TRUE(run()); - - bind("drop topic if exists tp1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, createStream) { - setDatabase("root", "test"); - - bind("create stream s1 as select * from t1"); - ASSERT_TRUE(run()); - - bind("create stream if not exists s1 as select * from t1"); - ASSERT_TRUE(run()); - - bind("create stream s1 into st1 as select * from t1"); - ASSERT_TRUE(run()); - - bind("create stream if not exists s1 trigger window_close watermark 10s into st1 as select * from t1"); - ASSERT_TRUE(run()); -} - -TEST_F(ParserTest, explain) { - setDatabase("root", "test"); - - bind("explain SELECT * FROM t1"); - ASSERT_TRUE(run()); - - bind("explain analyze SELECT * FROM t1"); - ASSERT_TRUE(run()); - - bind("explain analyze verbose true ratio 0.01 SELECT * FROM t1"); - ASSERT_TRUE(run()); -} diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index e6534f710db03b1f3bf36bab7ead2d281faceee8..4640ed99bd979821155fe3a544c18848503b33ef 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "planner.h" +#include "taoserror.h" #define QUERY_POLICY_VNODE 1 #define QUERY_POLICY_HYBRID 2 @@ -33,6 +34,8 @@ extern "C" { #define planDebug(param, ...) qDebug("PLAN: " param, __VA_ARGS__) #define planTrace(param, ...) qTrace("PLAN: " param, __VA_ARGS__) +int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...); + int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode); int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode); int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 18e59859accdcc9cb2efc0df20842ef43a5844f5..3e19ccbd82b8004849751fe941a81c9c2127f037 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -274,7 +274,12 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } if (TSDB_CODE_SUCCESS == code) { - code = nodesCollectFuncs(pSelect, fmIsScanPseudoColumnFunc, &pScan->pScanPseudoCols); + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_FROM, fmIsScanPseudoColumnFunc, &pScan->pScanPseudoCols); + } + + // rewrite the expression in subsequent clauses + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExprForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM); } pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->pMeta); @@ -440,7 +445,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { - code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pAgg->pAggFuncs); + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); } // rewrite the expression in subsequent clauses @@ -474,7 +479,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow, SLogicNode** pLogicNode) { - int32_t code = nodesCollectFuncs(pSelect, fmIsWindowClauseFunc, &pWindow->pFuncs); + int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_WINDOW, fmIsWindowClauseFunc, &pWindow->pFuncs); if (pCxt->pPlanCxt->streamQuery) { pWindow->triggerType = pCxt->pPlanCxt->triggerType; @@ -559,14 +564,6 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva return TSDB_CODE_OUT_OF_MEMORY; } - if (NULL != pInterval->pFill) { - pWindow->pFill = nodesCloneNode(pInterval->pFill); - if (NULL == pWindow->pFill) { - nodesDestroyNode(pWindow); - return TSDB_CODE_OUT_OF_MEMORY; - } - } - return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); } @@ -589,6 +586,38 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele return TSDB_CODE_FAILED; } +static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { + if (NULL == pSelect->pWindow || QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow) || + NULL == ((SIntervalWindowNode*)pSelect->pWindow)->pFill) { + return TSDB_CODE_SUCCESS; + } + + SFillNode* pFillNode = (SFillNode*)(((SIntervalWindowNode*)pSelect->pWindow)->pFill); + + SFillLogicNode* pFill = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_FILL); + if (NULL == pFill) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_WINDOW, NULL, COLLECT_COL_TYPE_ALL, &pFill->node.pTargets); + + pFill->mode = pFillNode->mode; + pFill->timeRange = pFillNode->timeRange; + pFill->pValues = nodesCloneNode(pFillNode->pValues); + pFill->pWStartTs = nodesCloneNode(pFillNode->pWStartTs); + if ((NULL != pFillNode->pValues && NULL == pFill->pValues) || NULL == pFill->pWStartTs) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pFill; + } else { + nodesDestroyNode(pFill); + } + + return code; +} + static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (NULL == pSelect->pOrderByList) { return TSDB_CODE_SUCCESS; @@ -747,11 +776,14 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { SLogicNode* pRoot = NULL; int32_t code = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable, &pRoot); + if (TSDB_CODE_SUCCESS == code) { + code = createChildLogicNode(pCxt, pSelect, createPartitionLogicNode, &pRoot); + } if (TSDB_CODE_SUCCESS == code) { code = createChildLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot); } if (TSDB_CODE_SUCCESS == code) { - code = createChildLogicNode(pCxt, pSelect, createPartitionLogicNode, &pRoot); + code = createChildLogicNode(pCxt, pSelect, createFillLogicNode, &pRoot); } if (TSDB_CODE_SUCCESS == code) { code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot); @@ -944,9 +976,16 @@ static int32_t createSetOperatorLogicNode(SLogicPlanContext* pCxt, SSetOperator* } static int32_t getMsgType(ENodeType sqlType) { - return (QUERY_NODE_CREATE_TABLE_STMT == sqlType || QUERY_NODE_CREATE_MULTI_TABLE_STMT == sqlType) - ? TDMT_VND_CREATE_TABLE - : TDMT_VND_SUBMIT; + switch (sqlType) { + case QUERY_NODE_CREATE_TABLE_STMT: + case QUERY_NODE_CREATE_MULTI_TABLE_STMT: + return TDMT_VND_CREATE_TABLE; + case QUERY_NODE_DROP_TABLE_STMT: + return TDMT_VND_DROP_TABLE; + default: + break; + } + return TDMT_VND_SUBMIT; } static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt, SLogicNode** pLogicNode) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 042b914927732edb7b8214f2976a230c1f883e8f..dbce9abf36c64739285480dc3ebe2cd0bab35e08 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -123,14 +123,40 @@ static SNodeList* osdGetAllFuncs(SLogicNode* pNode) { return NULL; } +static bool needOptimizeDataRequire(const SFunctionNode* pFunc) { + if (!fmIsSpecialDataRequiredFunc(pFunc->funcId)) { + return false; + } + SNode* pPara = NULL; + FOREACH(pPara, pFunc->pParameterList) { + if (QUERY_NODE_COLUMN != nodeType(pPara) && QUERY_NODE_VALUE != nodeType(pPara)) { + return false; + } + } + return true; +} + +static bool needOptimizeDynamicScan(const SFunctionNode* pFunc) { + if (!fmIsDynamicScanOptimizedFunc(pFunc->funcId)) { + return false; + } + SNode* pPara = NULL; + FOREACH(pPara, pFunc->pParameterList) { + if (QUERY_NODE_COLUMN != nodeType(pPara) && QUERY_NODE_VALUE != nodeType(pPara)) { + return false; + } + } + return true; +} + static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) { SNodeList* pAllFuncs = osdGetAllFuncs(pScan->node.pParent); SNode* pFunc = NULL; FOREACH(pFunc, pAllFuncs) { int32_t code = TSDB_CODE_SUCCESS; - if (fmIsSpecialDataRequiredFunc(((SFunctionNode*)pFunc)->funcId)) { + if (needOptimizeDataRequire((SFunctionNode*)pFunc)) { code = nodesListMakeStrictAppend(pSdrFuncs, nodesCloneNode(pFunc)); - } else if (fmIsDynamicScanOptimizedFunc(((SFunctionNode*)pFunc)->funcId)) { + } else if (needOptimizeDynamicScan((SFunctionNode*)pFunc)) { code = nodesListMakeStrictAppend(pDsoFuncs, nodesCloneNode(pFunc)); } if (TSDB_CODE_SUCCESS != code) { @@ -206,6 +232,8 @@ static int32_t cpdMergeCond(SNode** pDst, SNode** pSrc) { if (NULL == pLogicCond) { return TSDB_CODE_OUT_OF_MEMORY; } + pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL; + pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pLogicCond->condType = LOGIC_COND_TYPE_AND; int32_t code = nodesListMakeAppend(&pLogicCond->pParameterList, *pSrc); if (TSDB_CODE_SUCCESS == code) { @@ -520,6 +548,7 @@ static int32_t cpdPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SN default: break; } + planError("cpdPushCondToChild failed, invalid logic plan node %s", nodesNodeName(nodeType(pChild))); return TSDB_CODE_PLAN_INTERNAL_ERROR; } @@ -538,9 +567,14 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { if (QUERY_NODE_OPERATOR != nodeType(pCond)) { return false; } - SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; - SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; + SOperatorNode* pOper = (SOperatorNode*)pJoin->pOnConditions; + if (OP_TYPE_EQUAL != pOper->opType) { + return false; + } + + SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; + SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; if (cpdIsPrimaryKey(pOper->pLeft, pLeftCols)) { return cpdIsPrimaryKey(pOper->pRight, pRightCols); } else if (cpdIsPrimaryKey(pOper->pLeft, pRightCols)) { @@ -551,22 +585,19 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { static int32_t cpdCheckOpCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode* pOnCond) { if (!cpdIsPrimaryKeyEqualCond(pJoin, pOnCond)) { - snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression"); - return TSDB_CODE_FAILED; + return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } return TSDB_CODE_SUCCESS; } static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SLogicConditionNode* pOnCond) { if (LOGIC_COND_TYPE_AND != pOnCond->condType) { - snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression"); - return TSDB_CODE_FAILED; + return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } SNode* pCond = NULL; FOREACH(pCond, pOnCond->pParameterList) { if (!cpdIsPrimaryKeyEqualCond(pJoin, pCond)) { - snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression"); - return TSDB_CODE_FAILED; + return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } } return TSDB_CODE_SUCCESS; @@ -574,8 +605,7 @@ static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { - snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "not support cross join"); - return TSDB_CODE_FAILED; + return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN); } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions)) { return cpdCheckLogicCond(pCxt, pJoin, (SLogicConditionNode*)pJoin->pOnConditions); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 548957ab8e2db71da3ef0b1644afd760ef6eb8d4..05b48d2d6a0919904c07aad87463cde1ffe6c8e6 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -17,6 +17,7 @@ #include "catalog.h" #include "functionMgt.h" +#include "systable.h" #include "tglobal.h" typedef struct SSlotIdInfo { @@ -271,6 +272,7 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { } // pIndex is definitely not NULL, otherwise it is a bug if (NULL == pIndex) { + planError("doSetSlotId failed, invalid slot name %s", name); pCxt->errCode = TSDB_CODE_PLAN_INTERNAL_ERROR; return DEAL_RES_ERROR; } @@ -434,12 +436,15 @@ static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAdd pNodeAddr->epSet = vg->epSet; } -static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { +static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, + SPhysiNode** pPhyNode) { STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode( pCxt, pScanLogicNode->pMeta->tableInfo.precision, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); if (NULL == pTagScan) { return TSDB_CODE_OUT_OF_MEMORY; } + vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan, pPhyNode); } @@ -513,7 +518,7 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SPhysiNode** pPhyNode) { switch (pScanLogicNode->scanType) { case SCAN_TYPE_TAG: - return createTagScanPhysiNode(pCxt, pScanLogicNode, pPhyNode); + return createTagScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); case SCAN_TYPE_TABLE: return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); case SCAN_TYPE_SYSTEM_TABLE: @@ -875,12 +880,6 @@ static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChil pInterval->intervalUnit = pWindowLogicNode->intervalUnit; pInterval->slidingUnit = pWindowLogicNode->slidingUnit; - pInterval->pFill = nodesCloneNode(pWindowLogicNode->pFill); - if (NULL != pWindowLogicNode->pFill && NULL == pInterval->pFill) { - nodesDestroyNode(pInterval); - return TSDB_CODE_OUT_OF_MEMORY; - } - return createWindowPhysiNodeFinalize(pCxt, pChildren, &pInterval->window, pWindowLogicNode, pPhyNode); } @@ -1030,6 +1029,46 @@ static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi return code; } +static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SFillLogicNode* pFillNode, + SPhysiNode** pPhyNode) { + SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pFillNode, + QUERY_NODE_PHYSICAL_PLAN_FILL); + if (NULL == pFill) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pFill->mode = pFillNode->mode; + pFill->timeRange = pFillNode->timeRange; + + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + int32_t code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pFillNode->node.pTargets, &pFill->pTargets); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pFill->pTargets, pFill->node.pOutputDataBlockDesc); + } + + if (TSDB_CODE_SUCCESS == code) { + pFill->pWStartTs = nodesCloneNode(pFillNode->pWStartTs); + if (NULL == pFill->pWStartTs) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code && NULL != pFillNode->pValues) { + pFill->pValues = nodesCloneNode(pFillNode->pValues); + if (NULL == pFill->pValues) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pFill; + } else { + nodesDestroyNode(pFill); + } + + return code; +} + static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubplan* pSubplan, SNodeList* pChildren, SPhysiNode** pPhyNode) { switch (nodeType(pLogicNode)) { @@ -1049,6 +1088,8 @@ static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode return createSortPhysiNode(pCxt, pChildren, (SSortLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_PARTITION: return createPartitionPhysiNode(pCxt, pChildren, (SPartitionLogicNode*)pLogicNode, pPhyNode); + case QUERY_NODE_LOGIC_PLAN_FILL: + return createFillPhysiNode(pCxt, pChildren, (SFillLogicNode*)pLogicNode, pPhyNode); default: break; } diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..36625b28fb3d101a6c1ae0d5a33a5fda3bbecdb9 --- /dev/null +++ b/source/libs/planner/src/planUtil.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planInt.h" + +static char* getUsageErrFormat(int32_t errCode) { + switch (errCode) { + case TSDB_CODE_PLAN_EXPECTED_TS_EQUAL: + return "l.ts = r.ts is expected in join expression"; + case TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN: + return "not support cross join"; + default: + break; + } + return "Unknown error"; +} + +int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...) { + va_list vArgList; + va_start(vArgList, errCode); + vsnprintf(pBuf, len, getUsageErrFormat(errCode), vArgList); + va_end(vArgList); + return errCode; +} diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 70a969584a0042b02d00c410232b686ef1d73ac4..6336377279716b0d60947cc1b3f637dfd4eebebb 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -104,8 +104,9 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes; return TSDB_CODE_SUCCESS; } + int32_t inputSize = (NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes); pVal->node.resType.type = pParam->buffer_type; - pVal->node.resType.bytes = NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes; + pVal->node.resType.bytes = inputSize; switch (pParam->buffer_type) { case TSDB_DATA_TYPE_BOOL: pVal->datum.b = *((bool*)pParam->buffer); @@ -130,7 +131,6 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { break; case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: - case TSDB_DATA_TYPE_NCHAR: pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); if (NULL == pVal->datum.p) { return TSDB_CODE_OUT_OF_MEMORY; @@ -138,6 +138,21 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { varDataSetLen(pVal->datum.p, pVal->node.resType.bytes); strncpy(varDataVal(pVal->datum.p), (const char*)pParam->buffer, pVal->node.resType.bytes); break; + case TSDB_DATA_TYPE_NCHAR: { + pVal->node.resType.bytes *= TSDB_NCHAR_SIZE; + pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); + if (NULL == pVal->datum.p) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t output = 0; + if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes, &output)) { + return errno; + } + varDataSetLen(pVal->datum.p, output); + pVal->node.resType.bytes = output; + break; + } case TSDB_DATA_TYPE_TIMESTAMP: pVal->datum.i = *((int64_t*)pParam->buffer); break; @@ -181,13 +196,20 @@ static EDealRes updatePlanQueryId(SNode* pNode, void* pContext) { int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId) { int32_t size = taosArrayGetSize(pPlan->pPlaceholderValues); - + int32_t code = 0; + if (colIdx < 0) { for (int32_t i = 0; i < size; ++i) { - setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, i), pParams + i); + code = setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, i), pParams + i); + if (code) { + return code; + } } } else { - setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, colIdx), pParams); + code = setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, colIdx), pParams); + if (code) { + return code; + } } if (colIdx < 0 || ((colIdx + 1) == size)) { diff --git a/source/libs/planner/test/CMakeLists.txt b/source/libs/planner/test/CMakeLists.txt index f0c59b969f0827a3c1a24c424819d097c71891ba..a21b36fef6b3eecc51bdbe4abbb7fff3dc065098 100644 --- a/source/libs/planner/test/CMakeLists.txt +++ b/source/libs/planner/test/CMakeLists.txt @@ -30,4 +30,9 @@ if(${BUILD_WINGETOPT}) PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" ) target_link_libraries(plannerTest PUBLIC wingetopt) -endif() \ No newline at end of file +endif() + +add_test( + NAME plannerTest + COMMAND plannerTest +) diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..639af2875eb8548378d136bf1a26ad4be4521fec --- /dev/null +++ b/source/libs/planner/test/planBasicTest.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, AND/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanBasicTest : public PlannerTestBase {}; + +TEST_F(PlanBasicTest, selectClause) { + useDb("root", "test"); + + run("SELECT * FROM t1"); + run("SELECT 1 FROM t1"); + run("SELECT * FROM st1"); + run("SELECT 1 FROM st1"); +} + +TEST_F(PlanBasicTest, whereClause) { + useDb("root", "test"); + + run("SELECT * FROM t1 WHERE c1 > 10"); + + run("SELECT * FROM t1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59'"); +} + +TEST_F(PlanBasicTest, joinClause) { + useDb("root", "test"); + + run("SELECT t1.c1, t2.c2 FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts"); + run("SELECT t1.c1, t2.c2 FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts"); +} + +TEST_F(PlanBasicTest, func) { + useDb("root", "test"); + + run("SELECT DIFF(c1) FROM t1"); +} diff --git a/source/libs/planner/test/planDistinctTest.cpp b/source/libs/planner/test/planDistinctTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..58473dcff27839654fe8ea4f33c9d408c2c4b24f --- /dev/null +++ b/source/libs/planner/test/planDistinctTest.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanDistinctTest : public PlannerTestBase {}; + +TEST_F(PlanDistinctTest, basic) { + useDb("root", "test"); + + // distinct single col + run("select distinct c1 from t1"); + // distinct col list + run("select distinct c1, c2 from t1"); +} + +TEST_F(PlanDistinctTest, expr) { + useDb("root", "test"); + + run("select distinct c1, c2 + 10 from t1"); +} + +TEST_F(PlanDistinctTest, withOrderBy) { + useDb("root", "test"); + + run("select distinct c1 + 10 a from t1 order by a"); +} diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9ca1001f4cb24224d9dd223e366c9eed83db4fbe --- /dev/null +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanGroupByTest : public PlannerTestBase {}; + +TEST_F(PlanGroupByTest, basic) { + useDb("root", "test"); + + run("SELECT COUNT(*) FROM t1"); + + run("SELECT c1, MAX(c3), MIN(c3), COUNT(*) FROM t1 GROUP BY c1"); + + run("SELECT c1 + c3, c1 + COUNT(*) FROM t1 WHERE c2 = 'abc' GROUP BY c1, c3"); + + run("SELECT c1 + c3, SUM(c4 * c5) FROM t1 WHERE CONCAT(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); + + run("SELECT SUM(CEIL(c1)) FROM t1 GROUP BY CEIL(c1)"); +} + +TEST_F(PlanGroupByTest, withOrderBy) { + useDb("root", "test"); + + // ORDER BY aggfunc + run("SELECT COUNT(*), SUM(c1) FROM t1 ORDER BY SUM(c1)"); + // ORDER BY alias of aggfunc + // run("SELECT COUNT(*), SUM(c1) a FROM t1 ORDER BY a"); +} + +TEST_F(PlanGroupByTest, aggFunc) { + useDb("root", "test"); + + run("SELECT LAST(*), FIRST(*) FROM t1"); + + run("SELECT LAST(*), FIRST(*) FROM t1 GROUP BY c1"); +} + +TEST_F(PlanGroupByTest, selectFunc) { + useDb("root", "test"); + + // select function + run("SELECT MAX(c1), MIN(c1) FROM t1"); + // select function for GROUP BY clause + run("SELECT MAX(c1), MIN(c1) FROM t1 GROUP BY c1"); + // select function along with the columns of select row + run("SELECT MAX(c1), c2 FROM t1"); + run("SELECT MAX(c1), t1.* FROM t1"); + // select function along with the columns of select row, and with GROUP BY clause + run("SELECT MAX(c1), c2 FROM t1 GROUP BY c3"); + run("SELECT MAX(c1), t1.* FROM t1 GROUP BY c3"); +} diff --git a/source/libs/planner/test/planIntervalTest.cpp b/source/libs/planner/test/planIntervalTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c9bae46ca9438977f4078ceac82e6c7c4b3c680e --- /dev/null +++ b/source/libs/planner/test/planIntervalTest.cpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanIntervalTest : public PlannerTestBase {}; + +TEST_F(PlanIntervalTest, basic) { + useDb("root", "test"); + + run("SELECT COUNT(*) FROM t1 INTERVAL(10s)"); +} + +TEST_F(PlanIntervalTest, pseudoCol) { + useDb("root", "test"); + + run("SELECT _WSTARTTS, _WDURATION, _WENDTS, COUNT(*) FROM t1 INTERVAL(10s)"); +} + +TEST_F(PlanIntervalTest, fill) { + useDb("root", "test"); + + run("SELECT COUNT(*) FROM t1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " + "INTERVAL(10s) FILL(LINEAR)"); + + run("SELECT COUNT(*), SUM(c1) FROM t1 " + "WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " + "INTERVAL(10s) FILL(VALUE, 10, 20)"); +} + +TEST_F(PlanIntervalTest, selectFunc) { + useDb("root", "test"); + + // select function for INTERVAL clause + run("SELECT MAX(c1), MIN(c1) FROM t1 INTERVAL(10s)"); + // select function along with the columns of select row, and with INTERVAL clause + run("SELECT MAX(c1), c2 FROM t1 INTERVAL(10s)"); +} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbTxn.h b/source/libs/planner/test/planJoinTest.cpp similarity index 57% rename from source/libs/tdb/src/inc/tdbTxn.h rename to source/libs/planner/test/planJoinTest.cpp index ed9784614cf83850bb2f7deae8dead0b70671cc1..4098d383f8edc2461613cfb8ab9102035185b4b9 100644 --- a/source/libs/tdb/src/inc/tdbTxn.h +++ b/source/libs/planner/test/planJoinTest.cpp @@ -13,19 +13,20 @@ * along with this program. If not, see . */ -#ifndef _TDB_TXN_H_ -#define _TDB_TXN_H_ +#include "planTestUtil.h" +#include "planner.h" -#ifdef __cplusplus -extern "C" { -#endif +using namespace std; -int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, - int flags); -int tdbTxnClose(TXN *pTxn); +class PlanJoinTest : public PlannerTestBase {}; -#ifdef __cplusplus -} -#endif +TEST_F(PlanJoinTest, basic) { + useDb("root", "test"); + + run("select t1.c1, t2.c2 from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); -#endif /*_TDB_TXN_H_*/ \ No newline at end of file + run("select t1.*, t2.* from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); + + // run("select t1.c1, t2.c1 from st1s1 t1 join st1s2 t2 on t1.ts = t2.ts where t1.c1 > t2.c1 and t1.c2 = 'abc' and " + // "t2.c2 = 'qwe'"); +} diff --git a/source/dnode/vnode/src/vnd/vnodeInt.c b/source/libs/planner/test/planLimitTest.cpp similarity index 54% rename from source/dnode/vnode/src/vnd/vnodeInt.c rename to source/libs/planner/test/planLimitTest.cpp index 14dfc75cedcf5ddda581214f7ecf77442aef7060..3b8b3180bdd3b873736ba65ef362b6985ec60c0d 100644 --- a/source/dnode/vnode/src/vnd/vnodeInt.c +++ b/source/libs/planner/test/planLimitTest.cpp @@ -13,12 +13,29 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE -#include "vnd.h" -// #include "vnodeInt.h" +#include "planTestUtil.h" +#include "planner.h" -int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg) { return 0; } +using namespace std; -int32_t vnodeCompact(SVnode *pVnode) { return 0; } +class PlanLimitTest : public PlannerTestBase {}; -int32_t vnodeSync(SVnode *pVnode) { return 0; } \ No newline at end of file +TEST_F(PlanLimitTest, limit) { + useDb("root", "test"); + + run("select * from t1 limit 2"); + + run("select * from t1 limit 5 offset 2"); + + run("select * from t1 limit 2, 5"); +} + +TEST_F(PlanLimitTest, slimit) { + useDb("root", "test"); + + run("select * from t1 partition by c1 slimit 2"); + + run("select * from t1 partition by c1 slimit 5 soffset 2"); + + run("select * from t1 partition by c1 slimit 2, 5"); +} diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4938618db62fc496eb8ce935c713713289039852 --- /dev/null +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanOptimizeTest : public PlannerTestBase {}; + +TEST_F(PlanOptimizeTest, optimizeScanData) { + useDb("root", "test"); + + run("SELECT COUNT(*) FROM t1"); + + run("SELECT COUNT(c1) FROM t1"); + + run("SELECT COUNT(CAST(c1 AS BIGINT)) FROM t1"); +} + +TEST_F(PlanOptimizeTest, orderByPrimaryKey) { + useDb("root", "test"); + + run("SELECT * FROM t1 ORDER BY ts"); + run("SELECT * FROM t1 ORDER BY ts DESC"); + run("SELECT c1 FROM t1 ORDER BY ts"); + run("SELECT c1 FROM t1 ORDER BY ts DESC"); +} diff --git a/source/libs/planner/test/planOrderByTest.cpp b/source/libs/planner/test/planOrderByTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d95d1bdf1d429d9ea6c5e62c5a280dbb5c6de477 --- /dev/null +++ b/source/libs/planner/test/planOrderByTest.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanOrderByTest : public PlannerTestBase {}; + +TEST_F(PlanOrderByTest, basic) { + useDb("root", "test"); + + // order by key is in the projection list + run("select c1 from t1 order by c1"); + // order by key is not in the projection list + run("select c1 from t1 order by c2"); +} + +TEST_F(PlanOrderByTest, expr) { + useDb("root", "test"); + + run("select * from t1 order by c1 + 10, c2"); +} + +TEST_F(PlanOrderByTest, nullsOrder) { + useDb("root", "test"); + + run("select * from t1 order by c1 desc nulls first"); +} diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b70cb4d19ae99de436914d95e4192cf297b2435d --- /dev/null +++ b/source/libs/planner/test/planOtherTest.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanOtherTest : public PlannerTestBase {}; + +TEST_F(PlanOtherTest, createTopic) { + useDb("root", "test"); + + run("create topic tp as SELECT * FROM st1"); +} + +TEST_F(PlanOtherTest, createStream) { + useDb("root", "test"); + + run("create stream if not exists s1 trigger window_close watermark 10s into st1 as select count(*) from t1 " + "interval(10s)"); +} + +TEST_F(PlanOtherTest, createSmaIndex) { + useDb("root", "test"); + + run("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) interval(10s)"); +} + +TEST_F(PlanOtherTest, explain) { + useDb("root", "test"); + + run("explain SELECT * FROM t1"); + + run("explain analyze SELECT * FROM t1"); + + run("explain analyze verbose true ratio 0.01 SELECT * FROM t1"); +} \ No newline at end of file diff --git a/source/libs/planner/test/planPartByTest.cpp b/source/libs/planner/test/planPartByTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..230500e70266053684b2a3ee94876cd3e42efd68 --- /dev/null +++ b/source/libs/planner/test/planPartByTest.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanPartitionByTest : public PlannerTestBase {}; + +TEST_F(PlanPartitionByTest, basic) { + useDb("root", "test"); + + run("select * from t1 partition by c1"); +} + +TEST_F(PlanPartitionByTest, withAggFunc) { + useDb("root", "test"); + + run("select count(*) from t1 partition by c1"); +} + +TEST_F(PlanPartitionByTest, withInterval) { + useDb("root", "test"); + + // normal/child table + run("select count(*) from t1 partition by c1 interval(10s)"); + // super table + run("select count(*) from st1 partition by tag1, tag2 interval(10s)"); +} + +TEST_F(PlanPartitionByTest, withGroupBy) { + useDb("root", "test"); + + run("select count(*) from t1 partition by c1 group by c2"); +} diff --git a/source/libs/planner/test/planSTableTest.cpp b/source/libs/planner/test/planSTableTest.cpp index 0b60eaea32a2bb063c65167bcf5b7a7e2baad28d..6e349f8f9ca976a59bfc41682ed8285477a4dcd2 100644 --- a/source/libs/planner/test/planSTableTest.cpp +++ b/source/libs/planner/test/planSTableTest.cpp @@ -23,4 +23,6 @@ TEST_F(PlanSuperTableTest, tbname) { useDb("root", "test"); run("select tbname from st1"); + + run("select tbname, tag1, tag2 from st1"); } diff --git a/source/libs/planner/test/planSessionTest.cpp b/source/libs/planner/test/planSessionTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7d5d82692557207ad032e4a9172c52da86b75f44 --- /dev/null +++ b/source/libs/planner/test/planSessionTest.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanSessionTest : public PlannerTestBase {}; + +TEST_F(PlanSessionTest, basic) { + useDb("root", "test"); + + run("select count(*) from t1 session(ts, 10s)"); +} + +TEST_F(PlanSessionTest, selectFunc) { + useDb("root", "test"); + + // select function for SESSION clause + run("SELECT MAX(c1), MIN(c1) FROM t1 SESSION(ts, 10s)"); + // select function along with the columns of select row, and with SESSION clause + run("SELECT MAX(c1), c2 FROM t1 SESSION(ts, 10s)"); +} diff --git a/source/libs/planner/test/planStateTest.cpp b/source/libs/planner/test/planStateTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..83c9621916a3a080d578d29cf1a1abd6f0dc94d9 --- /dev/null +++ b/source/libs/planner/test/planStateTest.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanStateTest : public PlannerTestBase {}; + +TEST_F(PlanStateTest, basic) { + useDb("root", "test"); + + run("select count(*) from t1 state_window(c1)"); +} + +TEST_F(PlanStateTest, stateExpr) { + useDb("root", "test"); + + run("select count(*) from t1 state_window(c1 + 10)"); +} + +TEST_F(PlanStateTest, selectFunc) { + useDb("root", "test"); + + // select function for STATE_WINDOW clause + run("SELECT MAX(c1), MIN(c1) FROM t1 STATE_WINDOW(c3)"); + // select function along with the columns of select row, and with STATE_WINDOW clause + run("SELECT MAX(c1), c2 FROM t1 STATE_WINDOW(c3)"); +} diff --git a/source/libs/planner/test/planStmtTest.cpp b/source/libs/planner/test/planStmtTest.cpp index edf1990704a59e5cd08f64adc3543322e98b8102..6d56a3419e40c5d62684272aa1e54b252713f7cc 100644 --- a/source/libs/planner/test/planStmtTest.cpp +++ b/source/libs/planner/test/planStmtTest.cpp @@ -42,13 +42,13 @@ class PlanStmtTest : public PlannerTestBase { // todo } -private: + private: TAOS_MULTI_BIND* pBindParams_; - int32_t paramNo_; + int32_t paramNo_; }; TEST_F(PlanStmtTest, stmt) { useDb("root", "test"); - run("SELECT * FROM t1 where c1 = ?"); + run("select * from t1 where c1 = ?"); } diff --git a/source/libs/parser/test/parAlterTest.cpp b/source/libs/planner/test/planSubqueryTest.cpp similarity index 63% rename from source/libs/parser/test/parAlterTest.cpp rename to source/libs/planner/test/planSubqueryTest.cpp index 0e6a0041384badf168ddc4aa7774dd39a74ab77d..185f55db10e87d23c77425d000f91cf9b8b63ac6 100644 --- a/source/libs/parser/test/parAlterTest.cpp +++ b/source/libs/planner/test/planSubqueryTest.cpp @@ -13,14 +13,22 @@ * along with this program. If not, see . */ -#include "parTestUtil.h" +#include "planTestUtil.h" +#include "planner.h" using namespace std; -class ParserAlterTest : public ParserTestBase {}; +class PlanSubqeuryTest : public PlannerTestBase {}; -TEST_F(ParserAlterTest, stmt) { +TEST_F(PlanSubqeuryTest, basic) { useDb("root", "test"); - run("create database db1"); + run("select * from (select * from t1)"); +} + +TEST_F(PlanSubqeuryTest, doubleGroupBy) { + useDb("root", "test"); + + run("select count(*) from (select c1 + c3 a, c1 + count(*) b from t1 where c2 = 'abc' group by c1, c3) where a > 100 " + "group by b"); } diff --git a/source/libs/planner/test/planOptTest.cpp b/source/libs/planner/test/planSysTbTest.cpp similarity index 73% rename from source/libs/planner/test/planOptTest.cpp rename to source/libs/planner/test/planSysTbTest.cpp index deb20c65a4a3eb9c76ec455556f52977fb9a3218..fff6bfcca437887b28e70d59327a6873de13730d 100644 --- a/source/libs/planner/test/planOptTest.cpp +++ b/source/libs/planner/test/planSysTbTest.cpp @@ -18,13 +18,17 @@ using namespace std; -class PlanOptimizeTest : public PlannerTestBase {}; +class PlanSysTableTest : public PlannerTestBase {}; -TEST_F(PlanOptimizeTest, orderByPrimaryKey) { +TEST_F(PlanSysTableTest, show) { useDb("root", "test"); - run("select * from t1 order by ts"); - run("select * from t1 order by ts desc"); - run("select c1 from t1 order by ts"); - run("select c1 from t1 order by ts desc"); + run("show tables"); + run("show stables"); +} + +TEST_F(PlanSysTableTest, information) { + useDb("root", "information_schema"); + + run("show tables"); } diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index a7726973f0ebe9d6979bc8d4a39e6b38cd6fd191..464c636b66cfebf8fd84461fb2a41d804e1856c1 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -36,11 +36,11 @@ class PlannerEnv : public testing::Environment { static void parseArg(int argc, char* argv[]) { int opt = 0; const char* optstring = ""; - static struct option long_options[] = {{"dump", no_argument, NULL, 'd'}, {0, 0, 0, 0}}; + static struct option long_options[] = {{"dump", optional_argument, NULL, 'd'}, {0, 0, 0, 0}}; while ((opt = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) { switch (opt) { case 'd': - g_isDump = true; + setDumpModule(optarg); break; default: break; diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 2ecb59f0a12f46fe0fb613eccc60eb0029875f34..e434a6be968d582ab23f9c63f1ebd77f4186e7c0 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -34,7 +34,41 @@ using namespace testing; } \ } while (0); -bool g_isDump = false; +enum DumpModule { + DUMP_MODULE_NOTHING = 1, + DUMP_MODULE_PARSER, + DUMP_MODULE_LOGIC, + DUMP_MODULE_OPTIMIZED, + DUMP_MODULE_SPLIT, + DUMP_MODULE_SCALED, + DUMP_MODULE_PHYSICAL, + DUMP_MODULE_SUBPLAN, + DUMP_MODULE_ALL +}; + +DumpModule g_dumpModule = DUMP_MODULE_NOTHING; + +void setDumpModule(const char* pModule) { + if (NULL == pModule) { + g_dumpModule = DUMP_MODULE_ALL; + } else if (0 == strncasecmp(pModule, "parser", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_PARSER; + } else if (0 == strncasecmp(pModule, "logic", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_LOGIC; + } else if (0 == strncasecmp(pModule, "optimized", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_OPTIMIZED; + } else if (0 == strncasecmp(pModule, "split", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_SPLIT; + } else if (0 == strncasecmp(pModule, "scaled", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_SCALED; + } else if (0 == strncasecmp(pModule, "physical", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_PHYSICAL; + } else if (0 == strncasecmp(pModule, "subplan", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_SUBPLAN; + } else if (0 == strncasecmp(pModule, "all", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_PHYSICAL; + } +} class PlannerTestBaseImpl { public: @@ -66,11 +100,9 @@ class PlannerTestBaseImpl { SQueryPlan* pPlan = nullptr; doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan); - if (g_isDump) { - dump(); - } + dump(g_dumpModule); } catch (...) { - dump(); + dump(DUMP_MODULE_ALL); throw; } } @@ -106,25 +138,51 @@ class PlannerTestBaseImpl { res_.splitLogicPlan_.clear(); res_.scaledLogicPlan_.clear(); res_.physiPlan_.clear(); + res_.physiSubplans_.clear(); } - void dump() { + void dump(DumpModule module) { + if (DUMP_MODULE_NOTHING == module) { + return; + } + cout << "==========================================sql : [" << stmtEnv_.sql_ << "]" << endl; - cout << "syntax tree : " << endl; - cout << res_.ast_ << endl; - cout << "raw logic plan : " << endl; - cout << res_.rawLogicPlan_ << endl; - cout << "optimized logic plan : " << endl; - cout << res_.optimizedLogicPlan_ << endl; - cout << "split logic plan : " << endl; - cout << res_.splitLogicPlan_ << endl; - cout << "scaled logic plan : " << endl; - cout << res_.scaledLogicPlan_ << endl; - cout << "physical plan : " << endl; - cout << res_.physiPlan_ << endl; - cout << "physical subplan : " << endl; - for (const auto& subplan : res_.physiSubplans_) { - cout << subplan << endl; + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) { + cout << "syntax tree : " << endl; + cout << res_.ast_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_LOGIC == module) { + cout << "raw logic plan : " << endl; + cout << res_.rawLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_OPTIMIZED == module) { + cout << "optimized logic plan : " << endl; + cout << res_.optimizedLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_SPLIT == module) { + cout << "split logic plan : " << endl; + cout << res_.splitLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_SCALED == module) { + cout << "scaled logic plan : " << endl; + cout << res_.scaledLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_PHYSICAL == module) { + cout << "physical plan : " << endl; + cout << res_.physiPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_SUBPLAN == module) { + cout << "physical subplan : " << endl; + for (const auto& subplan : res_.physiSubplans_) { + cout << subplan << endl; + } } } diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index 7aabff811f06e6ad1485162102be1f350d8d0f94..7913ef531f1fc4cf0093db5ef0f1db8c3bdc2d35 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -32,6 +32,6 @@ class PlannerTestBase : public testing::Test { std::unique_ptr impl_; }; -extern bool g_isDump; +extern void setDumpModule(const char* pModule); #endif // PLAN_TEST_UTIL_H diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp deleted file mode 100644 index 11280768237d225aaa8a51bdc1d0b7ad558060e0..0000000000000000000000000000000000000000 --- a/source/libs/planner/test/plannerTest.cpp +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include - -#include - -#include "cmdnodes.h" -#include "parser.h" -#include "planInt.h" - -using namespace std; -using namespace testing; - -class PlannerTest : public Test { - protected: - void setDatabase(const string& acctId, const string& db) { - acctId_ = acctId; - db_ = db; - } - - void bind(const char* sql) { - reset(); - cxt_.acctId = atoi(acctId_.c_str()); - cxt_.db = db_.c_str(); - sqlBuf_ = string(sql); - transform(sqlBuf_.begin(), sqlBuf_.end(), sqlBuf_.begin(), ::tolower); - cxt_.sqlLen = strlen(sql); - cxt_.pSql = sqlBuf_.c_str(); - } - - bool run(bool streamQuery = false) { - int32_t code = qParseQuerySql(&cxt_, &query_); - - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] qParseQuerySql code:" << code << ", strerror:" << tstrerror(code) - << ", msg:" << errMagBuf_ << endl; - return false; - } - - const string syntaxTreeStr = toString(query_->pRoot, false); - - SLogicNode* pLogicNode = nullptr; - SPlanContext cxt = {0}; - cxt.queryId = 1; - cxt.acctId = 0; - cxt.streamQuery = streamQuery; - - setPlanContext(query_, &cxt); - code = createLogicPlan(&cxt, &pLogicNode); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] createLogicPlan code:" << code << ", strerror:" << tstrerror(code) << endl; - return false; - } - - cout << "====================sql : [" << cxt_.pSql << "]" << endl; - cout << "syntax tree : " << endl; - cout << syntaxTreeStr << endl; - cout << "unformatted logic plan : " << endl; - cout << toString((const SNode*)pLogicNode, false) << endl; - - code = optimizeLogicPlan(&cxt, pLogicNode); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] optimizeLogicPlan code:" << code << ", strerror:" << tstrerror(code) << endl; - return false; - } - - SLogicSubplan* pLogicSubplan = nullptr; - code = splitLogicPlan(&cxt, pLogicNode, &pLogicSubplan); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] splitLogicPlan code:" << code << ", strerror:" << tstrerror(code) << endl; - return false; - } - - SQueryLogicPlan* pLogicPlan = NULL; - code = scaleOutLogicPlan(&cxt, pLogicSubplan, &pLogicPlan); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] createPhysiPlan code:" << code << ", strerror:" << tstrerror(code) << endl; - return false; - } - - SArray* pExecNodeList = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SQueryNodeAddr)); - code = createPhysiPlan(&cxt, pLogicPlan, &plan_, pExecNodeList); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] createPhysiPlan code:" << code << ", strerror:" << tstrerror(code) << endl; - return false; - } - - cout << "unformatted physical plan : " << endl; - cout << toString((const SNode*)plan_, false) << endl; - SNode* pNode; - FOREACH(pNode, plan_->pSubplans) { - SNode* pSubplan; - FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { - cout << "unformatted physical subplan : " << endl; - cout << toString(pSubplan, false) << endl; - } - } - - return true; - } - - private: - static const int max_err_len = 1024; - - void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { - if (QUERY_NODE_CREATE_TOPIC_STMT == nodeType(pQuery->pRoot)) { - pCxt->pAstRoot = ((SCreateTopicStmt*)pQuery->pRoot)->pQuery; - pCxt->topicQuery = true; - } else if (QUERY_NODE_CREATE_INDEX_STMT == nodeType(pQuery->pRoot)) { - SMCreateSmaReq req = {0}; - tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req); - nodesStringToNode(req.ast, &pCxt->pAstRoot); - pCxt->streamQuery = true; - } else if (QUERY_NODE_CREATE_STREAM_STMT == nodeType(pQuery->pRoot)) { - SCreateStreamStmt* pStmt = (SCreateStreamStmt*)pQuery->pRoot; - pCxt->pAstRoot = pStmt->pQuery; - pCxt->streamQuery = true; - pCxt->triggerType = pStmt->pOptions->triggerType; - pCxt->watermark = (NULL != pStmt->pOptions->pWatermark ? ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i : 0); - } else { - pCxt->pAstRoot = pQuery->pRoot; - } - } - - void reset() { - memset(&cxt_, 0, sizeof(cxt_)); - memset(errMagBuf_, 0, max_err_len); - cxt_.pMsg = errMagBuf_; - cxt_.msgLen = max_err_len; - } - - string toString(const SNode* pRoot, bool format = true) { - char* pStr = NULL; - int32_t len = 0; - int32_t code = nodesNodeToString(pRoot, format, &pStr, &len); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] toString code:" << code << ", strerror:" << tstrerror(code) << endl; - return string(); - } - string str(pStr); - taosMemoryFreeClear(pStr); - return str; - } - - string acctId_; - string db_; - char errMagBuf_[max_err_len]; - string sqlBuf_; - SParseContext cxt_; - SQuery* query_; - SQueryPlan* plan_; -}; - -TEST_F(PlannerTest, selectBasic) { - setDatabase("root", "test"); - - bind("SELECT * FROM t1"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectConstant) { - setDatabase("root", "test"); - - bind("SELECT 2-1 FROM t1"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectStableBasic) { - setDatabase("root", "test"); - - bind("SELECT * FROM st1"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectJoin) { - setDatabase("root", "test"); - - bind("SELECT t1.c1, t2.c2 FROM st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); - ASSERT_TRUE(run()); - - bind("SELECT t1.*, t2.* FROM st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); - ASSERT_TRUE(run()); - - bind( - "SELECT t1.c1, t2.c1 FROM st1s1 t1 join st1s2 t2 on t1.ts = t2.ts where t1.c1 > t2.c1 and t1.c2 = 'abc' and " - "t2.c2 = 'qwe'"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectGroupBy) { - setDatabase("root", "test"); - - bind("SELECT count(*) FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT c1, max(c3), min(c3), count(*) FROM t1 GROUP BY c1"); - ASSERT_TRUE(run()); - - bind("SELECT c1 + c3, c1 + count(*) FROM t1 where c2 = 'abc' GROUP BY c1, c3"); - ASSERT_TRUE(run()); - - bind("SELECT c1 + c3, sum(c4 * c5) FROM t1 where concat(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); - ASSERT_TRUE(run()); - - bind("SELECT sum(ceil(c1)) FROM t1 GROUP BY ceil(c1)"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectSubquery) { - setDatabase("root", "test"); - - bind( - "SELECT count(*) FROM (SELECT c1 + c3 a, c1 + count(*) b FROM t1 where c2 = 'abc' GROUP BY c1, c3) where a > 100 " - "group by b"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectInterval) { - setDatabase("root", "test"); - - bind("SELECT count(*) FROM t1 interval(10s)"); - ASSERT_TRUE(run()); - - bind("SELECT _wstartts, _wduration, _wendts, count(*) FROM t1 interval(10s)"); - ASSERT_TRUE(run()); - - bind("SELECT count(*) FROM t1 interval(10s) fill(linear)"); - ASSERT_TRUE(run()); - - bind("SELECT count(*), sum(c1) FROM t1 interval(10s) fill(value, 10, 20)"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectSessionWindow) { - setDatabase("root", "test"); - - bind("SELECT count(*) FROM t1 session(ts, 10s)"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectStateWindow) { - setDatabase("root", "test"); - - bind("SELECT count(*) FROM t1 state_window(c1)"); - ASSERT_TRUE(run()); - - bind("SELECT count(*) FROM t1 state_window(c1 + 10)"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectPartitionBy) { - setDatabase("root", "test"); - - bind("SELECT * FROM t1 partition by c1"); - ASSERT_TRUE(run()); - - bind("SELECT count(*) FROM t1 partition by c1"); - ASSERT_TRUE(run()); - - bind("SELECT count(*) FROM t1 partition by c1 group by c2"); - ASSERT_TRUE(run()); - - bind("SELECT count(*) FROM st1 partition by tag1, tag2 interval(10s)"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectOrderBy) { - setDatabase("root", "test"); - - bind("SELECT c1 FROM t1 order by c1"); - ASSERT_TRUE(run()); - - bind("SELECT c1 FROM t1 order by c2"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 order by c1 + 10, c2"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 order by c1 desc nulls first"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectGroupByOrderBy) { - setDatabase("root", "test"); - - bind("select count(*), sum(c1) from t1 order by sum(c1)"); - ASSERT_TRUE(run()); - - bind("select count(*), sum(c1) a from t1 order by a"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectDistinct) { - setDatabase("root", "test"); - - bind("SELECT distinct c1 FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT distinct c1, c2 + 10 FROM t1"); - ASSERT_TRUE(run()); - - bind("SELECT distinct c1 + 10 a FROM t1 order by a"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectLimit) { - setDatabase("root", "test"); - - bind("SELECT * FROM t1 limit 2"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 limit 5 offset 2"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 limit 2, 5"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, selectSlimit) { - setDatabase("root", "test"); - - bind("SELECT * FROM t1 partition by c1 slimit 2"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 partition by c1 slimit 5 soffset 2"); - ASSERT_TRUE(run()); - - bind("SELECT * FROM t1 partition by c1 slimit 2, 5"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, showTables) { - setDatabase("root", "test"); - - bind("show tables"); - ASSERT_TRUE(run()); - - setDatabase("root", "information_schema"); - - bind("show tables"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, showStables) { - setDatabase("root", "test"); - - bind("show stables"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, createTopic) { - setDatabase("root", "test"); - - bind("create topic tp as SELECT * FROM st1"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, createStream) { - setDatabase("root", "test"); - - bind( - "create stream if not exists s1 trigger window_close watermark 10s into st1 as select count(*) from t1 " - "interval(10s)"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, createSmaIndex) { - setDatabase("root", "test"); - - bind("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) INTERVAL(10s)"); - ASSERT_TRUE(run()); -} - -TEST_F(PlannerTest, explain) { - setDatabase("root", "test"); - - bind("explain SELECT * FROM t1"); - ASSERT_TRUE(run()); - - bind("explain analyze SELECT * FROM t1"); - ASSERT_TRUE(run()); - - bind("explain analyze verbose true ratio 0.01 SELECT * FROM t1"); - ASSERT_TRUE(run()); -} diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index fd7c831399901a5c8d9922e011213e4267d748cd..822c214fe58bba136f773523da8ec7b0bb4ab768 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -17,6 +17,7 @@ #include "queryInt.h" #include "query.h" #include "trpc.h" +#include "systable.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation" @@ -180,6 +181,25 @@ int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int3 return TSDB_CODE_SUCCESS; } +int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SGetUserAuthReq req = {0}; + strncpy(req.user, input, sizeof(req.user)); + + int32_t bufLen = tSerializeSGetUserAuthReq(NULL, 0, &req); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSGetUserAuthReq(pBuf, bufLen, &req); + + *msg = pBuf; + *msgLen = bufLen; + + return TSDB_CODE_SUCCESS; +} + + int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { SUseDbOutput *pOut = output; SUseDbRsp usedbRsp = {0}; @@ -418,6 +438,20 @@ int32_t queryProcessRetrieveFuncRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_SUCCESS; } +int32_t queryProcessGetUserAuthRsp(void *output, char *msg, int32_t msgSize) { + if (NULL == output || NULL == msg || msgSize <= 0) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + if (tDeserializeSGetUserAuthRsp(msg, msgSize, (SGetUserAuthRsp *)output) != 0) { + qError("tDeserializeSGetUserAuthRsp failed, msgSize:%d", msgSize); + return TSDB_CODE_INVALID_MSG; + } + + return TSDB_CODE_SUCCESS; +} + + void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryBuildTableMetaReqMsg; @@ -426,6 +460,8 @@ void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryBuildGetDBCfgMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryBuildGetIndexMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryBuildRetrieveFuncMsg; + queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = queryBuildGetUserAuthMsg; + queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp; @@ -434,6 +470,7 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryProcessGetDbCfgRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryProcessGetIndexRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryProcessRetrieveFuncRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = queryProcessGetUserAuthRsp; } #pragma GCC diagnostic pop diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index de73f1db0b496fa6fe865831177e150464ef916e..ca471262ff747b2466159a93571ac77120b3b9ac 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -20,9 +20,11 @@ extern "C" { #endif +#include "osDef.h" #include "qworker.h" #include "tlockfree.h" #include "ttimer.h" +#include "tref.h" #define QW_DEFAULT_SCHEDULER_NUMBER 10000 #define QW_DEFAULT_TASK_NUMBER 10000 @@ -85,6 +87,12 @@ typedef struct SQWMsg { SQWConnInfo connInfo; } SQWMsg; +typedef struct SQWHbParam { + bool inUse; + int32_t qwrId; + int64_t refId; +} SQWHbParam; + typedef struct SQWHbInfo { SSchedulerHbRsp rsp; SQWConnInfo connInfo; @@ -137,7 +145,8 @@ typedef struct SQWSchStatus { } SQWSchStatus; // Qnode/Vnode level task management -typedef struct SQWorkerMgmt { +typedef struct SQWorker { + int64_t refId; SQWorkerCfg cfg; int8_t nodeType; int32_t nodeId; @@ -148,9 +157,17 @@ typedef struct SQWorkerMgmt { SHashObj *schHash; // key: schedulerId, value: SQWSchStatus SHashObj *ctxHash; // key: queryId+taskId, value: SQWTaskCtx SMsgCb msgCb; +} SQWorker; + +typedef struct SQWorkerMgmt { + SRWLatch lock; + int32_t qwRef; + int32_t qwNum; + SQWHbParam param[1024]; + int32_t paramIdx; } SQWorkerMgmt; -#define QW_FPARAMS_DEF SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId +#define QW_FPARAMS_DEF SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId #define QW_IDS() sId, qId, tId, rId #define QW_FPARAMS() mgmt, QW_IDS() @@ -209,13 +226,13 @@ typedef struct SQWorkerMgmt { } \ } while (0) -#define QW_ELOG(param, ...) qError("QW:%p " param, mgmt, __VA_ARGS__) -#define QW_DLOG(param, ...) qDebug("QW:%p " param, mgmt, __VA_ARGS__) +#define QW_ELOG(_param, ...) qError("QW:%p " _param, mgmt, __VA_ARGS__) +#define QW_DLOG(_param, ...) qDebug("QW:%p " _param, mgmt, __VA_ARGS__) -#define QW_DUMP(param, ...) \ +#define QW_DUMP(_param, ...) \ do { \ if (gQWDebug.dumpEnable) { \ - qDebug("QW:%p " param, mgmt, __VA_ARGS__); \ + qDebug("QW:%p " _param, mgmt, __VA_ARGS__); \ } \ } while (0) @@ -282,6 +299,14 @@ typedef struct SQWorkerMgmt { } \ } while (0) + +extern SQWorkerMgmt gQwMgmt; + +static FORCE_INLINE SQWorker *qwAcquire(int64_t refId) { return (SQWorker *)taosAcquireRef(atomic_load_32(&gQwMgmt.qwRef), refId); } + +static FORCE_INLINE int32_t qwRelease(int64_t refId) { return taosReleaseRef(gQwMgmt.qwRef, refId); } + + #ifdef __cplusplus } #endif diff --git a/source/libs/qworker/inc/qworkerMsg.h b/source/libs/qworker/inc/qworkerMsg.h index fb1950d16bb6dafcbf8c44537e94a33d51b58cf3..fbd9ce91bc96369ef6d21f63b0094efe05a33b93 100644 --- a/source/libs/qworker/inc/qworkerMsg.h +++ b/source/libs/qworker/inc/qworkerMsg.h @@ -28,7 +28,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg); -int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req); +int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req); int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code); int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code); @@ -41,10 +41,10 @@ int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code); int32_t qwBuildAndSendExplainRsp(SQWConnInfo *pConn, SExplainExecInfo *execInfo, int32_t num); void qwFreeFetchRsp(void *msg); int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp); -int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp); +int32_t qwGetSchTasksStatus(SQWorker *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp); int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *rsp, int32_t code); int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn); -int32_t qwRegisterHbBrokenLinkArg(SQWorkerMgmt *mgmt, uint64_t sId, SQWConnInfo *pConn); +int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t sId, SQWConnInfo *pConn); #ifdef __cplusplus } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index b51bc5083e826eeeda453f6a5eb2fd7de51aec84..bd7ee6321aa067357d36d6739ea252ce9e00fdc1 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -10,6 +10,11 @@ #include "tname.h" SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = true}; +SQWorkerMgmt gQwMgmt = { + .lock = 0, + .qwRef = -1, + .qwNum = 0, +}; int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) { if (!gQWDebug.statusEnable) { @@ -98,7 +103,7 @@ _return: void qwDbgDumpSchInfo(SQWSchStatus *sch, int32_t i) {} -void qwDbgDumpMgmtInfo(SQWorkerMgmt *mgmt) { +void qwDbgDumpMgmtInfo(SQWorker *mgmt) { if (!gQWDebug.dumpEnable) { return; } @@ -186,7 +191,7 @@ int32_t qwSetTaskStatus(QW_FPARAMS_DEF, SQWTaskStatus *task, int8_t status) { return TSDB_CODE_SUCCESS; } -int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType) { +int32_t qwAddSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType) { SQWSchStatus newSch = {0}; newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); @@ -213,7 +218,7 @@ int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType) { return TSDB_CODE_SUCCESS; } -int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch, int32_t nOpt) { +int32_t qwAcquireSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch, int32_t nOpt) { while (true) { QW_LOCK(rwType, &mgmt->schLock); *sch = taosHashGet(mgmt->schHash, &sId, sizeof(sId)); @@ -240,15 +245,15 @@ int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, return TSDB_CODE_SUCCESS; } -int32_t qwAcquireAddScheduler(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch) { +int32_t qwAcquireAddScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch) { return qwAcquireSchedulerImpl(mgmt, sId, rwType, sch, QW_NOT_EXIST_ADD); } -int32_t qwAcquireScheduler(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch) { +int32_t qwAcquireScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch) { return qwAcquireSchedulerImpl(mgmt, sId, rwType, sch, QW_NOT_EXIST_RET_ERR); } -void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); } +void qwReleaseScheduler(int32_t rwType, SQWorker *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); } int32_t qwAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, SQWTaskStatus **task) { char id[sizeof(qId) + sizeof(tId)] = {0}; @@ -384,7 +389,7 @@ int32_t qwAddTaskCtx(QW_FPARAMS_DEF) { QW_RET(qwAddTaskCtxImpl(QW_FPARAMS(), fal int32_t qwAddAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { return qwAddTaskCtxImpl(QW_FPARAMS(), true, ctx); } -void qwReleaseTaskCtx(SQWorkerMgmt *mgmt, void *ctx) { taosHashRelease(mgmt->ctxHash, ctx); } +void qwReleaseTaskCtx(SQWorker *mgmt, void *ctx) { taosHashRelease(mgmt->ctxHash, ctx); } void qwFreeTaskHandle(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle) { // Note: free/kill may in RC @@ -606,7 +611,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { QW_RET(code); } -int32_t qwGenerateSchHbRsp(SQWorkerMgmt *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { +int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { int32_t taskNum = 0; hbInfo->connInfo = sch->hbConnInfo; @@ -940,7 +945,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex code = qStringToSubplan(qwMsg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { - QW_TASK_ELOG("task string to subplan failed, code:%x - %s", code, tstrerror(code)); + code = TSDB_CODE_INVALID_MSG; + QW_TASK_ELOG("task physical plan to subplan failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); } @@ -1262,7 +1268,7 @@ _return: QW_RET(TSDB_CODE_SUCCESS); } -int32_t qwProcessHbLinkBroken(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { +int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; SQWSchStatus * sch = NULL; @@ -1288,7 +1294,7 @@ int32_t qwProcessHbLinkBroken(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq QW_RET(TSDB_CODE_SUCCESS); } -int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { +int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; SQWSchStatus * sch = NULL; @@ -1333,7 +1339,19 @@ _return: } void qwProcessHbTimerEvent(void *param, void *tmrId) { - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)param; + SQWHbParam* hbParam = (SQWHbParam*)param; + if (hbParam->qwrId != atomic_load_32(&gQwMgmt.qwRef)) { + return; + } + + int64_t refId = hbParam->refId; + SQWorker *mgmt = qwAcquire(refId); + if (NULL == mgmt) { + QW_DLOG("qwAcquire %" PRIx64 "failed", refId); + taosMemoryFree(param); + return; + } + SQWSchStatus *sch = NULL; int32_t taskNum = 0; SQWHbInfo * rspList = NULL; @@ -1347,6 +1365,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { if (schNum <= 0) { QW_UNLOCK(QW_READ, &mgmt->schLock); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); + qwRelease(refId); return; } @@ -1355,6 +1374,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { QW_UNLOCK(QW_READ, &mgmt->schLock); QW_ELOG("calloc %d SQWHbInfo failed", schNum); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); + qwRelease(refId); return; } @@ -1396,6 +1416,74 @@ _return: taosMemoryFreeClear(rspList); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); + qwRelease(refId); +} + +void qwCloseRef(void) { + taosWLockLatch(&gQwMgmt.lock); + if (atomic_load_32(&gQwMgmt.qwNum) <= 0 && gQwMgmt.qwRef >= 0) { + taosCloseRef(gQwMgmt.qwRef); + gQwMgmt.qwRef= -1; + } + taosWUnLockLatch(&gQwMgmt.lock); +} + +void qwDestroyImpl(void *pMgmt) { + SQWorker *mgmt = (SQWorker *)pMgmt; + + taosTmrStopA(&mgmt->hbTimer); + taosTmrCleanUp(mgmt->timer); + + // TODO STOP ALL QUERY + + // TODO FREE ALL + + taosHashCleanup(mgmt->ctxHash); + taosHashCleanup(mgmt->schHash); + + taosMemoryFree(mgmt); + + atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + + qwCloseRef(); +} + +int32_t qwOpenRef(void) { + taosWLockLatch(&gQwMgmt.lock); + if (gQwMgmt.qwRef < 0) { + gQwMgmt.qwRef= taosOpenRef(100, qwDestroyImpl); + if (gQwMgmt.qwRef < 0) { + taosWUnLockLatch(&gQwMgmt.lock); + qError("init qworker ref failed"); + QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + taosWUnLockLatch(&gQwMgmt.lock); + + return TSDB_CODE_SUCCESS; +} + +void qwSetHbParam(int64_t refId, SQWHbParam **pParam) { + int32_t paramIdx = 0; + int32_t newParamIdx = 0; + + while (true) { + paramIdx = atomic_load_32(&gQwMgmt.paramIdx); + if (paramIdx == tListLen(gQwMgmt.param)) { + newParamIdx = 0; + } else { + newParamIdx = paramIdx + 1; + } + + if (paramIdx == atomic_val_compare_exchange_32(&gQwMgmt.paramIdx, paramIdx, newParamIdx)) { + break; + } + } + + gQwMgmt.param[paramIdx].qwrId = gQwMgmt.qwRef; + gQwMgmt.param[paramIdx].refId = refId; + + *pParam = &gQwMgmt.param[paramIdx]; } int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, const SMsgCb *pMsgCb) { @@ -1404,10 +1492,21 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW QW_RET(TSDB_CODE_QRY_INVALID_INPUT); } - int32_t code = 0; - SQWorkerMgmt *mgmt = taosMemoryCalloc(1, sizeof(SQWorkerMgmt)); + int32_t qwNum = atomic_add_fetch_32(&gQwMgmt.qwNum, 1); + if (1 == qwNum) { + memset(gQwMgmt.param, 0, sizeof(gQwMgmt.param)); + } + + int32_t code = qwOpenRef(); + if (code) { + atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + QW_RET(code); + } + + SQWorker *mgmt = taosMemoryCalloc(1, sizeof(SQWorker)); if (NULL == mgmt) { - qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt)); + qError("calloc %d failed", (int32_t)sizeof(SQWorker)); + atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1449,16 +1548,25 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, mgmt, mgmt->timer); - if (NULL == mgmt->hbTimer) { - qError("start hb timer failed"); - QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - mgmt->nodeType = nodeType; mgmt->nodeId = nodeId; mgmt->msgCb = *pMsgCb; + mgmt->refId = taosAddRef(gQwMgmt.qwRef, mgmt); + if (mgmt->refId < 0) { + qError("taosAddRef qw failed, error:%s", tstrerror(terrno)); + QW_ERR_JRET(terrno); + } + + SQWHbParam *param = NULL; + qwSetHbParam(mgmt->refId, ¶m); + + mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, (void*)param, mgmt->timer); + if (NULL == mgmt->hbTimer) { + qError("start hb timer failed"); + QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + *qWorkerMgmt = mgmt; qDebug("qworker initialized for node, type:%d, id:%d, handle:%p", mgmt->nodeType, mgmt->nodeId, mgmt); @@ -1467,13 +1575,17 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW _return: - taosHashCleanup(mgmt->schHash); - taosHashCleanup(mgmt->ctxHash); - - taosTmrCleanUp(mgmt->timer); - - taosMemoryFreeClear(mgmt); + if (mgmt->refId >= 0) { + qwRelease(mgmt->refId); + } else { + taosHashCleanup(mgmt->schHash); + taosHashCleanup(mgmt->ctxHash); + taosTmrCleanUp(mgmt->timer); + taosMemoryFreeClear(mgmt); + atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + } + QW_RET(code); } @@ -1482,22 +1594,14 @@ void qWorkerDestroy(void **qWorkerMgmt) { return; } - SQWorkerMgmt *mgmt = *qWorkerMgmt; + SQWorker *mgmt = *qWorkerMgmt; - taosTmrStopA(&mgmt->hbTimer); - taosTmrCleanUp(mgmt->timer); - - // TODO STOP ALL QUERY - - // TODO FREE ALL - - taosHashCleanup(mgmt->ctxHash); - taosHashCleanup(mgmt->schHash); - - taosMemoryFreeClear(*qWorkerMgmt); + if (taosRemoveRef(gQwMgmt.qwRef, mgmt->refId)) { + qError("remove qw from ref list failed, refId:%" PRIx64, mgmt->refId); + } } -int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp) { +int32_t qwGetSchTasksStatus(SQWorker *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp) { /* SQWSchStatus *sch = NULL; int32_t taskNum = 0; @@ -1544,7 +1648,7 @@ int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRs return TSDB_CODE_SUCCESS; } -int32_t qwUpdateSchLastAccess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { +int32_t qwUpdateSchLastAccess(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { SQWSchStatus *sch = NULL; /* @@ -1557,7 +1661,7 @@ int32_t qwUpdateSchLastAccess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, ui return TSDB_CODE_SUCCESS; } -int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t *taskStatus) { +int32_t qwGetTaskStatus(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t *taskStatus) { SQWSchStatus * sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -1584,7 +1688,7 @@ int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t QW_RET(code); } -int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { +int32_t qwCancelTask(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { SQWSchStatus * sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 64df4f02bf7e826589f8b2b3d61f9972328dacd4..d4f6c2fd004f33018a74f9dd6ea779c5d7cfea8c 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -34,6 +34,7 @@ void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) rsp->compressed = input->compressed; rsp->compLen = htonl(len); rsp->numOfRows = htonl(input->numOfRows); + rsp->numOfCols = htonl(input->numOfCols); } void qwFreeFetchRsp(void *msg) { @@ -319,7 +320,7 @@ int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { return TSDB_CODE_SUCCESS; } -int32_t qwRegisterHbBrokenLinkArg(SQWorkerMgmt *mgmt, uint64_t sId, SQWConnInfo *pConn) { +int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t sId, SQWConnInfo *pConn) { SSchedulerHbReq req = {0}; req.header.vgId = mgmt->nodeId; req.sId = sId; @@ -362,7 +363,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; SSubQueryMsg *msg = pMsg->pCont; - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { QW_ELOG("invalid query msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -404,7 +405,7 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { SQueryContinueReq *msg = (SQueryContinueReq *)pMsg->pCont; bool needStop = false; SQWTaskCtx *handles = NULL; - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid cquery msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -435,7 +436,7 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; SResReadyReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid task ready msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -477,7 +478,7 @@ int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; msg->sId = htobe64(msg->sId); uint64_t sId = msg->sId; @@ -498,7 +499,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { } SResFetchReq *msg = pMsg->pCont; - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -538,7 +539,7 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; int32_t code = 0; STaskCancelReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { @@ -578,7 +579,7 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; STaskDropReq *msg = pMsg->pCont; - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid task drop msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -620,7 +621,7 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; SSchedulerHbReq req = {0}; - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; if (NULL == pMsg->pCont) { QW_ELOG("invalid hb msg, msg:%p, msgLen:%d", pMsg->pCont, pMsg->contLen); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 864894fa42d09816750a9468bb2f9a54f8700835..80e5669cc2b5891f1c73d158097d2b4429117f98 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3765,6 +3765,7 @@ int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); info->scalarMode = stat.scalarMode; + fltDebug("scalar mode: %d", info->scalarMode); if (!info->scalarMode) { FLT_ERR_JRET(fltInitFromNode(pNode, info, options)); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index eb6be4ca3c0feb52d6b308da823b4fe2185fad6e..f5fab814ffdcb8b585e0171e0880944eda5e2557 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -75,7 +75,15 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { if (valueNode->node.resType.type != type) { out.columnData->info.type = type; - out.columnData->info.bytes = tDataTypes[type].bytes; + if (IS_VAR_DATA_TYPE(type)) { + if (IS_VAR_DATA_TYPE(valueNode->node.resType.type)) { + out.columnData->info.bytes = valueNode->node.resType.bytes * TSDB_NCHAR_SIZE; + } else { + out.columnData->info.bytes = 64 * TSDB_NCHAR_SIZE; + } + } else { + out.columnData->info.bytes = tDataTypes[type].bytes; + } code = doConvertDataType(valueNode, &out); if (code != TSDB_CODE_SUCCESS) { @@ -598,7 +606,7 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { res->datum.p = taosMemoryCalloc(res->node.resType.bytes + VARSTR_HEADER_SIZE + 1, 1); memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData)); } else { - memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes); + nodesSetValueNodeValue(res, output.columnData->pData); } } @@ -638,7 +646,7 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { res->datum.p = output.columnData->pData; output.columnData->pData = NULL; } else { - memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes); + nodesSetValueNodeValue(res, output.columnData->pData); } nodesDestroyNode(*pNode); @@ -680,7 +688,7 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { res->datum.p = output.columnData->pData; output.columnData->pData = NULL; } else { - memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes); + nodesSetValueNodeValue(res, output.columnData->pData); } } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 5c1b95e2147eb5bed3da7f0008011d85d12eff1f..77caf67a2bdb274660bca49d03abd0440895ed7e 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -15,7 +15,15 @@ typedef int16_t (*_len_fn)(char *, int32_t); /** Math functions **/ static double tlog(double v, double base) { - return log(v) / log(base); + double a = log(v); + double b = log(base); + if (isnan(a) || isinf(a)) { + return a; + } else if (isnan(b) || isinf(b)) { + return b; + } else { + return a / b; + } } int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { @@ -160,22 +168,64 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S } double *out = (double *)pOutputData->pData; + double result; + + int32_t numOfRows = TMAX(pInput[0].numOfRows, pInput[1].numOfRows); + if (pInput[0].numOfRows == pInput[1].numOfRows) { + for (int32_t i = 0; i < numOfRows; ++i) { + if (colDataIsNull_s(pInputData[0], i) || + colDataIsNull_s(pInputData[1], i)) { + colDataAppendNULL(pOutputData, i); + continue; + } + result = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, i)); + if (isinf(result) || isnan(result)) { + colDataAppendNULL(pOutputData, i); + } else { + out[i] = result; + } + } + } else if (pInput[0].numOfRows == 1) { //left operand is constant + if (colDataIsNull_s(pInputData[0], 0)) { + colDataAppendNNULL(pOutputData, 0, pInput[1].numOfRows); + } else { + for (int32_t i = 0; i < numOfRows; ++i) { + if (colDataIsNull_s(pInputData[1], i)) { + colDataAppendNULL(pOutputData, i); + continue; + } - for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_s(pInputData[0], i) || - colDataIsNull_s(pInputData[1], 0)) { - colDataAppendNULL(pOutputData, i); - continue; + result = valFn(getValueFn[0](pInputData[0]->pData, 0), getValueFn[1](pInputData[1]->pData, i)); + if (isinf(result) || isnan(result)) { + colDataAppendNULL(pOutputData, i); + continue; + } + + out[i] = result; + } } - double result = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, 0)); - if (isinf(result) || isnan(result)) { - colDataAppendNULL(pOutputData, i); + } else if (pInput[1].numOfRows == 1) { + if (colDataIsNull_s(pInputData[1], 0)) { + colDataAppendNNULL(pOutputData, 0, pInput[0].numOfRows); } else { - out[i] = result; + for (int32_t i = 0; i < numOfRows; ++i) { + if (colDataIsNull_s(pInputData[0], i)) { + colDataAppendNULL(pOutputData, i); + continue; + } + + result = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, 0)); + if (isinf(result) || isnan(result)) { + colDataAppendNULL(pOutputData, i); + continue; + } + + out[i] = result; + } } } - pOutput->numOfRows = pInput->numOfRows; + pOutput->numOfRows = numOfRows; return TSDB_CODE_SUCCESS; } @@ -307,7 +357,8 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - int16_t *out = (int16_t *)pOutputData->pData; + ASSERT(pOutputData->info.type == TSDB_DATA_TYPE_BIGINT); + int64_t *out = (int64_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { @@ -658,6 +709,10 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int16_t outputType = GET_PARAM_TYPE(&pOutput[0]); int64_t outputLen = GET_PARAM_BYTES(&pOutput[0]); + if (IS_VAR_DATA_TYPE(outputType)) { + outputLen += VARSTR_HEADER_SIZE; + } + char *outputBuf = taosMemoryCalloc(outputLen * pInput[0].numOfRows, 1); char *output = outputBuf; @@ -739,29 +794,30 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_NCHAR: { int32_t outputCharLen = (outputLen - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + int32_t len; if (inputType == TSDB_DATA_TYPE_BOOL) { char tmp[8] = {0}; - int32_t len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false" ); + len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false" ); bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); if (!ret) { return TSDB_CODE_FAILED; } varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_BINARY) { - int32_t len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen; + len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen; bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); if (!ret) { return TSDB_CODE_FAILED; } varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = TMIN(outputLen, varDataLen(input) + VARSTR_HEADER_SIZE); - memcpy(output, input, len); - varDataSetLen(output, len - VARSTR_HEADER_SIZE); + len = TMIN(outputLen - VARSTR_HEADER_SIZE, varDataLen(input)); + memcpy(output, input, len + VARSTR_HEADER_SIZE); + varDataSetLen(output, len); } else { char tmp[400] = {0}; NUM_TO_STRING(inputType, input, sizeof(tmp), tmp); - int32_t len = (int32_t)strlen(tmp); + len = (int32_t)strlen(tmp); len = outputCharLen > len ? len : outputCharLen; bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); if (!ret) { @@ -769,6 +825,10 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } varDataSetLen(output, len); } + //for constant conversion, need to set proper length of pOutput description + if (len < outputLen - VARSTR_HEADER_SIZE) { + pOutput->columnData->info.bytes = len; + } break; } default: { @@ -1453,3 +1513,9 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t*) colDataGetData(pInput->columnData, 4)); return TSDB_CODE_SUCCESS; } + +int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + ASSERT(inputNum == 1); + colDataAppend(pOutput->columnData, pOutput->numOfRows, colDataGetData(pInput->columnData, 0), false); + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index a75b2521bdded3737b11a6fc931762fef8cb4cb8..1e37533f2c3bc647df43fe9a599dc76ec5fbe8d2 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -297,6 +297,22 @@ static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIn taosMemoryFree(t); } +static FORCE_INLINE void ncharToVar(char* buf, SScalarParam* pOut, int32_t rowIndex) { + int32_t inputLen = varDataLen(buf); + + char* t = taosMemoryCalloc(1, inputLen + VARSTR_HEADER_SIZE); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t)); + if (len < 0) { + taosMemoryFree(t); + return; + } + varDataSetLen(t, len); + + colDataAppend(pOut->columnData, rowIndex, t, false); + taosMemoryFree(t); +} + + //TODO opt performance, tmp is not needed. int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { int32_t bufSize = pIn->columnData->info.bytes; @@ -313,6 +329,10 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in func = varToUnsigned; } else if (IS_FLOAT_TYPE(outType)) { func = varToFloat; + } else if (outType == TSDB_DATA_TYPE_BINARY) { // nchar -> binary + ASSERT(inType == TSDB_DATA_TYPE_NCHAR); + func = ncharToVar; + vton = true; } else if (outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar ASSERT(inType == TSDB_DATA_TYPE_VARCHAR); func = varToNchar; @@ -608,7 +628,7 @@ int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 0, 7, 0, 0, /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, -/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7, 7, 7, 7, 0, 0, 0, 0, +/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 0, 0, 0, /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, 7, 0, 0, /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, @@ -1003,8 +1023,7 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData* colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } - *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - * getVectorDoubleValueFnRight(RIGHT_COL, 0); + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, 0); } } } @@ -1030,8 +1049,7 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } - *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - * getVectorDoubleValueFnRight(RIGHT_COL, i); + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, i); } } else if (pLeft->numOfRows == 1) { vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 0f6961018ce4cbbba9e4129bed2e42a8e3e0d64e..5906ee8970f7725fb10139522eab3d276ef3f7f0 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -85,7 +85,10 @@ typedef struct SSchedulerMgmt { uint64_t taskId; // sequential taksId uint64_t sId; // schedulerId SSchedulerCfg cfg; + SRWLatch lock; + bool exit; int32_t jobRef; + int32_t jobNum; SSchedulerStat stat; SHashObj *hbConnections; } SSchedulerMgmt; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 31d2da93802e18ed00306b909daba44665cc622e..53772601caf36fe3fd4961471420535b3742e6bf 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -21,7 +21,9 @@ #include "tref.h" #include "trpc.h" -SSchedulerMgmt schMgmt = {0}; +SSchedulerMgmt schMgmt = { + .jobRef = -1, +}; FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); } @@ -70,6 +72,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray *pNodeList, const char *sql, int64_t startTs, bool syncSchedule) { int32_t code = 0; + int64_t refId = -1; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); @@ -114,15 +117,17 @@ int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray tsem_init(&pJob->rspSem, 0, 0); - int64_t refId = taosAddRef(schMgmt.jobRef, pJob); + refId = taosAddRef(schMgmt.jobRef, pJob); if (refId < 0) { SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); SCH_ERR_JRET(terrno); } + atomic_add_fetch_32(&schMgmt.jobNum, 1); + if (NULL == schAcquireJob(refId)) { SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId); - SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } pJob->refId = refId; @@ -137,7 +142,11 @@ int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray _return: - schFreeJobImpl(pJob); + if (refId < 0) { + schFreeJobImpl(pJob); + } else { + taosRemoveRef(schMgmt.jobRef, refId); + } SCH_RET(code); } @@ -245,6 +254,7 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); return TSDB_CODE_SUCCESS; case TDMT_VND_CREATE_TABLE_RSP: + case TDMT_VND_DROP_TABLE_RSP: case TDMT_VND_SUBMIT_RSP: break; default: @@ -369,7 +379,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } for (int32_t n = 0; n < childNum; ++n) { - SSubplan * child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n); + SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n); SSchTask **childTask = taosHashGet(planToTask, &child, POINTER_BYTES); if (NULL == childTask || NULL == *childTask) { SCH_TASK_ELOG("subplan children relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -401,7 +411,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } for (int32_t n = 0; n < parentNum; ++n) { - SSubplan * parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n); + SSubplan *parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n); SSchTask **parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES); if (NULL == parentTask || NULL == *parentTask) { SCH_TASK_ELOG("subplan parent relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -491,7 +501,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SSchLevel level = {0}; SNodeListNode *plans = NULL; int32_t taskNum = 0; - SSchLevel * pLevel = NULL; + SSchLevel *pLevel = NULL; level.status = JOB_TASK_STATUS_NOT_START; @@ -1075,19 +1085,47 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch case TDMT_VND_CREATE_TABLE_RSP: { SVCreateTbBatchRsp batchRsp = {0}; if (msg) { - SCH_ERR_JRET(tDeserializeSVCreateTbBatchRsp(msg, msgSize, &batchRsp)); - if (batchRsp.pArray) { - int32_t num = taosArrayGetSize(batchRsp.pArray); - for (int32_t i = 0; i < num; ++i) { - SVCreateTbRsp *rsp = taosArrayGet(batchRsp.pArray, i); + SCoder coder = {0}; + tCoderInit(&coder, TD_LITTLE_ENDIAN, msg, msgSize, TD_DECODER); + code = tDecodeSVCreateTbBatchRsp(&coder, &batchRsp); + if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) { + for (int32_t i = 0; i < batchRsp.nRsps; ++i) { + SVCreateTbRsp *rsp = batchRsp.pRsps + i; if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) { - taosArrayDestroy(batchRsp.pArray); + tCoderClear(&coder); SCH_ERR_JRET(rsp->code); + } else if (TSDB_CODE_SUCCESS != rsp->code) { + code = rsp->code; } } + } + tCoderClear(&coder); + SCH_ERR_JRET(code); + } - taosArrayDestroy(batchRsp.pArray); + SCH_ERR_JRET(rspCode); + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + break; + } + case TDMT_VND_DROP_TABLE_RSP: { + SVDropTbBatchRsp batchRsp = {0}; + if (msg) { + SCoder coder = {0}; + tCoderInit(&coder, TD_LITTLE_ENDIAN, msg, msgSize, TD_DECODER); + code = tDecodeSVDropTbBatchRsp(&coder, &batchRsp); + if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) { + for (int32_t i = 0; i < batchRsp.nRsps; ++i) { + SVDropTbRsp *rsp = batchRsp.pRsps + i; + if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) { + tCoderClear(&coder); + SCH_ERR_JRET(rsp->code); + } else if (TSDB_CODE_SUCCESS != rsp->code) { + code = rsp->code; + } + } } + tCoderClear(&coder); + SCH_ERR_JRET(code); } SCH_ERR_JRET(rspCode); @@ -1267,7 +1305,7 @@ int32_t schUpdateTaskExecNodeHandle(SSchTask *pTask, void *handle, int32_t rspCo int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode) { int32_t code = 0; SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - SSchTask * pTask = NULL; + SSchTask *pTask = NULL; SSchJob *pJob = schAcquireJob(pParam->refId); if (NULL == pJob) { @@ -1316,6 +1354,10 @@ int32_t schHandleCreateTableCallback(void *param, const SDataBuf *pMsg, int32_t return schHandleCallback(param, pMsg, TDMT_VND_CREATE_TABLE_RSP, code); } +int32_t schHandleDropTableCallback(void *param, const SDataBuf *pMsg, int32_t code) { + return schHandleCallback(param, pMsg, TDMT_VND_DROP_TABLE_RSP, code); +} + int32_t schHandleQueryCallback(void *param, const SDataBuf *pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_QUERY_RSP, code); } @@ -1412,6 +1454,9 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { case TDMT_VND_CREATE_TABLE: *fp = schHandleCreateTableCallback; break; + case TDMT_VND_DROP_TABLE: + *fp = schHandleDropTableCallback; + break; case TDMT_VND_SUBMIT: *fp = schHandleSubmitCallback; break; @@ -1617,8 +1662,8 @@ _return: int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { int32_t code = 0; SSchHbCallbackParam *param = NULL; - SMsgSendInfo * pMsgSendInfo = NULL; - SQueryNodeAddr * addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SMsgSendInfo *pMsgSendInfo = NULL; + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; @@ -1759,10 +1804,10 @@ int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { } SRpcCtxVal dst = {0}; - void * pIter = taosHashIterate(pSrc->args, NULL); + void *pIter = taosHashIterate(pSrc->args, NULL); while (pIter) { SRpcCtxVal *pVal = (SRpcCtxVal *)pIter; - int32_t * msgType = taosHashGetKey(pIter, NULL); + int32_t *msgType = taosHashGetKey(pIter, NULL); dst = *pVal; dst.val = NULL; @@ -1916,7 +1961,7 @@ _return: int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { uint32_t msgSize = 0; - void * msg = NULL; + void *msg = NULL; int32_t code = 0; bool isCandidateAddr = false; bool persistHandle = false; @@ -1931,6 +1976,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, switch (msgType) { case TDMT_VND_CREATE_TABLE: + case TDMT_VND_DROP_TABLE: case TDMT_VND_SUBMIT: { msgSize = pTask->msgLen; msg = taosMemoryCalloc(1, msgSize); @@ -2239,6 +2285,19 @@ int32_t schCancelJob(SSchJob *pJob) { // TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST } +void schCloseJobRef(void) { + if (!atomic_load_8((int8_t *)&schMgmt.exit)) { + return; + } + + SCH_LOCK(SCH_WRITE, &schMgmt.lock); + if (atomic_load_32(&schMgmt.jobNum) <= 0 && schMgmt.jobRef >= 0) { + taosCloseRef(schMgmt.jobRef); + schMgmt.jobRef = -1; + } + SCH_UNLOCK(SCH_WRITE, &schMgmt.lock); +} + void schFreeJobImpl(void *job) { if (NULL == job) { return; @@ -2284,6 +2343,10 @@ void schFreeJobImpl(void *job) { taosMemoryFreeClear(pJob); qDebug("QID:0x%" PRIx64 " job freed, refId:%" PRIx64 ", pointer:%p", queryId, refId, pJob); + + atomic_sub_fetch_32(&schMgmt.jobNum, 1); + + schCloseJobRef(); } static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, @@ -2368,7 +2431,7 @@ _return: } int32_t schedulerInit(SSchedulerCfg *cfg) { - if (schMgmt.jobRef) { + if (schMgmt.jobRef >= 0) { qError("scheduler already initialized"); return TSDB_CODE_QRY_INVALID_INPUT; } @@ -2673,7 +2736,7 @@ int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); for (int32_t m = 0; m < pLevel->taskNum; ++m) { - SSchTask * pTask = taosArrayGet(pLevel->subTasks, m); + SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); SQuerySubDesc subDesc = {.tid = pTask->taskId, .status = pTask->status}; taosArrayPush(pSub, &subDesc); @@ -2732,7 +2795,9 @@ void schedulerFreeTaskList(SArray *taskList) { } void schedulerDestroy(void) { - if (schMgmt.jobRef) { + atomic_store_8((int8_t *)&schMgmt.exit, 1); + + if (schMgmt.jobRef >= 0) { SSchJob *pJob = taosIterateRef(schMgmt.jobRef, 0); int64_t refId = 0; @@ -2745,9 +2810,6 @@ void schedulerDestroy(void) { pJob = taosIterateRef(schMgmt.jobRef, refId); } - - taosCloseRef(schMgmt.jobRef); - schMgmt.jobRef = 0; } if (schMgmt.hbConnections) { diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c index 8aaaa414ca265f31f09f26f877b2ec3dea7ce5b3..b8198364c24965074c2c3a44653dd1f0cdd78558 100644 --- a/source/libs/stream/src/tstream.c +++ b/source/libs/stream/src/tstream.c @@ -28,6 +28,7 @@ static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* if (buf == NULL) { return -1; } + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { ((SMsgHead*)buf)->vgId = 0; req.taskId = pTask->inplaceDispatcher.taskId; @@ -151,15 +152,15 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in // sink if (pTask->sinkType == TASK_SINK__TABLE) { - // - blockDebugShowData(pRes); + /*blockDebugShowData(pRes);*/ + ASSERT(pTask->tbSink.pTSchema); + SSubmitReq* pReq = tdBlockToSubmit(pRes, pTask->tbSink.pTSchema); + tPrintFixedSchemaSubmitReq(pReq, pTask->tbSink.pTSchema); } else if (pTask->sinkType == TASK_SINK__SMA) { pTask->smaSink.smaHandle(pTask->ahandle, pTask->smaSink.smaId, pRes); // } else if (pTask->sinkType == TASK_SINK__FETCH) { // - } else if (pTask->sinkType == TASK_SINK__SHOW) { - blockDebugShowData(pRes); } else { ASSERT(pTask->sinkType == TASK_SINK__NONE); } @@ -275,13 +276,12 @@ int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask) { } if (pTask->sinkType == TASK_SINK__TABLE) { - if (tEncodeI8(pEncoder, pTask->tbSink.reserved) < 0) return -1; + /*if (tEncodeI8(pEncoder, pTask->tbSink.reserved) < 0) return -1;*/ + if (tEncodeSSchemaWrapper(pEncoder, pTask->tbSink.pSchemaWrapper) < 0) return -1; } else if (pTask->sinkType == TASK_SINK__SMA) { if (tEncodeI64(pEncoder, pTask->smaSink.smaId) < 0) return -1; } else if (pTask->sinkType == TASK_SINK__FETCH) { if (tEncodeI8(pEncoder, pTask->fetchSink.reserved) < 0) return -1; - } else if (pTask->sinkType == TASK_SINK__SHOW) { - if (tEncodeI8(pEncoder, pTask->showSink.reserved) < 0) return -1; } else { ASSERT(pTask->sinkType == TASK_SINK__NONE); } @@ -321,13 +321,14 @@ int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask) { } if (pTask->sinkType == TASK_SINK__TABLE) { - if (tDecodeI8(pDecoder, &pTask->tbSink.reserved) < 0) return -1; + /*if (tDecodeI8(pDecoder, &pTask->tbSink.reserved) < 0) return -1;*/ + pTask->tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); + if (pTask->tbSink.pSchemaWrapper == NULL) return -1; + if (tDecodeSSchemaWrapper(pDecoder, pTask->tbSink.pSchemaWrapper) < 0) return -1; } else if (pTask->sinkType == TASK_SINK__SMA) { if (tDecodeI64(pDecoder, &pTask->smaSink.smaId) < 0) return -1; } else if (pTask->sinkType == TASK_SINK__FETCH) { if (tDecodeI8(pDecoder, &pTask->fetchSink.reserved) < 0) return -1; - } else if (pTask->sinkType == TASK_SINK__SHOW) { - if (tDecodeI8(pDecoder, &pTask->showSink.reserved) < 0) return -1; } else { ASSERT(pTask->sinkType == TASK_SINK__NONE); } diff --git a/source/libs/stream/src/tstreamUpdate.c b/source/libs/stream/src/tstreamUpdate.c index a207ec92650c8ddb259df575365f476f6e424fc3..850a17b6dd465565fb92ce8a02292d37aea5165e 100644 --- a/source/libs/stream/src/tstreamUpdate.c +++ b/source/libs/stream/src/tstreamUpdate.c @@ -54,15 +54,28 @@ static int64_t adjustInterval(int64_t interval, int32_t precision) { if (precision != TSDB_TIME_PRECISION_MILLI) { val = convertTimePrecision(interval, precision, TSDB_TIME_PRECISION_MILLI); } - if (val < MIN_INTERVAL) { - val = MIN_INTERVAL; - } else if (val > MAX_INTERVAL) { + + if (val <= 0 || val > MAX_INTERVAL) { val = MAX_INTERVAL; + } else if (val < MIN_INTERVAL) { + val = MIN_INTERVAL; + } + + if (precision != TSDB_TIME_PRECISION_MILLI) { + val = convertTimePrecision(val, TSDB_TIME_PRECISION_MILLI, precision); } - val = convertTimePrecision(val, TSDB_TIME_PRECISION_MILLI, precision); return val; } +static int64_t adjustWatermark(int64_t interval, int32_t watermark) { + if (watermark <= 0 || watermark > MAX_NUM_SCALABLE_BF * interval) { + watermark = MAX_NUM_SCALABLE_BF * interval; + } else if (watermark < MIN_NUM_SCALABLE_BF * interval) { + watermark = MIN_NUM_SCALABLE_BF * interval; + } + return watermark; +} + SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark) { return updateInfoInit(pInterval->interval, pInterval->precision, watermark); } @@ -76,14 +89,9 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma pInfo->pTsSBFs = NULL; pInfo->minTS = -1; pInfo->interval = adjustInterval(interval, precision); - pInfo->watermark = watermark; + pInfo->watermark = adjustWatermark(pInfo->interval, watermark); - uint64_t bfSize = (uint64_t)(watermark / pInfo->interval); - if (bfSize < MIN_NUM_SCALABLE_BF) { - bfSize = MIN_NUM_SCALABLE_BF; - } else if (bfSize > MAX_NUM_SCALABLE_BF) { - bfSize = MAX_NUM_SCALABLE_BF; - } + uint64_t bfSize = (uint64_t)(pInfo->watermark / pInfo->interval); pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(SScalableBf)); if (pInfo->pTsSBFs == NULL) { diff --git a/source/libs/stream/test/tstreamUpdateTest.cpp b/source/libs/stream/test/tstreamUpdateTest.cpp index ed016ead44e21ef2fd84e87f4686d0feb1d6b315..32ed974f727827b7bd7cd1dbe0e81c0acbb6214c 100644 --- a/source/libs/stream/test/tstreamUpdateTest.cpp +++ b/source/libs/stream/test/tstreamUpdateTest.cpp @@ -90,11 +90,21 @@ TEST(TD_STREAM_UPDATE_TEST, update) { } } + SUpdateInfo *pSU4 = updateInfoInit(-1, TSDB_TIME_PRECISION_MILLI, -1); + GTEST_ASSERT_EQ(pSU4->watermark, 120 * pSU4->interval); + GTEST_ASSERT_EQ(pSU4->interval, MILLISECOND_PER_MINUTE); + + SUpdateInfo *pSU5 = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); + GTEST_ASSERT_EQ(pSU5->watermark, 120 * pSU4->interval); + GTEST_ASSERT_EQ(pSU5->interval, MILLISECOND_PER_MINUTE); + updateInfoDestroy(pSU); updateInfoDestroy(pSU1); updateInfoDestroy(pSU2); updateInfoDestroy(pSU3); + updateInfoDestroy(pSU4); + updateInfoDestroy(pSU5); } int main(int argc, char* argv[]) { diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 4b2bc4130ab7ae9988cb927446e680d6352702ca..945d59646b08ae303c5ffa36f6fb4d3a1fb85452 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -37,7 +37,7 @@ bool syncEnvIsStart() { int32_t syncEnvStart() { int32_t ret = 0; taosSeedRand(taosGetTimestampSec()); - //gSyncEnv = doSyncEnvStart(gSyncEnv); + // gSyncEnv = doSyncEnvStart(gSyncEnv); gSyncEnv = doSyncEnvStart(); assert(gSyncEnv != NULL); sTrace("syncEnvStart ok!"); @@ -97,14 +97,14 @@ static SSyncEnv *doSyncEnvStart() { // start tmr thread pSyncEnv->pTimerManager = taosTmrInit(1000, 50, 10000, "SYNC-ENV"); - atomic_store_8(&(pSyncEnv->isStart), 1); + atomic_store_8(&(pSyncEnv->isStart), 1); return pSyncEnv; } static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { assert(pSyncEnv == gSyncEnv); if (pSyncEnv != NULL) { - atomic_store_8(&(pSyncEnv->isStart), 0); + atomic_store_8(&(pSyncEnv->isStart), 0); taosTmrCleanUp(pSyncEnv->pTimerManager); taosMemoryFree(pSyncEnv); } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 262c9d55309da56df9364d00399b2fb15e0258e8..9a8dcc57d904aceb581b159aae755ba2170fbdfe 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -141,6 +141,18 @@ const char* syncGetMyRoleStr(int64_t rid) { return s; } +int32_t syncGetVgId(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_STATE_ERROR; + } + assert(rid == pSyncNode->rid); + int32_t vgId = pSyncNode->vgId; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return vgId; +} + SyncTerm syncGetMyTerm(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -153,6 +165,29 @@ SyncTerm syncGetMyTerm(int64_t rid) { return term; } +void syncGetEpSet(int64_t rid, SEpSet* pEpSet) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + memset(pEpSet, 0, sizeof(*pEpSet)); + return; + } + assert(rid == pSyncNode->rid); + pEpSet->numOfEps = 0; + for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { + snprintf(pEpSet->eps[i].fqdn, sizeof(pEpSet->eps[i].fqdn), "%s", (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodeFqdn); + pEpSet->eps[i].port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; + (pEpSet->numOfEps)++; + + sInfo("syncGetEpSet index:%d %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + + } + pEpSet->inUse = pSyncNode->pRaftCfg->cfg.myIndex; + + sInfo("syncGetEpSet pEpSet->inUse:%d ", pEpSet->inUse); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); +} + int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -260,6 +295,8 @@ void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS) { } int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { + sTrace("syncPropose msgType:%d ", pMsg->msgType); + int32_t ret = TAOS_SYNC_PROPOSE_SUCCESS; SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -459,18 +496,17 @@ void syncNodeStart(SSyncNode* pSyncNode) { // start raft if (pSyncNode->replicaNum == 1) { syncNodeBecomeLeader(pSyncNode); - + syncNodeLog2("==state change become leader immediately==", pSyncNode); - + // Raft 3.6.2 Committing entries from previous terms - + // use this now syncNodeAppendNoop(pSyncNode); syncMaybeAdvanceCommitIndex(pSyncNode); // maybe only one replica return; } - syncNodeBecomeFollower(pSyncNode); // for test diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 71ac3d97ed87eea6b81b7a16ecddb206a66b0cb0..18f8ddec5c8994392db5ad8d6ba42f793b78782b 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,6 +22,65 @@ extern "C" { #endif +typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); + +// exposed types +typedef struct STEnv TENV; +typedef struct STDB TDB; +typedef struct STDBC TDBC; +typedef struct STxn TXN; + +// TENV +int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv); +int tdbEnvClose(TENV *pEnv); +int tdbBegin(TENV *pEnv, TXN *pTxn); +int tdbCommit(TENV *pEnv, TXN *pTxn); + +// TDB +int tdbDbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb); +int tdbDbClose(TDB *pDb); +int tdbDbDrop(TDB *pDb); +int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn); +int tdbDbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); + +// TDBC +int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn); +int tdbDbcClose(TDBC *pDbc); +int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c); +int tdbDbcMoveToFirst(TDBC *pDbc); +int tdbDbcMoveToLast(TDBC *pDbc); +int tdbDbcMoveToNext(TDBC *pDbc); +int tdbDbcMoveToPrev(TDBC *pDbc); +int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); +int tdbDbcDelete(TDBC *pDbc); +int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert); + +// TXN +#define TDB_TXN_WRITE 0x1 +#define TDB_TXN_READ_UNCOMMITTED 0x2 + +int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, + int flags); +int tdbTxnClose(TXN *pTxn); + +// other +void tdbFree(void *); + +struct STxn { + int flags; + int64_t txnId; + void *(*xMalloc)(void *, size_t); + void (*xFree)(void *, void *); + void *xArg; +}; + +// error code +enum { TDB_CODE_SUCCESS = 0, TDB_CODE_MAX }; + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0cee494ae8e3fa6be5c1f2607dfd0b1066c3d568..0d13ec4d2b6d205d16238a41213c457583a1b953 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -19,17 +19,17 @@ #define TDB_BTREE_LEAF 0x2 struct SBTree { - SPgno root; - int keyLen; - int valLen; - SPager *pPager; - FKeyComparator kcmpr; - int pageSize; - int maxLocal; - int minLocal; - int maxLeaf; - int minLeaf; - void *pBuf; + SPgno root; + int keyLen; + int valLen; + SPager *pPager; + tdb_cmpr_fn_t kcmpr; + int pageSize; + int maxLocal; + int minLocal; + int maxLeaf; + int minLeaf; + void *pBuf; }; #define TDB_BTREE_PAGE_COMMON_HDR u8 flags; @@ -58,16 +58,6 @@ typedef struct { SBTree *pBt; } SBtreeInitPageArg; -typedef struct { - int kLen; - const u8 *pKey; - int vLen; - const u8 *pVal; - SPgno pgno; - u8 *pBuf; -} SCellDecoder; - -static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst); static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); static int tdbBtreeOpenImpl(SBTree *pBt); static int tdbBtreeInitPage(SPage *pPage, void *arg, int init); @@ -76,11 +66,10 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); static int tdbBtreeBalance(SBTC *pBtc); static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); -static int tdbBtcMoveToNext(SBTC *pBtc); static int tdbBtcMoveDownward(SBTC *pBtc); static int tdbBtcMoveUpward(SBTC *pBtc); -int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { +int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, tdb_cmpr_fn_t kcmpr, SBTree **ppBt) { SBTree *pBt; int ret; @@ -149,67 +138,90 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in } if (btc.idx == -1) { - idx = 0; + btc.idx = 0; } else { if (c > 0) { - idx = btc.idx + 1; - } else if (c < 0) { - idx = btc.idx; - } else { - // TDB does NOT allow same key - tdbBtcClose(&btc); + btc.idx++; + } else if (c == 0) { + // dup key not allowed ASSERT(0); return -1; } } - // make sure enough space to hold the cell - szBuf = kLen + vLen + 14; - pBuf = TDB_REALLOC(pBt->pBuf, pBt->pageSize > szBuf ? szBuf : pBt->pageSize); - if (pBuf == NULL) { - tdbBtcClose(&btc); + ret = tdbBtcUpsert(&btc, pKey, kLen, pVal, vLen, 1); + if (ret < 0) { ASSERT(0); + tdbBtcClose(&btc); return -1; } - pBt->pBuf = pBuf; - pCell = (SCell *)pBt->pBuf; - // encode cell - ret = tdbBtreeEncodeCell(btc.pPage, pKey, kLen, pVal, vLen, pCell, &szCell); + tdbBtcClose(&btc); + return 0; +} + +int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { + SBTC btc; + int c; + int ret; + + tdbBtcOpen(&btc, pBt, pTxn); + + // move the cursor + ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); if (ret < 0) { tdbBtcClose(&btc); ASSERT(0); return -1; } - // mark the page dirty - ret = tdbPagerWrite(pBt->pPager, btc.pPage); - if (ret < 0) { + if (btc.idx < 0 || c != 0) { tdbBtcClose(&btc); - ASSERT(0); return -1; } - // insert the cell - ret = tdbPageInsertCell(btc.pPage, idx, pCell, szCell, 0); - if (ret < 0) { + // delete the key + if (tdbBtcDelete(&btc) < 0) { tdbBtcClose(&btc); + return -1; + } + + tdbBtcClose(&btc); + return 0; +} + +int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, int nData, TXN *pTxn) { + SBTC btc; + int c; + int ret; + + tdbBtcOpen(&btc, pBt, pTxn); + + // move the cursor + ret = tdbBtcMoveTo(&btc, pKey, nKey, &c); + if (ret < 0) { ASSERT(0); + tdbBtcClose(&btc); return -1; } - // check if need balance - if (btc.pPage->nOverflow > 0) { - ret = tdbBtreeBalance(&btc); - if (ret < 0) { - tdbBtcClose(&btc); - ASSERT(0); - return -1; + if (btc.idx == -1) { + btc.idx = 0; + c = 1; + } else { + if (c > 0) { + btc.idx = btc.idx + 1; } } - tdbBtcClose(&btc); + ret = tdbBtcUpsert(&btc, pKey, nKey, pData, nData, c); + if (ret < 0) { + ASSERT(0); + tdbBtcClose(&btc); + return -1; + } + tdbBtcClose(&btc); return 0; } @@ -243,7 +255,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL tdbBtreeDecodeCell(btc.pPage, pCell, &cd); if (ppKey) { - pTKey = TDB_REALLOC(*ppKey, cd.kLen); + pTKey = tdbRealloc(*ppKey, cd.kLen); if (pTKey == NULL) { tdbBtcClose(&btc); ASSERT(0); @@ -255,7 +267,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL } if (ppVal) { - pTVal = TDB_REALLOC(*ppVal, cd.vLen); + pTVal = tdbRealloc(*ppVal, cd.vLen); if (pTVal == NULL) { tdbBtcClose(&btc); ASSERT(0); @@ -563,14 +575,14 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx SCell *pCell; int szLCell, szRCell; + // balance page (iNew) and (iNew-1) for (;;) { pCell = tdbPageGetCell(pOlds[infoNews[iNew - 1].iPage], infoNews[iNew - 1].oIdx); - if (childNotLeaf) { - szLCell = szRCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); + szLCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); + if (!childNotLeaf) { + szRCell = szLCell; } else { - szLCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); - int iPage = infoNews[iNew - 1].iPage; int oIdx = infoNews[iNew - 1].oIdx + 1; SPage *pPage; @@ -747,6 +759,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } } + if (TDB_BTREE_PAGE_IS_ROOT(pParent) && TDB_PAGE_TOTAL_CELLS(pParent) == 0) { + i8 flags = TDB_BTREE_ROOT | TDB_BTREE_PAGE_IS_LEAF(pNews[0]); + // copy content to the parent page + tdbBtreeInitPage(pParent, &(SBtreeInitPageArg){.flags = flags, .pBt = pBt}, 0); + tdbPageCopy(pNews[0], pParent); + } + for (int i = 0; i < 3; i++) { if (pDivCell[i]) { tdbOsFree(pDivCell[i]); @@ -1018,6 +1037,7 @@ int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn) { pBtc->iPage = -1; pBtc->pPage = NULL; pBtc->idx = -1; + memset(&pBtc->coder, 0, sizeof(SCellDecoder)); if (pTxn == NULL) { pBtc->pTxn = &pBtc->txn; @@ -1060,6 +1080,8 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { return 0; } } else { + ASSERT(0); +#if 0 // move from a position int iPage = 0; @@ -1077,6 +1099,7 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { tdbBtcMoveUpward(pBtc); } +#endif } // move downward @@ -1125,6 +1148,8 @@ int tdbBtcMoveToLast(SBTC *pBtc) { return 0; } } else { + ASSERT(0); +#if 0 int iPage = 0; // downward search @@ -1147,6 +1172,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) { tdbBtcMoveUpward(pBtc); } +#endif } // move downward @@ -1185,7 +1211,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd); - pKey = TDB_REALLOC(*ppKey, cd.kLen); + pKey = tdbRealloc(*ppKey, cd.kLen); if (pKey == NULL) { return -1; } @@ -1196,9 +1222,9 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { if (ppVal) { // TODO: vLen may be zero - pVal = TDB_REALLOC(*ppVal, cd.vLen); + pVal = tdbRealloc(*ppVal, cd.vLen); if (pVal == NULL) { - TDB_FREE(pKey); + tdbFree(pKey); return -1; } @@ -1216,7 +1242,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { return 0; } -static int tdbBtcMoveToNext(SBTC *pBtc) { +int tdbBtcMoveToNext(SBTC *pBtc) { int nCells; int ret; SCell *pCell; @@ -1262,6 +1288,43 @@ static int tdbBtcMoveToNext(SBTC *pBtc) { return 0; } +int tdbBtcMoveToPrev(SBTC *pBtc) { + if (pBtc->idx < 0) return -1; + + pBtc->idx--; + if (pBtc->idx >= 0) { + return 0; + } + + // move upward + for (;;) { + if (pBtc->iPage == 0) { + pBtc->idx = -1; + return 0; + } + + tdbBtcMoveUpward(pBtc); + pBtc->idx--; + if (pBtc->idx >= 0) { + break; + } + } + + // move downward + for (;;) { + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) break; + + tdbBtcMoveDownward(pBtc); + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage) - 1; + } else { + pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + } + } + + return 0; +} + static int tdbBtcMoveDownward(SBTC *pBtc) { int ret; SPgno pgno; @@ -1307,17 +1370,174 @@ static int tdbBtcMoveUpward(SBTC *pBtc) { return 0; } -static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { - int ret; - int nCells; - int c; - SBTree *pBt; - SCell *pCell; - SPager *pPager; - SCellDecoder cd = {0}; +int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen) { + SCell *pCell; - pBt = pBtc->pBt; - pPager = pBt->pPager; + if (pBtc->idx < 0 || pBtc->idx >= TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { + return -1; + } + + pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); + tdbBtreeDecodeCell(pBtc->pPage, pCell, &pBtc->coder); + + if (ppKey) { + *ppKey = (void *)pBtc->coder.pKey; + *kLen = pBtc->coder.kLen; + } + + if (ppVal) { + *ppVal = (void *)pBtc->coder.pVal; + *vLen = pBtc->coder.vLen; + } + + return 0; +} + +int tdbBtcDelete(SBTC *pBtc) { + int idx = pBtc->idx; + int nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + SPager *pPager = pBtc->pBt->pPager; + const void *pKey; + i8 iPage; + SPage *pPage; + SPgno pgno; + SCell *pCell; + int szCell; + int nKey; + int ret; + + ASSERT(idx >= 0 && idx < nCells); + + // drop the cell on the leaf + ret = tdbPagerWrite(pPager, pBtc->pPage); + if (ret < 0) { + ASSERT(0); + return -1; + } + + tdbPageDropCell(pBtc->pPage, idx); + + // update interior page or do balance + if (idx == nCells - 1) { + if (idx) { + pBtc->idx--; + tdbBtcGet(pBtc, &pKey, &nKey, NULL, NULL); + + // loop to update the interial page + pgno = TDB_PAGE_PGNO(pBtc->pPage); + for (iPage = pBtc->iPage - 1; iPage >= 0; iPage--) { + pPage = pBtc->pgStack[iPage]; + idx = pBtc->idxStack[iPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + + if (idx < nCells) { + ret = tdbPagerWrite(pPager, pPage); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // update the cell with new key + pCell = tdbOsMalloc(nKey + 9); + tdbBtreeEncodeCell(pPage, pKey, nKey, &pgno, sizeof(pgno), pCell, &szCell); + + ret = tdbPageUpdateCell(pPage, idx, pCell, szCell); + if (ret < 0) { + tdbOsFree(pCell); + ASSERT(0); + return -1; + } + tdbOsFree(pCell); + break; + } else { + pgno = TDB_PAGE_PGNO(pPage); + } + } + } else { + // delete the leaf page and do balance + ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0); + + ret = tdbBtreeBalance(pBtc); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + } + + return 0; +} + +int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int nData, int insert) { + SCell *pCell; + int szCell; + int nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + int szBuf; + void *pBuf; + int ret; + + ASSERT(pBtc->idx >= 0); + + // alloc space + szBuf = kLen + nData + 14; + pBuf = tdbRealloc(pBtc->pBt->pBuf, pBtc->pBt->pageSize > szBuf ? szBuf : pBtc->pBt->pageSize); + if (pBuf == NULL) { + ASSERT(0); + return -1; + } + pBtc->pBt->pBuf = pBuf; + pCell = (SCell *)pBtc->pBt->pBuf; + + // encode cell + ret = tdbBtreeEncodeCell(pBtc->pPage, pKey, kLen, pData, nData, pCell, &szCell); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // mark dirty + ret = tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // insert or update + if (insert) { + ASSERT(pBtc->idx <= nCells); + + ret = tdbPageInsertCell(pBtc->pPage, pBtc->idx, pCell, szCell, 0); + } else { + ASSERT(pBtc->idx < nCells); + + ret = tdbPageUpdateCell(pBtc->pPage, pBtc->idx, pCell, szCell); + } + if (ret < 0) { + ASSERT(0); + return -1; + } + + // check balance + if (pBtc->pPage->nOverflow > 0) { + ret = tdbBtreeBalance(pBtc); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + + return 0; +} + +int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { + int ret; + int nCells; + int c; + SCell *pCell; + SBTree *pBt = pBtc->pBt; + SPager *pPager = pBt->pPager; + const void *pTKey; + int tkLen; if (pBtc->iPage < 0) { // move from a clear cursor @@ -1334,6 +1554,8 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { // for empty tree, just return with an invalid position if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) return 0; } else { + ASSERT(0); +#if 0 SPage *pPage; int idx; int iPage = 0; @@ -1368,11 +1590,12 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { if (pBtc->iPage == iPage) break; tdbBtcMoveUpward(pBtc); } +#endif } // search downward to the leaf for (;;) { - int lidx, ridx, midx; + int lidx, ridx; SPage *pPage; pPage = pBtc->pPage; @@ -1381,13 +1604,11 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { ridx = nCells - 1; ASSERT(nCells > 0); - ASSERT(pBtc->idx == -1); // compare first cell - midx = lidx; - pCell = tdbPageGetCell(pPage, midx); - tdbBtreeDecodeCell(pPage, pCell, &cd); - c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen); + pBtc->idx = lidx; + tdbBtcGet(pBtc, &pTKey, &tkLen, NULL, NULL); + c = pBt->kcmpr(pKey, kLen, pTKey, tkLen); if (c <= 0) { ridx = lidx - 1; } else { @@ -1396,10 +1617,9 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { // compare last cell if (lidx <= ridx) { - midx = ridx; - pCell = tdbPageGetCell(pPage, midx); - tdbBtreeDecodeCell(pPage, pCell, &cd); - c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen); + pBtc->idx = ridx; + tdbBtcGet(pBtc, &pTKey, &tkLen, NULL, NULL); + c = pBt->kcmpr(pKey, kLen, pTKey, tkLen); if (c >= 0) { lidx = ridx + 1; } else { @@ -1411,24 +1631,15 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { for (;;) { if (lidx > ridx) break; - midx = (lidx + ridx) >> 1; - - pCell = tdbPageGetCell(pPage, midx); - ret = tdbBtreeDecodeCell(pPage, pCell, &cd); - if (ret < 0) { - // TODO: handle error - ASSERT(0); - return -1; - } - - // Compare the key values - c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen); + pBtc->idx = (lidx + ridx) >> 1; + tdbBtcGet(pBtc, &pTKey, &tkLen, NULL, NULL); + c = pBt->kcmpr(pKey, kLen, pTKey, tkLen); if (c < 0) { // pKey < cd.pKey - ridx = midx - 1; + ridx = pBtc->idx - 1; } else if (c > 0) { // pKey > cd.pKey - lidx = midx + 1; + lidx = pBtc->idx + 1; } else { // pKey == cd.pKey break; @@ -1437,14 +1648,11 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { // keep search downward or break if (TDB_BTREE_PAGE_IS_LEAF(pPage)) { - pBtc->idx = midx; *pCRst = c; break; } else { - if (c <= 0) { - pBtc->idx = midx; - } else { - pBtc->idx = midx + 1; + if (c > 0) { + pBtc->idx += 1; } tdbBtcMoveDownward(pBtc); } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index f4468e7e827da8b8455db921df2b027e0b778d3a..553bb2c64641fb15bdd2b25ec187825249b51173 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -24,7 +24,7 @@ struct STDBC { SBTC btc; }; -int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, TENV *pEnv, TDB **ppDb) { +int tdbDbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb) { TDB *pDb; SPager *pPager; int ret; @@ -79,6 +79,12 @@ int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int va return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn); } +int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); } + +int tdbDbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { + return tdbBtreeUpsert(pDb->pBt, pKey, kLen, pVal, vLen, pTxn); +} + int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } @@ -87,7 +93,7 @@ int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, vo return tdbBtreePGet(pDb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen); } -int tdbDbcOpen(TDB *pDb, TDBC **ppDbc) { +int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn) { int ret; TDBC *pDbc = NULL; @@ -97,34 +103,41 @@ int tdbDbcOpen(TDB *pDb, TDBC **ppDbc) { return -1; } - tdbBtcOpen(&pDbc->btc, pDb->pBt, NULL); - - // TODO: move to first now, we can move to any key-value - // and in any direction, design new APIs. - ret = tdbBtcMoveToFirst(&pDbc->btc); - if (ret < 0) { - ASSERT(0); - return -1; - } + tdbBtcOpen(&pDbc->btc, pDb->pBt, pTxn); *ppDbc = pDbc; return 0; } -int tdbDbNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { +int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c) { return tdbBtcMoveTo(&pDbc->btc, pKey, kLen, c); } + +int tdbDbcMoveToFirst(TDBC *pDbc) { return tdbBtcMoveToFirst(&pDbc->btc); } + +int tdbDbcMoveToLast(TDBC *pDbc) { return tdbBtcMoveToLast(&pDbc->btc); } + +int tdbDbcMoveToNext(TDBC *pDbc) { return tdbBtcMoveToNext(&pDbc->btc); } + +int tdbDbcMoveToPrev(TDBC *pDbc) { return tdbBtcMoveToPrev(&pDbc->btc); } + +int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen) { + return tdbBtcGet(&pDbc->btc, ppKey, pkLen, ppVal, pvLen); +} + +int tdbDbcDelete(TDBC *pDbc) { return tdbBtcDelete(&pDbc->btc); } + +int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); } +int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert) { + return tdbBtcUpsert(&pDbc->btc, pKey, nKey, pData, nData, insert); +} + int tdbDbcClose(TDBC *pDbc) { if (pDbc) { + tdbBtcClose(&pDbc->btc); tdbOsFree(pDbc); } return 0; } - -int tdbDbcInsert(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen) { - // TODO - ASSERT(0); - return 0; -} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c index 779435861e312a590613c2db61abb90468e5f7f3..d64b31234e170fc60c7b24d86905404c2c044fbd 100644 --- a/source/libs/tdb/src/db/tdbEnv.c +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -15,7 +15,7 @@ #include "tdbInt.h" -int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv) { +int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv) { TENV *pEnv; int dsize; int zsize; @@ -49,14 +49,14 @@ int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv) { pEnv->jfd = -1; - ret = tdbPCacheOpen(pageSize, cacheSize, &(pEnv->pCache)); + ret = tdbPCacheOpen(szPage, pages, &(pEnv->pCache)); if (ret < 0) { return -1; } pEnv->nPgrHash = 8; tsize = sizeof(SPager *) * pEnv->nPgrHash; - pEnv->pgrHash = TDB_REALLOC(pEnv->pgrHash, tsize); + pEnv->pgrHash = tdbRealloc(pEnv->pgrHash, tsize); if (pEnv->pgrHash == NULL) { return -1; } @@ -103,11 +103,6 @@ int tdbCommit(TENV *pEnv, TXN *pTxn) { return 0; } -int tdbRollback(TENV *pEnv, TXN *pTxn) { - ASSERT(0); - return 0; -} - SPager *tdbEnvGetPager(TENV *pEnv, const char *fname) { u32 hash; SPager **ppPager; diff --git a/source/libs/tdb/src/db/tdbOs.c b/source/libs/tdb/src/db/tdbOs.c index 5ca3578985e96715cd6facbc88499282b802eefa..a2fce8925b8072658df78953b9c0876597299b22 100644 --- a/source/libs/tdb/src/db/tdbOs.c +++ b/source/libs/tdb/src/db/tdbOs.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tdbInt.h" +#include "tdbOs.h" #ifndef TDB_FOR_TDENGINE diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 0ccf7f5f92ccd406f93a30a96e094cc1f312b306..646b83298ead54fe1d262d93f867e5c21198f27c 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -135,7 +135,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) // 1. Search the hash table pPage = pCache->pgHash[tdbPCachePageHash(pPgid) % pCache->nHash]; while (pPage) { - if (TDB_IS_SAME_PAGE(&(pPage->pgid), pPgid)) break; + if (memcmp(pPage->pgid.fileid, pPgid->fileid, TDB_FILE_ID_LEN) == 0 && pPage->pgid.pgno == pPgid->pgno) break; pPage = pPage->pHashNext; } diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index f0fdd82944c831a9e3e19cb1378db171c4b51564..92272fb43898bfa423aeb2c392fa312382e9393b 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -171,6 +171,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl return 0; } +int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell) { + tdbPageDropCell(pPage, idx); + return tdbPageInsertCell(pPage, idx, pCell, szCell, 0); +} + int tdbPageDropCell(SPage *pPage, int idx) { int lidx; SCell *pCell; diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index 09e126772b1c46e2adfb29b4323a561df6c770a1..3746d9358f8eacbbe2a572c079f510e2e84e277d 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -15,6 +15,26 @@ #include "tdbInt.h" +void *tdbRealloc(void *ptr, size_t size) { + void *nPtr; + if ((ptr) == NULL || ((int *)(ptr))[-1] < (size)) { + nPtr = tdbOsRealloc((ptr) ? (char *)(ptr) - sizeof(int) : NULL, (size) + sizeof(int)); + if (nPtr) { + ((int *)nPtr)[0] = (size); + nPtr = (char *)nPtr + sizeof(int); + } + } else { + nPtr = (ptr); + } + return nPtr; +} + +void tdbFree(void *p) { + if (p) { + tdbOsFree((char *)(p) - sizeof(int)); + } +} + int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { int64_t stDev = 0, stIno = 0; diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h deleted file mode 100644 index 2a9e8533d2487673e86e65b47b1aa79d30447bb8..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_BTREE_H_ -#define _TD_BTREE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SBTree SBTree; -typedef struct SBTC SBTC; -typedef struct SBtInfo { - SPgno root; - int nLevel; - int nData; -} SBtInfo; - -struct SBTC { - SBTree *pBt; - i8 iPage; - SPage *pPage; - int idx; - int idxStack[BTREE_MAX_DEPTH + 1]; - SPage *pgStack[BTREE_MAX_DEPTH + 1]; - TXN *pTxn; - TXN txn; -}; - -// SBTree -int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt); -int tdbBtreeClose(SBTree *pBt); -int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); -int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); -int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); - -// SBTC -int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn); -int tdbBtcMoveToFirst(SBTC *pBtc); -int tdbBtcMoveToLast(SBTC *pBtc); -int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); -int tdbBtcClose(SBTC *pBtc); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_BTREE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h deleted file mode 100644 index f9ff3786337249e612c13423c8764db73caad270..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/inc/tdbDb.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_DB_H_ -#define _TD_TDB_DB_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct STDB TDB; -typedef struct STDBC TDBC; - -// TDB -int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, TENV *pEnv, TDB **ppDb); -int tdbDbClose(TDB *pDb); -int tdbDbDrop(TDB *pDb); -int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); -int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); -int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); - -// TDBC -int tdbDbcOpen(TDB *pDb, TDBC **ppDbc); -int tdbDbNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); -int tdbDbcClose(TDBC *pDbc); -int tdbDbcInsert(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_DB_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbEnv.h b/source/libs/tdb/src/inc/tdbEnv.h deleted file mode 100644 index 54f5818be1b6442fd7af45c1ceae572c21f731f1..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/inc/tdbEnv.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TDB_ENV_H_ -#define _TDB_ENV_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct STEnv { - char *rootDir; - char *jfname; - int jfd; - SPCache *pCache; - SPager *pgrList; - int nPager; - int nPgrHash; - SPager **pgrHash; -} TENV; - -int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv); -int tdbEnvClose(TENV *pEnv); -int tdbBegin(TENV *pEnv, TXN *pTxn); -int tdbCommit(TENV *pEnv, TXN *pTxn); -int tdbRollback(TENV *pEnv, TXN *pTxn); - -void tdbEnvAddPager(TENV *pEnv, SPager *pPager); -void tdbEnvRemovePager(TENV *pEnv, SPager *pPager); -SPager *tdbEnvGetPager(TENV *pEnv, const char *fname); - -#ifdef __cplusplus -} -#endif - -#endif /*_TDB_ENV_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index b85b7174bcdb68417bac6c16c40549ab538275c9..76dacf7b84e9ca9c365a9d7c9854643fba267757 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -31,6 +31,12 @@ typedef uint16_t u16; typedef uint32_t u32; typedef uint64_t u64; +// SPgno +typedef u32 SPgno; + +#include "tdbOs.h" +#include "tdbUtil.h" + // p must be u8 * #define TDB_GET_U24(p) ((p)[0] * 65536 + *(u16 *)((p) + 1)) #define TDB_PUT_U24(p, v) \ @@ -41,10 +47,6 @@ typedef uint64_t u64; (p)[0] = (tv >> 16) & 0xff; \ } while (0) -// SPgno -typedef u32 SPgno; -#define TDB_IVLD_PGNO ((pgno_t)0) - // fileid #define TDB_FILE_ID_LEN 24 @@ -54,51 +56,21 @@ typedef struct { SPgno pgno; } SPgid; -#define TDB_IVLD_PGID (SPgid){0, TDB_IVLD_PGNO}; - -static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) { - SPgid *pgid1 = (SPgid *)p1; - SPgid *pgid2 = (SPgid *)p2; - int rcode; - - rcode = memcmp(pgid1->fileid, pgid2->fileid, TDB_FILE_ID_LEN); - if (rcode) { - return rcode; - } else { - if (pgid1->pgno > pgid2->pgno) { - return 1; - } else if (pgid1->pgno < pgid2->pgno) { - return -1; - } else { - return 0; - } - } -} - -#define TDB_IS_SAME_PAGE(pPgid1, pPgid2) (tdbCmprPgId(pPgid1, pPgid2) == 0) - // pgsz_t #define TDB_MIN_PGSIZE 512 // 512B #define TDB_MAX_PGSIZE 16777216 // 16M #define TDB_DEFAULT_PGSIZE 4096 #define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE)) -// cache -#define TDB_DEFAULT_CACHE_SIZE (256 * 4096) // 1M - // dbname #define TDB_MAX_DBNAME_LEN 24 #define TDB_VARIANT_LEN ((int)-1) -typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); - #define TDB_JOURNAL_NAME "tdb.journal" #define TDB_FILENAME_LEN 128 -#define TDB_DEFAULT_FANOUT 6 - #define BTREE_MAX_DEPTH 20 #define TDB_FLAG_IS(flags, flag) ((flags) == (flag)) @@ -112,37 +84,258 @@ typedef struct SPCache SPCache; typedef struct SPage SPage; // transaction -#define TDB_TXN_WRITE 0x1 -#define TDB_TXN_READ_UNCOMMITTED 0x2 -typedef struct STxn { - int flags; - i64 txnId; - void *(*xMalloc)(void *, size_t); - void (*xFree)(void *, void *); - void *xArg; -} TXN; #define TDB_TXN_IS_WRITE(PTXN) ((PTXN)->flags & TDB_TXN_WRITE) #define TDB_TXN_IS_READ(PTXN) (!TDB_TXN_IS_WRITE(PTXN)) #define TDB_TXN_IS_READ_UNCOMMITTED(PTXN) ((PTXN)->flags & TDB_TXN_READ_UNCOMMITTED) -#include "tdbOs.h" - -#include "tdbUtil.h" - -#include "tdbPCache.h" +// tdbEnv.c ==================================== +void tdbEnvAddPager(TENV *pEnv, SPager *pPager); +void tdbEnvRemovePager(TENV *pEnv, SPager *pPager); +SPager *tdbEnvGetPager(TENV *pEnv, const char *fname); -#include "tdbPager.h" +// tdbBtree.c ==================================== +typedef struct SBTree SBTree; +typedef struct SBTC SBTC; +typedef struct SBtInfo { + SPgno root; + int nLevel; + int nData; +} SBtInfo; -#include "tdbBtree.h" +typedef struct { + int kLen; + const u8 *pKey; + int vLen; + const u8 *pVal; + SPgno pgno; + u8 *pBuf; +} SCellDecoder; + +struct SBTC { + SBTree *pBt; + i8 iPage; + SPage *pPage; + int idx; + int idxStack[BTREE_MAX_DEPTH + 1]; + SPage *pgStack[BTREE_MAX_DEPTH + 1]; + SCellDecoder coder; + TXN *pTxn; + TXN txn; +}; + +// SBTree +int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, tdb_cmpr_fn_t kcmpr, SBTree **ppBt); +int tdbBtreeClose(SBTree *pBt); +int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn); +int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, int nData, TXN *pTxn); +int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); + +// SBTC +int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn); +int tdbBtcClose(SBTC *pBtc); +int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst); +int tdbBtcMoveToFirst(SBTC *pBtc); +int tdbBtcMoveToLast(SBTC *pBtc); +int tdbBtcMoveToNext(SBTC *pBtc); +int tdbBtcMoveToPrev(SBTC *pBtc); +int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen); +int tdbBtcDelete(SBTC *pBtc); +int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int nData, int insert); + +// tdbPager.c ==================================== + +int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager); +int tdbPagerClose(SPager *pPager); +int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); +int tdbPagerWrite(SPager *pPager, SPage *pPage); +int tdbPagerBegin(SPager *pPager, TXN *pTxn); +int tdbPagerCommit(SPager *pPager, TXN *pTxn); +int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, + TXN *pTxn); +void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); +int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); + +// tdbPCache.c ==================================== +#define TDB_PCACHE_PAGE \ + u8 isAnchor; \ + u8 isLocal; \ + u8 isDirty; \ + i32 nRef; \ + SPage *pCacheNext; \ + SPage *pFreeNext; \ + SPage *pHashNext; \ + SPage *pLruNext; \ + SPage *pLruPrev; \ + SPage *pDirtyNext; \ + SPager *pPager; \ + SPgid pgid; + +// For page ref +#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0) +#define TDB_REF_PAGE(pPage) atomic_add_fetch_32(&((pPage)->nRef), 1) +#define TDB_UNREF_PAGE(pPage) atomic_sub_fetch_32(&((pPage)->nRef), 1) +#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef)) + +int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache); +int tdbPCacheClose(SPCache *pCache); +SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn); +void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn); +int tdbPCacheGetPageSize(SPCache *pCache); + +// tdbPage.c ==================================== +typedef u8 SCell; + +// PAGE APIS implemented +typedef struct { + int szOffset; + int szPageHdr; + int szFreeCell; + // cell number + int (*getCellNum)(SPage *); + void (*setCellNum)(SPage *, int); + // cell content offset + int (*getCellBody)(SPage *); + void (*setCellBody)(SPage *, int); + // first free cell offset (0 means no free cells) + int (*getCellFree)(SPage *); + void (*setCellFree)(SPage *, int); + // total free bytes + int (*getFreeBytes)(SPage *); + void (*setFreeBytes)(SPage *, int); + // cell offset at idx + int (*getCellOffset)(SPage *, int); + void (*setCellOffset)(SPage *, int, int); + // free cell info + void (*getFreeCellInfo)(SCell *pCell, int *szCell, int *nxOffset); + void (*setFreeCellInfo)(SCell *pCell, int szCell, int nxOffset); +} SPageMethods; + +#pragma pack(push, 1) + +// Page footer +typedef struct { + u8 cksm[4]; +} SPageFtr; +#pragma pack(pop) + +struct SPage { + tdb_spinlock_t lock; + int pageSize; + u8 *pData; + SPageMethods *pPageMethods; + // Fields below used by pager and am + u8 *pPageHdr; + u8 *pCellIdx; + u8 *pFreeStart; + u8 *pFreeEnd; + SPageFtr *pPageFtr; + int nOverflow; + SCell *apOvfl[4]; + int aiOvfl[4]; + int kLen; // key length of the page, -1 for unknown + int vLen; // value length of the page, -1 for unknown + int maxLocal; + int minLocal; + int (*xCellSize)(const SPage *, SCell *); + // Fields used by SPCache + TDB_PCACHE_PAGE +}; + +// For page lock +#define P_LOCK_SUCC 0 +#define P_LOCK_BUSY 1 +#define P_LOCK_FAIL -1 + +static inline int tdbTryLockPage(tdb_spinlock_t *pLock) { + int ret; + if (tdbSpinlockTrylock(pLock) == 0) { + ret = P_LOCK_SUCC; + } else if (errno == EBUSY) { + ret = P_LOCK_BUSY; + } else { + ret = P_LOCK_FAIL; + } + return ret; +} -#include "tdbEnv.h" +#define TDB_INIT_PAGE_LOCK(pPage) tdbSpinlockInit(&((pPage)->lock), 0) +#define TDB_DESTROY_PAGE_LOCK(pPage) tdbSpinlockDestroy(&((pPage)->lock)) +#define TDB_LOCK_PAGE(pPage) tdbSpinlockLock(&((pPage)->lock)) +#define TDB_UNLOCK_PAGE(pPage) tdbSpinlockUnlock(&((pPage)->lock)) +#define TDB_TRY_LOCK_PAGE(pPage) tdbTryLockPage(&((pPage)->lock)) + +// APIs +#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) +#define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx) +#define TDB_PAGE_FREE_SIZE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage) +#define TDB_PAGE_PGNO(pPage) ((pPage)->pgid.pgno) +#define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) +#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) + +int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); +int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); +void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); +void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); +int tdbPageDropCell(SPage *pPage, int idx); +int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell); +void tdbPageCopy(SPage *pFromPage, SPage *pToPage); +int tdbPageCapacity(int pageSize, int amHdrSize); + +static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { + SCell *pCell; + int iOvfl; + int lidx; + + ASSERT(idx >= 0 && idx < TDB_PAGE_TOTAL_CELLS(pPage)); + + iOvfl = 0; + for (; iOvfl < pPage->nOverflow; iOvfl++) { + if (pPage->aiOvfl[iOvfl] == idx) { + pCell = pPage->apOvfl[iOvfl]; + return pCell; + } else if (pPage->aiOvfl[iOvfl] > idx) { + break; + } + } -#include "tdbDb.h" + lidx = idx - iOvfl; + ASSERT(lidx >= 0 && lidx < pPage->pPageMethods->getCellNum(pPage)); + pCell = pPage->pData + pPage->pPageMethods->getCellOffset(pPage, lidx); -#include "tdbPage.h" + return pCell; +} -#include "tdbTxn.h" +struct STEnv { + char *rootDir; + char *jfname; + int jfd; + SPCache *pCache; + SPager *pgrList; + int nPager; + int nPgrHash; + SPager **pgrHash; +}; + +struct SPager { + char *dbFileName; + char *jFileName; + int pageSize; + uint8_t fid[TDB_FILE_ID_LEN]; + tdb_fd_t fd; + tdb_fd_t jfd; + SPCache *pCache; + SPgno dbFileSize; + SPgno dbOrigSize; + SPage *pDirty; + u8 inTran; + SPager *pNext; // used by TENV + SPager *pHashNext; // used by TENV +}; #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbPCache.h b/source/libs/tdb/src/inc/tdbPCache.h deleted file mode 100644 index a77cf6248cf2cac87f9e7b75f122ed88221c21cf..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/inc/tdbPCache.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_PAGE_CACHE_H_ -#define _TD_PAGE_CACHE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#define TDB_PCACHE_PAGE \ - u8 isAnchor; \ - u8 isLocal; \ - u8 isDirty; \ - i32 nRef; \ - SPage *pCacheNext; \ - SPage *pFreeNext; \ - SPage *pHashNext; \ - SPage *pLruNext; \ - SPage *pLruPrev; \ - SPage *pDirtyNext; \ - SPager *pPager; \ - SPgid pgid; - -// For page ref -#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0) -#if 0 -#define TDB_REF_PAGE(pPage) (++(pPage)->nRef) -#define TDB_UNREF_PAGE(pPage) (--(pPage)->nRef) -#define TDB_GET_PAGE_REF(pPage) ((pPage)->nRef) -#else -#define TDB_REF_PAGE(pPage) atomic_add_fetch_32(&((pPage)->nRef), 1) -#define TDB_UNREF_PAGE(pPage) atomic_sub_fetch_32(&((pPage)->nRef), 1) -#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef)) -#endif - -int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache); -int tdbPCacheClose(SPCache *pCache); -SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn); -void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn); -int tdbPCacheGetPageSize(SPCache *pCache); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_PAGE_CACHE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h deleted file mode 100644 index 3707a922ee84d46849d57647a6bb6f3853a508d2..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/inc/tdbPage.h +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TDB_PAGE_H_ -#define _TDB_PAGE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef u8 SCell; - -// PAGE APIS implemented -typedef struct { - int szOffset; - int szPageHdr; - int szFreeCell; - // cell number - int (*getCellNum)(SPage *); - void (*setCellNum)(SPage *, int); - // cell content offset - int (*getCellBody)(SPage *); - void (*setCellBody)(SPage *, int); - // first free cell offset (0 means no free cells) - int (*getCellFree)(SPage *); - void (*setCellFree)(SPage *, int); - // total free bytes - int (*getFreeBytes)(SPage *); - void (*setFreeBytes)(SPage *, int); - // cell offset at idx - int (*getCellOffset)(SPage *, int); - void (*setCellOffset)(SPage *, int, int); - // free cell info - void (*getFreeCellInfo)(SCell *pCell, int *szCell, int *nxOffset); - void (*setFreeCellInfo)(SCell *pCell, int szCell, int nxOffset); -} SPageMethods; - -#pragma pack(push,1) - -// Page footer -typedef struct { - u8 cksm[4]; -} SPageFtr; -#pragma pack(pop) - -struct SPage { - tdb_spinlock_t lock; - int pageSize; - u8 *pData; - SPageMethods *pPageMethods; - // Fields below used by pager and am - u8 *pPageHdr; - u8 *pCellIdx; - u8 *pFreeStart; - u8 *pFreeEnd; - SPageFtr *pPageFtr; - int nOverflow; - SCell *apOvfl[4]; - int aiOvfl[4]; - int kLen; // key length of the page, -1 for unknown - int vLen; // value length of the page, -1 for unknown - int maxLocal; - int minLocal; - int (*xCellSize)(const SPage *, SCell *); - // Fields used by SPCache - TDB_PCACHE_PAGE -}; - -// For page lock -#define P_LOCK_SUCC 0 -#define P_LOCK_BUSY 1 -#define P_LOCK_FAIL -1 - -static inline int tdbTryLockPage(tdb_spinlock_t *pLock) { - int ret; - if (tdbSpinlockTrylock(pLock) == 0) { - ret = P_LOCK_SUCC; - } else if (errno == EBUSY) { - ret = P_LOCK_BUSY; - } else { - ret = P_LOCK_FAIL; - } - return ret; -} - -#define TDB_INIT_PAGE_LOCK(pPage) tdbSpinlockInit(&((pPage)->lock), 0) -#define TDB_DESTROY_PAGE_LOCK(pPage) tdbSpinlockDestroy(&((pPage)->lock)) -#define TDB_LOCK_PAGE(pPage) tdbSpinlockLock(&((pPage)->lock)) -#define TDB_UNLOCK_PAGE(pPage) tdbSpinlockUnlock(&((pPage)->lock)) -#define TDB_TRY_LOCK_PAGE(pPage) tdbTryLockPage(&((pPage)->lock)) - -// APIs -#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) -#define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx) -#define TDB_PAGE_FREE_SIZE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage) -#define TDB_PAGE_PGNO(pPage) ((pPage)->pgid.pgno) -#define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) -#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) - -int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); -int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); -void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); -void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); -int tdbPageDropCell(SPage *pPage, int idx); -void tdbPageCopy(SPage *pFromPage, SPage *pToPage); -int tdbPageCapacity(int pageSize, int amHdrSize); - -static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { - SCell *pCell; - int iOvfl; - int lidx; - - ASSERT(idx >= 0 && idx < TDB_PAGE_TOTAL_CELLS(pPage)); - - iOvfl = 0; - for (; iOvfl < pPage->nOverflow; iOvfl++) { - if (pPage->aiOvfl[iOvfl] == idx) { - pCell = pPage->apOvfl[iOvfl]; - return pCell; - } else if (pPage->aiOvfl[iOvfl] > idx) { - break; - } - } - - lidx = idx - iOvfl; - ASSERT(lidx >= 0 && lidx < pPage->pPageMethods->getCellNum(pPage)); - pCell = pPage->pData + pPage->pPageMethods->getCellOffset(pPage, lidx); - - return pCell; -} - -#ifdef __cplusplus -} -#endif - -#endif /*_TDB_PAGE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPager.h b/source/libs/tdb/src/inc/tdbPager.h deleted file mode 100644 index c39f833c738ecd9a1ad63628d2ae1a371bd37bf0..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/inc/tdbPager.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TDB_PAGER_H_ -#define _TDB_PAGER_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -struct SPager { - char *dbFileName; - char *jFileName; - int pageSize; - uint8_t fid[TDB_FILE_ID_LEN]; - tdb_fd_t fd; - tdb_fd_t jfd; - SPCache *pCache; - SPgno dbFileSize; - SPgno dbOrigSize; - SPage *pDirty; - u8 inTran; - SPager *pNext; // used by TENV - SPager *pHashNext; // used by TENV -}; - -int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager); -int tdbPagerClose(SPager *pPager); -int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); -int tdbPagerWrite(SPager *pPager, SPage *pPage); -int tdbPagerBegin(SPager *pPager, TXN *pTxn); -int tdbPagerCommit(SPager *pPager, TXN *pTxn); -int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, - TXN *pTxn); -void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); -int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); - -#ifdef __cplusplus -} -#endif - -#endif /*_TDB_PAGER_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index cffd3f27469598c58a31513e11d9b72b8539fc5f..29a505fa783e848e9603dc1c7dfd12c1b01bd622 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -31,27 +31,7 @@ extern "C" { int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size); -static inline void *tdbRealloc(void *ptr, size_t size) { - void *nPtr; - if ((ptr) == NULL || ((int *)(ptr))[-1] < (size)) { - nPtr = tdbOsRealloc((ptr) ? (char *)(ptr) - sizeof(int) : NULL, (size) + sizeof(int)); - if (nPtr) { - ((int *)nPtr)[0] = (size); - nPtr = (char *)nPtr + sizeof(int); - } - } else { - nPtr = (ptr); - } - return nPtr; -} -#define TDB_REALLOC(PTR, SIZE) tdbRealloc(PTR, SIZE) - -#define TDB_FREE(PTR) \ - do { \ - if (PTR) { \ - tdbOsFree((char *)(PTR) - sizeof(int)); \ - } \ - } while (0) +void *tdbRealloc(void *ptr, size_t size); static inline void *tdbDefaultMalloc(void *arg, size_t size) { void *ptr; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 49763ae937d252e3289ab964f3c2c868bb82ea20..e575ac156f0a2bbddd4ed36287a8f84e0a40c3d2 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -1,7 +1,7 @@ #include #include "os.h" -#include "tdbInt.h" +#include "tdb.h" #include @@ -12,7 +12,7 @@ typedef struct SPoolMem { } SPoolMem; static SPoolMem *openPool() { - SPoolMem *pPool = (SPoolMem *)tdbOsMalloc(sizeof(*pPool)); + SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool)); pPool->prev = pPool->next = pPool; pPool->size = 0; @@ -32,7 +32,7 @@ static void clearPool(SPoolMem *pPool) { pMem->prev->next = pMem->next; pPool->size -= pMem->size; - tdbOsFree(pMem); + taosMemoryFree(pMem); } while (1); assert(pPool->size == 0); @@ -40,7 +40,7 @@ static void clearPool(SPoolMem *pPool) { static void closePool(SPoolMem *pPool) { clearPool(pPool); - tdbOsFree(pPool); + taosMemoryFree(pPool); } static void *poolMalloc(void *arg, size_t size) { @@ -48,7 +48,7 @@ static void *poolMalloc(void *arg, size_t size) { SPoolMem *pPool = (SPoolMem *)arg; SPoolMem *pMem; - pMem = (SPoolMem *)tdbOsMalloc(sizeof(*pMem) + size); + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); if (pMem == NULL) { assert(0); } @@ -75,7 +75,7 @@ static void poolFree(void *arg, void *ptr) { pMem->prev->next = pMem->next; pPool->size -= pMem->size; - tdbOsFree(pMem); + taosMemoryFree(pMem); } static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { @@ -115,13 +115,13 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in return cret; } -TEST(tdb_test, simple_test) { - int ret; - TENV *pEnv; - TDB *pDb; - FKeyComparator compFunc; - int nData = 10000000; - TXN txn; +TEST(tdb_test, simple_insert1) { + int ret; + TENV *pEnv; + TDB *pDb; + tdb_cmpr_fn_t compFunc; + int nData = 1000000; + TXN txn; taosRemoveDir("tdb"); @@ -131,7 +131,7 @@ TEST(tdb_test, simple_test) { // Create a database compFunc = tKeyCmpr; - ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, &pDb); + ret = tdbDbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -189,7 +189,7 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); } - TDB_FREE(pVal); + tdbFree(pVal); } { // Iterate to query the DB data @@ -199,11 +199,13 @@ TEST(tdb_test, simple_test) { int vLen, kLen; int count = 0; - ret = tdbDbcOpen(pDb, &pDBC); + ret = tdbDbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); + tdbDbcMoveToFirst(pDBC); + for (;;) { - ret = tdbDbNext(pDBC, &pKey, &kLen, &pVal, &vLen); + ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; @@ -217,8 +219,8 @@ TEST(tdb_test, simple_test) { tdbDbcClose(pDBC); - TDB_FREE(pKey); - TDB_FREE(pVal); + tdbFree(pKey); + tdbFree(pVal); } } @@ -233,13 +235,13 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(ret, 0); } -TEST(tdb_test, simple_test2) { - int ret; - TENV *pEnv; - TDB *pDb; - FKeyComparator compFunc; - int nData = 1000000; - TXN txn; +TEST(tdb_test, simple_insert2) { + int ret; + TENV *pEnv; + TDB *pDb; + tdb_cmpr_fn_t compFunc; + int nData = 1000000; + TXN txn; taosRemoveDir("tdb"); @@ -249,7 +251,7 @@ TEST(tdb_test, simple_test2) { // Create a database compFunc = tDefaultKeyCmpr; - ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, &pDb); + ret = tdbDbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -280,16 +282,18 @@ TEST(tdb_test, simple_test2) { int vLen, kLen; int count = 0; - ret = tdbDbcOpen(pDb, &pDBC); + ret = tdbDbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); + tdbDbcMoveToFirst(pDBC); + for (;;) { - ret = tdbDbNext(pDBC, &pKey, &kLen, &pVal, &vLen); + ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; - std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; - std::cout.write((char *)pVal, vLen) /* << " " << vLen */; - std::cout << std::endl; + // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; + // std::cout.write((char *)pVal, vLen) /* << " " << vLen */; + // std::cout << std::endl; count++; } @@ -298,8 +302,8 @@ TEST(tdb_test, simple_test2) { tdbDbcClose(pDBC); - TDB_FREE(pKey); - TDB_FREE(pVal); + tdbFree(pKey); + tdbFree(pVal); } } @@ -316,4 +320,164 @@ TEST(tdb_test, simple_test2) { // Close Env ret = tdbEnvClose(pEnv); GTEST_ASSERT_EQ(ret, 0); +} + +TEST(tdb_test, simple_delete1) { + int ret; + TDB *pDb; + char key[128]; + char data[128]; + TXN txn; + TENV *pEnv; + SPoolMem *pPool; + void *pKey = NULL; + void *pData = NULL; + int nKey; + TDBC *pDbc; + int nData; + int nKV = 69; + + taosRemoveDir("tdb"); + + pPool = openPool(); + + // open env + ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); + GTEST_ASSERT_EQ(ret, 0); + + // open database + ret = tdbDbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb); + GTEST_ASSERT_EQ(ret, 0); + + tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + tdbBegin(pEnv, &txn); + + // loop to insert batch data + for (int iData = 0; iData < nKV; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + ret = tdbDbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // query the data + for (int iData = 0; iData < nKV; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + GTEST_ASSERT_EQ(ret, 0); + GTEST_ASSERT_EQ(memcmp(data, pData, nData), 0); + } + + // loop to delete some data + for (int iData = nKV - 1; iData > 30; iData--) { + sprintf(key, "key%d", iData); + + ret = tdbDbDelete(pDb, key, strlen(key), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // query the data + for (int iData = 0; iData < nKV; iData++) { + sprintf(key, "key%d", iData); + + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + if (iData <= 30) { + GTEST_ASSERT_EQ(ret, 0); + } else { + GTEST_ASSERT_EQ(ret, -1); + } + } + + // loop to iterate the data + tdbDbcOpen(pDb, &pDbc, NULL); + + ret = tdbDbcMoveToFirst(pDbc); + GTEST_ASSERT_EQ(ret, 0); + + pKey = NULL; + pData = NULL; + for (;;) { + ret = tdbDbcNext(pDbc, &pKey, &nKey, &pData, &nData); + if (ret < 0) break; + + std::cout.write((char *)pKey, nKey) /* << " " << kLen */ << " "; + std::cout.write((char *)pData, nData) /* << " " << vLen */; + std::cout << std::endl; + } + + tdbDbcClose(pDbc); + + tdbCommit(pEnv, &txn); + + closePool(pPool); + + tdbDbClose(pDb); + tdbEnvClose(pEnv); +} + +TEST(tdb_test, simple_upsert1) { + int ret; + TENV *pEnv; + TDB *pDb; + int nData = 100000; + char key[64]; + char data[64]; + void *pData = NULL; + SPoolMem *pPool; + TXN txn; + + taosRemoveDir("tdb"); + + // open env + ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); + GTEST_ASSERT_EQ(ret, 0); + + // open database + ret = tdbDbOpen("db.db", -1, -1, NULL, pEnv, &pDb); + GTEST_ASSERT_EQ(ret, 0); + + pPool = openPool(); + // insert some data + tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + tdbBegin(pEnv, &txn); + + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + ret = tdbDbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // query the data + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + GTEST_ASSERT_EQ(ret, 0); + GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); + } + + // upsert some data + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d-u", iData); + ret = tdbDbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + tdbCommit(pEnv, &txn); + + // query the data + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d-u", iData); + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + GTEST_ASSERT_EQ(ret, 0); + GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); + } + + tdbDbClose(pDb); + tdbEnvClose(pEnv); } \ No newline at end of file diff --git a/source/libs/tdb/test/tdbUtilTest.cpp b/source/libs/tdb/test/tdbUtilTest.cpp index 5fd45daa3e59780927616b2155310cf339d527ca..57372995cc8b4ece29c6fab95cdbf0363d12a759 100644 --- a/source/libs/tdb/test/tdbUtilTest.cpp +++ b/source/libs/tdb/test/tdbUtilTest.cpp @@ -1,20 +1,20 @@ #include -#include "tdbInt.h" +#include "tdb.h" #include TEST(tdb_util_test, simple_test) { - int vEncode = 5000; - int vDecode; - int nEncode; - int nDecode; - u8 buffer[128]; + // int vEncode = 5000; + // int vDecode; + // int nEncode; + // int nDecode; + // uint8_t buffer[128]; - nEncode = tdbPutVarInt(buffer, vEncode); + // nEncode = tdbPutVarInt(buffer, vEncode); - nDecode = tdbGetVarInt(buffer, &vDecode); + // nDecode = tdbGetVarInt(buffer, &vDecode); - GTEST_ASSERT_EQ(nEncode, nDecode); - GTEST_ASSERT_EQ(vEncode, vDecode); + // GTEST_ASSERT_EQ(nEncode, nDecode); + // GTEST_ASSERT_EQ(vEncode, vDecode); } \ No newline at end of file diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index fecb5d927951a2b98ea8ff716b26a4cab1391fc9..846cf6f967664091b6b1610a81e21bd8a22bbccf 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -46,9 +46,21 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } + uint32_t ip = 0; + if (pInit->connType == TAOS_CONN_SERVER) { + ip = taosGetIpv4FromFqdn(pInit->localFqdn); + if (ip == 0xFFFFFFFF) { + tError("invalid fqdn: %s", pInit->localFqdn); + terrno = TSDB_CODE_RPC_FQDN_ERROR; + taosMemoryFree(pRpc); + return NULL; + } + } + pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; - pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + pRpc->tcphandle = + (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); if (pRpc->tcphandle == NULL) { taosMemoryFree(pRpc); return NULL; @@ -101,18 +113,8 @@ void* rpcReallocCont(void* ptr, int contLen) { } void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) { - SRpcMsg rpcMsg; - memset(&rpcMsg, 0, sizeof(rpcMsg)); - - SMEpSet msg = {.epSet = *pEpSet}; - int32_t len = tSerializeSMEpSet(NULL, 0, &msg); - rpcMsg.pCont = rpcMallocCont(len); - tSerializeSMEpSet(rpcMsg.pCont, len, &msg); - - rpcMsg.code = TSDB_CODE_RPC_REDIRECT; - rpcMsg.handle = thandle; - - rpcSendResponse(&rpcMsg); + // deprecated api + assert(0); } int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c20303547cb6b45b602273562d0b338f35e4e776..718be6aa64755afea556961e2c765a7558c60ddf 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -891,7 +891,6 @@ static void doDelayTask(void* param) { SCliMsg* pMsg = arg->param1; SCliThrdObj* pThrd = arg->param2; - cliHandleReq(pMsg, pThrd); taosMemoryFree(arg); @@ -914,8 +913,9 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tmsg_t msgType = pCtx->msgType; if ((pTransInst->retry != NULL && (pTransInst->retry(pResp->code))) || ((pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && msgType == TDMT_MND_CONNECT)) { - pCtx->retryCount += 1; + pMsg->sent = 0; pMsg->st = taosGetTimestampUs(); + pCtx->retryCount += 1; if (msgType == TDMT_MND_CONNECT && pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { if (pCtx->retryCount < pEpSet->numOfEps) { pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; @@ -936,20 +936,27 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tDeserializeSMEpSet(pResp->pCont, pResp->contLen, &emsg); pCtx->epSet = emsg.epSet; } + addConnToPool(pThrd->pool, pConn); + tTrace("use remote epset, current in use: %d, retry count:%d, try limit: %d", pEpSet->inUse, pCtx->retryCount + 1, + TRANS_RETRY_COUNT_LIMIT); + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - addConnToPool(pThrd, pConn); return -1; } } if (pCtx->pSem != NULL) { - tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); - memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); + tTrace("%s cli conn %p(sync) handle resp", pTransInst->label, pConn); + if (pCtx->pRsp == NULL) { + tTrace("%s cli conn %p(sync) failed to resp, ignore", pTransInst->label, pConn); + } else { + memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); + } tsem_post(pCtx->pSem); + pCtx->pRsp = NULL; } else { tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); pTransInst->cfp(pTransInst->parent, pResp, pEpSet); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 01a20a466ac7419c6b803fa4b171ecd9f5e8daea..98e9e67edeb532d3ac992863ac54b3fd166182b3 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -425,10 +425,19 @@ void transDQDestroy(SDelayQueue* queue) { } int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs) { + uint64_t now = taosGetTimestampMs(); SDelayTask* task = taosMemoryCalloc(1, sizeof(SDelayTask)); task->func = func; task->arg = arg; - task->execTime = taosGetTimestampMs() + timeoutMs; + task->execTime = now + timeoutMs; + + HeapNode* minNode = heapMin(queue->heap); + if (minNode) { + SDelayTask* minTask = container_of(minNode, SDelayTask, node); + if (minTask->execTime < task->execTime) { + timeoutMs = minTask->execTime <= now ? 0 : minTask->execTime - now; + } + } tTrace("timer %p put task into queue, timeoutMs: %" PRIu64 "", queue->timer, timeoutMs); heapInsert(queue->heap, &task->node); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 9c905771d54c64ea333a19917a4b5cb1f853e233..56b5ddd38a2e5698af8af77b2cecf458fff5b357 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -207,7 +207,7 @@ static bool addHandleToAcceptloop(void* arg); SExHandle* exh2 = uvAcquireExHandle(refId); \ if (exh2 == NULL || refId != exh2->refId) { \ tTrace("server handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ - exh1->refId, refId); \ + exh2 ? exh2->refId : 0, refId); \ goto _return1; \ } \ } else if (refId == 0) { \ @@ -765,8 +765,10 @@ static void destroyConn(SSrvConn* conn, bool clear) { transDestroyBuffer(&conn->readBuf); if (clear) { tTrace("server conn %p to be destroyed", conn); - uv_shutdown_t* req = taosMemoryMalloc(sizeof(uv_shutdown_t)); - uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); + // uv_shutdown_t* req = taosMemoryMalloc(sizeof(uv_shutdown_t)); + uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); + // uv_close(conn->pTcp) + // uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); } } static void uvDestroyConn(uv_handle_t* handle) { @@ -813,18 +815,20 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, for (int i = 0; i < srv->numOfThreads; i++) { SWorkThrdObj* thrd = (SWorkThrdObj*)taosMemoryCalloc(1, sizeof(SWorkThrdObj)); + thrd->pTransInst = shandle; thrd->quit = false; srv->pThreadObj[i] = thrd; + thrd->pTransInst = shandle; srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); - int fds[2]; - if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { + + uv_os_sock_t fds[2]; + if (uv_socketpair(SOCK_STREAM, 0, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { goto End; } uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write - thrd->pTransInst = shandle; thrd->fd = fds[0]; thrd->pipe = &(srv->pipe[i][1]); // init read @@ -841,6 +845,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; } } + if (false == taosValidIpAndPort(srv->ip, srv->port)) { + tError("failed to bind, reason: %s", terrstr()); + goto End; + } if (false == addHandleToAcceptloop(srv)) { goto End; } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 748128e34e0a55a8ecd886713018359066ace8d4..19e4defafc47dfb0b6b84a856ddac8d8484b42c0 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -40,11 +40,11 @@ int wordexp(char *words, wordexp_t *pwordexp, int flags) { memset(pwordexp->wordPos, 0, 1025); if (_fullpath(pwordexp->wordPos, words, 1024) == NULL) { pwordexp->we_wordv[0] = words; - printf("failed to parse relative path:%s to abs path", words); + printf("failed to parse relative path:%s to abs path\n", words); return -1; } - printf("parse relative path:%s to abs path:%s", words, pwordexp->wordPos); + // printf("parse relative path:%s to abs path:%s\n", words, pwordexp->wordPos); return 0; } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 786e58a6d261fb5eca7f4d0ecedaf9336e5f6bb1..d378b5234aedb6d8e520a7ca6781a17826def3bc 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -22,20 +22,6 @@ #define W_OK 2 #define R_OK 4 -#if defined(_MSDOS) -#define open _open -#endif - -#if defined(_WIN32) -extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */ -extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ -#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) -#define open openU -#else /* _ANSI_SOURCE */ -#define open openA -#endif /* defined(_UTF8_SOURCE) */ -#endif /* defined(_WIN32) */ - #define _SEND_FILE_STEP_ 1000 #else @@ -228,9 +214,6 @@ int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { void autoDelFileListAdd(const char *path) { return; } TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { -#ifdef WINDOWS - return NULL; -#else int fd = -1; FILE *fp = NULL; if (tdFileOptions & TD_FILE_STREAM) { @@ -263,7 +246,11 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0; access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; + #ifdef WINDOWS + fd = _open(path, access, _S_IREAD|_S_IWRITE); + #else fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); + #endif if (fd == -1) { return NULL; } @@ -286,7 +273,6 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { pFile->fp = fp; pFile->refId = 0; return pFile; -#endif } int64_t taosCloseFile(TdFilePtr *ppFile) { diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 6aa85200828b826dc0fb0715ed8cf55b11435a4f..8cac660039a0fbba90a3e69c48be619a6c7b29cc 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -638,6 +638,48 @@ int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { return 0; } +bool taosValidIpAndPort(uint32_t ip, uint16_t port) { + struct sockaddr_in serverAdd; + SocketFd fd; + int32_t reuse; + + // printf("open tcp server socket:0x%x:%hu", ip, port); + + bzero((char *)&serverAdd, sizeof(serverAdd)); + serverAdd.sin_family = AF_INET; + serverAdd.sin_addr.s_addr = ip; + serverAdd.sin_port = (uint16_t)htons(port); + + if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { + // printf("failed to open TCP socket: %d (%s)", errno, strerror(errno)); + taosCloseSocketNoCheck1(fd); + return false; + } + + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(fd); + return false; + } + pSocket->refId = 0; + pSocket->fd = fd; + + /* set REUSEADDR option, so the portnumber can be re-used */ + reuse = 1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; + } + /* bind socket to server address */ + if (bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { + // printf("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); + taosCloseSocket(&pSocket); + return false; + } + taosCloseSocket(&pSocket); + return true; +} TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; SocketFd fd; diff --git a/source/os/src/osString.c b/source/os/src/osString.c index ed596a051d34b003a016211a48356f2da2b431de..375c5001f4a1025cdaa44e317e57548170022fd5 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -195,6 +195,36 @@ int32_t taosUcs4len(TdUcs4 *ucs4) { return n; } +//dst buffer size should be at least 2*len + 1 +int32_t taosHexEncode(const char *src, char *dst, int32_t len) { + if (!dst) { + return -1; + } + + for (int32_t i = 0; i < len; ++i) { + sprintf(dst + i * 2, "%02x", src[i] & 0xff); + } + + return 0; +} + +int32_t taosHexDecode(const char *src, char *dst, int32_t len) { + if (!dst) { + return -1; + } + + uint16_t hn, ln, out; + for (int i = 0, j = 0; i < len * 2; i += 2, ++j ) { + hn = src[i] > '9' ? src[i] - 'A' + 10 : src[i] - '0'; + ln = src[i + 1] > '9' ? src[i + 1] - 'A' + 10 : src[i + 1] - '0'; + + out = (hn << 4) | ln; + memcpy(dst + j, &out, 1); + } + + return 0; +} + int32_t taosWcharWidth(TdWchar wchar) { return wcwidth(wchar); } int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size) { return wcswidth(pWchar, size); } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index ccc302e6467770f946e8a9031c63d330d185b73e..3c3612854c8a6bf8f943e55261fd61aeff340394 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -866,6 +866,16 @@ void taosSetCoreDump(bool enable) { SysNameInfo taosGetSysNameInfo() { #ifdef WINDOWS + SysNameInfo info = {0}; + DWORD dwVersion = GetVersion(); + + tstrncpy(info.sysname, getenv("OS"), sizeof(info.sysname)); + tstrncpy(info.nodename, getenv("COMPUTERNAME"), sizeof(info.nodename)); + sprintf_s(info.release, sizeof(info.release), "%d", dwVersion & 0x0F); + sprintf_s(info.version, sizeof(info.release), "%d", (dwVersion >> 8) & 0x0F); + tstrncpy(info.machine, getenv("PROCESSOR_ARCHITECTURE"), sizeof(info.machine)); + + return info; #elif defined(_TD_DARWIN_64) SysNameInfo info = {0}; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index cb9ef8770125988d5e5e7101be4375576f166fae..8f48e0585e79294dd65649a37a12a2fbcb17beb4 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -687,13 +687,13 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { const char *filepath = ".env"; if (envFile != NULL && strlen(envFile)>0) { if (!taosCheckExistFile(envFile)) { - uError("fial to load env file: %s", envFile); + uError("failed to load env file: %s", envFile); return -1; } filepath = envFile; }else { if (!taosCheckExistFile(filepath)) { - uInfo("fial to load env file: %s", filepath); + uInfo("failed to load env file: %s", filepath); return 0; } } @@ -826,7 +826,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { if (strncmp(url, "jsonFile", 8) == 0) { char *filepath = p; if (!taosCheckExistFile(filepath)) { - uError("fial to load json file: %s", filepath); + uError("failed to load json file: %s", filepath); return -1; } @@ -957,13 +957,13 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl const char *filepath = ".env"; if (envFile != NULL && strlen(envFile)>0) { if (!taosCheckExistFile(envFile)) { - uError("fial to load env file: %s", envFile); + uError("failed to load env file: %s", envFile); return -1; } filepath = envFile; }else { if (!taosCheckExistFile(filepath)) { - uInfo("fial to load env file: %s", filepath); + uInfo("failed to load env file: %s", filepath); return 0; } } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 8f2ba2dcc446941c884fb66e177bc4904baf2647..1470496c686a1aa8c44af4e380ef7249fd5526b8 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -93,6 +93,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_OPS_NOT_SUPPORT, "Operation not support TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") +TAOS_DEFINE_ERROR(TSDB_CODE_DUP_KEY, "Cannot add duplicate keys to hash") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") @@ -250,6 +251,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, "Tag does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST, "Column already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SINGLE_STB_MODE_DB, "Database is single stable mode") // mnode-infoSchema TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME, "Invalid system table name") @@ -269,7 +271,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CANT_PARALLEL, "Invalid stage to kill") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT, "Conflict transaction not completed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") // mnode-mq TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists") @@ -316,6 +319,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TSDB_STATE, "Invalid tsdb state") TAOS_DEFINE_ERROR(TSDB_CODE_VND_TB_NOT_EXIST, "Table not exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_SMA_NOT_EXIST, "SMA not exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_HASH_MISMATCH, "Hash value mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_NOT_EXIST, "Table does not exists") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") @@ -439,6 +443,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR, "scheduler status erro TAOS_DEFINE_ERROR(TSDB_CODE_SCH_INTERNAL_ERROR, "scheduler internal error") TAOS_DEFINE_ERROR(TSDB_CODE_QW_MSG_ERROR, "Invalid msg order") +// parser +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TABLE_NOT_EXIST, "Table does not exist") + //planner TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "planner internal error") diff --git a/source/util/src/thash.c b/source/util/src/thash.c index d2b07875a36357d33309ea07d0b527de7bbd9ec4..4bbf6ccf450ae4c11685d5012d4acdf2348b551e 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -310,6 +310,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) { int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size) { if (pHashObj == NULL || key == NULL || keyLen == 0) { + terrno = TSDB_CODE_INVALID_PTR; return -1; } @@ -378,6 +379,8 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo } doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode); + } else { + terrno = TSDB_CODE_DUP_KEY; } taosHashEntryWUnlock(pHashObj, pe); diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index d8343bc427875cffa4b64698c6e71b5e36eddd2e..8963a4f94eb23799374a355f9a44a24a6962e2e5 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -175,7 +175,6 @@ static int32_t taosProcQueuePush(SProcObj *pProc, SProcQueue *pQueue, const char if (handle != 0 && ftype == PROC_FUNC_REQ) { if (taosHashPut(pProc->hash, &handle, sizeof(int64_t), &handleRef, sizeof(int64_t)) != 0) { taosThreadMutexUnlock(&pQueue->mutex); - terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } } @@ -227,8 +226,8 @@ static int32_t taosProcQueuePush(SProcObj *pProc, SProcQueue *pQueue, const char taosThreadMutexUnlock(&pQueue->mutex); tsem_post(&pQueue->sem); - uTrace("proc:%s, push msg at pos:%d ftype:%d remain:%d, head:%d %p body:%d %p", pQueue->name, pos, ftype, - pQueue->items, headLen, pHead, bodyLen, pBody); + uTrace("proc:%s, push msg at pos:%d ftype:%d remain:%d handle:%p ref:%" PRId64 ", head:%d %p body:%d %p", + pQueue->name, pos, ftype, pQueue->items, (void *)handle, handleRef, headLen, pHead, bodyLen, pBody); return 0; } @@ -455,25 +454,28 @@ void taosProcCleanup(SProcObj *pProc) { } int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, - void *handle, int64_t handleRef, EProcFuncType ftype) { + void *handle, int64_t ref, EProcFuncType ftype) { if (ftype != PROC_FUNC_REQ) { terrno = TSDB_CODE_INVALID_PARA; return -1; } - return taosProcQueuePush(pProc, pProc->pChildQueue, pHead, headLen, pBody, bodyLen, (int64_t)handle, handleRef, - ftype); + return taosProcQueuePush(pProc, pProc->pChildQueue, pHead, headLen, pBody, bodyLen, (int64_t)handle, ref, ftype); } int64_t taosProcRemoveHandle(SProcObj *pProc, void *handle) { int64_t h = (int64_t)handle; taosThreadMutexLock(&pProc->pChildQueue->mutex); - int64_t *handleRef = taosHashGet(pProc->hash, &h, sizeof(int64_t)); + int64_t *pRef = taosHashGet(pProc->hash, &h, sizeof(int64_t)); + int64_t ref = 0; + if (pRef != NULL) { + ref = *pRef; + } + taosHashRemove(pProc->hash, &h, sizeof(int64_t)); taosThreadMutexUnlock(&pProc->pChildQueue->mutex); - if (handleRef == NULL) return 0; - return *handleRef; + return ref; } void taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle)) { diff --git a/source/util/src/tskiplist2.c b/source/util/src/tskiplist2.c new file mode 100644 index 0000000000000000000000000000000000000000..77f5ed5051d888622b1605a95b6b6cb85a4a56c4 --- /dev/null +++ b/source/util/src/tskiplist2.c @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tskiplist2.h" + +struct SSLNode { + int8_t level; + SSLNode *forwards[]; +}; + +struct SSkipList2 { + int8_t level; + uint32_t seed; + int32_t size; + const SSLCfg *pCfg; + SSLNode *pHead[]; +}; + +static void *slMalloc(void *pPool, int32_t size); +static void slFree(void *pPool, void *p); +static int32_t slCmprFn(const void *pKey, int32_t nKey, const void *pData, int32_t nData); + +const SSLCfg slDefaultCfg = {.maxLevel = SL_MAX_LEVEL, + .nKey = -1, + .nData = -1, + .cmprFn = slCmprFn, + .pPool = NULL, + .xMalloc = slMalloc, + .xFree = slFree}; + +int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl) { + SSkipList2 *pSl = NULL; + int32_t size; + + *ppSl = NULL; + if (pCfg == NULL) pCfg = &slDefaultCfg; + + // check config (TODO) + + // malloc handle + size = sizeof(*pSl) + sizeof(SSLNode *) * pCfg->maxLevel * 2; + pSl = pCfg->xMalloc(pCfg->pPool, size); + if (pSl == NULL) { + return -1; + } + + pSl->level = 0; + pSl->seed = taosRand(); + pSl->size = 0; + pSl->pCfg = pCfg; + + // init an empty skiplist + for (int32_t i = 0; i < pCfg->maxLevel * 2; i++) { + pSl->pHead[i] = NULL; + } + + *ppSl = pSl; + return 0; +} + +int32_t slClose(SSkipList2 *pSl) { + if (pSl) { + slClear(pSl); + if (pSl->pCfg->xFree) { + pSl->pCfg->xFree(pSl->pCfg->pPool, pSl); + } + } + + return 0; +} + +int32_t slClear(SSkipList2 *pSl) { + // loop to clear sl + for (;;) { + // (TODO) + } + + // init sl (TODO) + + return 0; +} + +int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc) { + pSlc->pSl = pSl; + + for (int i = 0; i < SL_MAX_LEVEL; i++) { + if (i < pSl->pCfg->maxLevel) { + } else { + pSlc->forwards[i] = NULL; + } + } + + // TODO + return 0; +} + +int32_t slcClose(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey) { + // TODO + return 0; +} + +int32_t slcMoveToNext(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveToPrev(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveToFirst(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveToLast(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData) { + // TODO + return 0; +} + +int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData) { + // TODO + return 0; +} + +int32_t slcDrop(SSLCursor *pSlc) { + // TODO + return 0; +} + +static FORCE_INLINE void *slMalloc(void *pPool, int32_t size) { return taosMemoryMalloc(size); } + +static FORCE_INLINE void slFree(void *pPool, void *p) { taosMemoryFree(p); } + +static int32_t slCmprFn(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2) { + ASSERT(nKey1 >= 0 && nKey2 >= 0); + + int32_t nKey = nKey1 > nKey2 ? nKey2 : nKey1; + int32_t c; + + c = memcmp(pKey1, pKey2, nKey); + if (c == 0) { + if (nKey1 > nKey2) { + c = 1; + } else if (nKey1 < nKey2) { + c = -1; + } + } + + return c; +} \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ce459414c44c6adcb8c8a77b03ea790480a41cd2 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,5 @@ +taospy +numpy +fabric2 +psutil +pandas diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index b1ab5253addda28221b88929e45677438d29f790..828216d1d3a28608f5d990ef0658dea3ad8fae7c 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -11,7 +11,8 @@ int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE}; +int32_t optrIdxList[] = {0, 1, 2}; typedef struct { char* oper; @@ -32,7 +33,7 @@ OperInfo operInfo[] = { {"like", 2, false}, {"not like", 2, false}, {"match", 2, false}, - {"nmake", 2, false}, + {"nmatch", 2, false}, }; int32_t operatorList[] = {0, 1, 2, 3, 4, 5, 6, 7}; @@ -139,9 +140,7 @@ CaseCfg gCase[] = { {"insert:MPME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 1, -1}, // 22 - //{"query:SUBT-FULL", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, querySUBTTest1, 10, 10, 1, 3, 0, 1, 2}, - - {"query:SUBT-FULL", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, querySUBTTest1, 1, 10, 1, 3, 0, 1, 2}, + {"query:SUBT-FULL", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, querySUBTTest1, 10, 10, 1, 3, 0, 1, 2}, }; @@ -161,6 +160,8 @@ typedef struct { int32_t bindRowNum; //row num for once bind int32_t bindColTypeNum; int32_t* bindColTypeList; + int32_t optrIdxListNum; + int32_t* optrIdxList; int32_t runTimes; int32_t caseIdx; // static case idx int32_t caseNum; // num in static case list @@ -168,8 +169,8 @@ typedef struct { int32_t caseRunNum; // total run case num } CaseCtrl; -#if 1 -CaseCtrl gCaseCtrl = { +#if 0 +CaseCtrl gCaseCtrl = { // default .bindNullNum = 0, .prepareStb = false, .printCreateTblSql = false, @@ -178,25 +179,50 @@ CaseCtrl gCaseCtrl = { .rowNum = 0, .bindColNum = 0, .bindRowNum = 0, -// .bindColTypeNum = 0, -// .bindColTypeList = NULL, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .optrIdxListNum = 0, + .optrIdxList = NULL, .checkParamNum = false, .printRes = true, .runTimes = 0, -// .caseIdx = -1, -// .caseNum = -1, + .caseIdx = -1, + .caseNum = -1, .caseRunIdx = -1, -// .caseRunNum = -1, + .caseRunNum = -1, +}; +#endif - .bindColTypeNum = tListLen(bindColTypeList), - .bindColTypeList = bindColTypeList, +#if 0 +CaseCtrl gCaseCtrl = { // query case with specified col&oper + .bindNullNum = 0, + .prepareStb = false, + .printCreateTblSql = false, + .printQuerySql = true, + .printStmtSql = true, + .rowNum = 0, + .bindColNum = 0, + .bindRowNum = 0, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .optrIdxListNum = 0, + .optrIdxList = NULL, + .checkParamNum = false, + .printRes = true, + .runTimes = 0, + .caseRunIdx = -1, + .optrIdxListNum = 0, + .optrIdxList = NULL, + .bindColTypeNum = 0, + .bindColTypeList = NULL, .caseIdx = 22, .caseNum = 1, .caseRunNum = 1, - }; -#else -CaseCtrl gCaseCtrl = { +#endif + +#if 1 +CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindNullNum = 0, .prepareStb = false, .printCreateTblSql = false, @@ -205,17 +231,22 @@ CaseCtrl gCaseCtrl = { .rowNum = 0, .bindColNum = 0, .bindRowNum = 0, - .bindColTypeNum = tListLen(bindColTypeList), - .bindColTypeList = bindColTypeList, - .checkParamNum = false, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .optrIdxListNum = 0, + .optrIdxList = NULL, + .checkParamNum = true, .printRes = true, .runTimes = 0, - .caseIdx = -1, - .caseNum = -1, .caseRunIdx = -1, - .caseRunNum = -1, + .optrIdxListNum = tListLen(optrIdxList), + .optrIdxList = optrIdxList, + .bindColTypeNum = tListLen(bindColTypeList), + .bindColTypeList = bindColTypeList, + .caseIdx = 22, + .caseNum = 1, + .caseRunNum = 1, }; - #endif int32_t taosGetTimeOfDay(struct timeval *tv) { @@ -318,7 +349,7 @@ void generateInsertSQL(BindData *data) { len += sprintf(data->sql + len, "ubigdata"); break; default: - printf("invalid col type:%d", data->pBind[c].buffer_type); + printf("!!!invalid col type:%d", data->pBind[c].buffer_type); exit(1); } } @@ -336,19 +367,23 @@ void generateInsertSQL(BindData *data) { len += sprintf(data->sql + len, ")"); if (gCaseCtrl.printStmtSql) { - printf("SQL: %s\n", data->sql); + printf("\tSQL: %s\n", data->sql); } } -void bpAppendOperatorParam(BindData *data, int32_t *len, int32_t dataType) { +void bpAppendOperatorParam(BindData *data, int32_t *len, int32_t dataType, int32_t idx) { OperInfo *pInfo = NULL; - - if (TSDB_DATA_TYPE_VARCHAR == dataType || TSDB_DATA_TYPE_NCHAR == dataType) { - pInfo = &operInfo[varoperatorList[rand() % tListLen(varoperatorList)]]; + + if (gCaseCtrl.optrIdxListNum > 0) { + pInfo = &operInfo[gCaseCtrl.optrIdxList[idx]]; } else { - pInfo = &operInfo[operatorList[rand() % tListLen(operatorList)]]; + if (TSDB_DATA_TYPE_VARCHAR == dataType || TSDB_DATA_TYPE_NCHAR == dataType) { + pInfo = &operInfo[varoperatorList[rand() % tListLen(varoperatorList)]]; + } else { + pInfo = &operInfo[operatorList[rand() % tListLen(operatorList)]]; + } } - + switch (pInfo->paramNum) { case 2: if (pInfo->enclose) { @@ -358,7 +393,7 @@ void bpAppendOperatorParam(BindData *data, int32_t *len, int32_t dataType) { } break; default: - printf("invalid paramNum:%d\n", pInfo->paramNum); + printf("!!!invalid paramNum:%d\n", pInfo->paramNum); exit(1); } } @@ -414,19 +449,42 @@ void generateQuerySQL(BindData *data, int32_t tblIdx) { len += sprintf(data->sql + len, "ubigdata"); break; default: - printf("invalid col type:%d", data->pBind[c].buffer_type); + printf("!!!invalid col type:%d", data->pBind[c].buffer_type); exit(1); } - bpAppendOperatorParam(data, &len, data->pBind[c].buffer_type); + bpAppendOperatorParam(data, &len, data->pBind[c].buffer_type, c); } } if (gCaseCtrl.printStmtSql) { - printf("SQL: %s\n", data->sql); + printf("\tSTMT SQL: %s\n", data->sql); } } +void generateErrorSQL(BindData *data, int32_t tblIdx) { + int32_t len = 0; + data->sql = taosMemoryCalloc(1, 1024); + + switch(tblIdx) { + case 0: + len = sprintf(data->sql, "insert into %s%d values (?, 1)", bpTbPrefix, tblIdx); + break; + case 1: + len = sprintf(data->sql, "select * from ?"); + break; + case 2: + len = sprintf(data->sql, "select * from %s%d where ? = ?", bpTbPrefix, tblIdx); + break; + default: + len = sprintf(data->sql, "select count(*) from %s%d group by ?", bpTbPrefix, tblIdx); + break; + } + + if (gCaseCtrl.printStmtSql) { + printf("\tSTMT SQL: %s\n", data->sql); + } +} void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { if (bindIdx < gCurCase->bindColNum) { @@ -551,7 +609,7 @@ int32_t prepareColData(BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; default: - printf("invalid col type:%d", dataType); + printf("!!!invalid col type:%d", dataType); exit(1); } @@ -677,6 +735,8 @@ int32_t prepareQueryData(BindData *data, int32_t tblIdx) { } + + void destroyData(BindData *data) { taosMemoryFree(data->tsData); taosMemoryFree(data->boolData); @@ -709,7 +769,7 @@ void bpFetchRows(TAOS_RES *result, bool printr, int32_t *rows) { if (printr) { memset(temp, 0, sizeof(temp)); taos_print_row(temp, row, fields, num_fields); - printf("[%s]\n", temp); + printf("\t[%s]\n", temp); } } } @@ -718,7 +778,7 @@ void bpExecQuery(TAOS * taos, char* sql, bool printr, int32_t *rows) { TAOS_RES *result = taos_query(taos, sql); int code = taos_errno(result); if (code != 0) { - printf("failed to query table, reason:%s\n", taos_errstr(result)); + printf("!!!failed to query table, reason:%s\n", taos_errstr(result)); taos_free_result(result); exit(1); } @@ -791,7 +851,7 @@ int32_t bpAppendValueString(char *buf, int type, void *value, int32_t valueLen, break; default: - printf("invalid data type:%d\n", type); + printf("!!!invalid data type:%d\n", type); exit(1); } } @@ -803,13 +863,13 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { if (gCurCase->bindRowNum > 1) { if (0 == (n++%2)) { if (taos_stmt_bind_param_batch(stmt, bind)) { - printf("taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } else { for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { if (taos_stmt_bind_single_param_batch(stmt, bind++, i)) { - printf("taos_stmt_bind_single_param_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_bind_single_param_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -817,12 +877,12 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { } else { if (0 == (n++%2)) { if (taos_stmt_bind_param_batch(stmt, bind)) { - printf("taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } else { if (taos_stmt_bind_param(stmt, bind)) { - printf("taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -834,12 +894,12 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { void bpCheckIsInsert(TAOS_STMT *stmt, int32_t insert) { int32_t isInsert = 0; if (taos_stmt_is_insert(stmt, &isInsert)) { - printf("taos_stmt_is_insert error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_is_insert error:%s\n", taos_stmt_errstr(stmt)); exit(1); } if (insert != isInsert) { - printf("is insert failed\n"); + printf("!!!is insert failed\n"); exit(1); } } @@ -847,12 +907,12 @@ void bpCheckIsInsert(TAOS_STMT *stmt, int32_t insert) { void bpCheckParamNum(TAOS_STMT *stmt) { int32_t num = 0; if (taos_stmt_num_params(stmt, &num)) { - printf("taos_stmt_num_params error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_num_params error:%s\n", taos_stmt_errstr(stmt)); exit(1); } if (gCurCase->bindColNum != num) { - printf("is insert failed\n"); + printf("!!!is insert failed\n"); exit(1); } } @@ -861,7 +921,7 @@ void bpCheckAffectedRows(TAOS_STMT *stmt, int32_t times) { int32_t rows = taos_stmt_affected_rows(stmt); int32_t insertNum = gCurCase->rowNum * gCurCase->tblNum * times; if (insertNum != rows) { - printf("affected rows %d mis-match with insert num %d\n", rows, insertNum); + printf("!!!affected rows %d mis-match with insert num %d\n", rows, insertNum); exit(1); } } @@ -869,7 +929,7 @@ void bpCheckAffectedRows(TAOS_STMT *stmt, int32_t times) { void bpCheckAffectedRowsOnce(TAOS_STMT *stmt, int32_t expectedNum) { int32_t rows = taos_stmt_affected_rows_once(stmt); if (expectedNum != rows) { - printf("affected rows %d mis-match with expected num %d\n", rows, expectedNum); + printf("!!!affected rows %d mis-match with expected num %d\n", rows, expectedNum); exit(1); } } @@ -904,16 +964,16 @@ void bpCheckQueryResult(TAOS_STMT *stmt, TAOS *taos, char *stmtSql, TAOS_MULTI_B } if (gCaseCtrl.printQuerySql) { - printf("Query SQL: %s\n", sql); + printf("\tQuery SQL: %s\n", sql); } bpExecQuery(taos, sql, gCaseCtrl.printRes, &sqlResNum); if (sqlResNum != stmtResNum) { - printf("sql res num %d mis-match stmt res num %d\n", sqlResNum, stmtResNum); + printf("!!!sql res num %d mis-match stmt res num %d\n", sqlResNum, stmtResNum); exit(1); } - printf("sql res num match stmt res num %d\n", stmtResNum); + printf("***sql res num match stmt res num %d\n", stmtResNum); } /* prepare [settbname [bind add]] exec */ @@ -923,7 +983,7 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -936,7 +996,7 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -951,14 +1011,14 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -978,7 +1038,7 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -993,7 +1053,7 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1007,14 +1067,14 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1033,7 +1093,7 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1046,7 +1106,7 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1061,13 +1121,13 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1087,7 +1147,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1100,7 +1160,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1115,12 +1175,12 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1141,7 +1201,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1154,7 +1214,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1169,7 +1229,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1179,12 +1239,12 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1206,7 +1266,7 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1221,7 +1281,7 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1235,12 +1295,12 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1264,7 +1324,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1277,7 +1337,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { sprintf(buf, "t%d", t); code = taos_stmt_set_tbname(stmt, buf); if (code != 0){ - printf("taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1292,13 +1352,13 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } } @@ -1319,6 +1379,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { return 0; } +/* select * from table */ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; @@ -1328,7 +1389,53 @@ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + for (int32_t n = 0; n< (gCurCase->rowNum/gCurCase->bindRowNum); ++n) { + bpCheckIsInsert(stmt, 0); + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + + if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckQueryResult(stmt, taos, data.sql, data.pBind + n * gCurCase->bindColNum); + } + + bpCheckIsInsert(stmt, 0); + + destroyData(&data); + } + + return 0; +} + +/* value in query sql */ +int querySUBTTest2(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + memset(&data, 0, sizeof(data)); + prepareQueryData(&data, t); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1344,12 +1451,12 @@ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { } if (taos_stmt_add_batch(stmt)) { - printf("taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); exit(1); } if (taos_stmt_execute(stmt) != 0) { - printf("taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); exit(1); } @@ -1365,6 +1472,28 @@ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { } +int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + memset(&data, 0, sizeof(data)); + generateErrorSQL(&data, t); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("*taos_stmt_prepare error as expected, error:%s\n", taos_stmt_errstr(stmt)); + } else { + printf("!!!taos_stmt_prepare succeed, which should be error\n"); + exit(1); + } + + destroyData(&data); + } + + return 0; +} + + #if 0 //1 tables 10 records @@ -1388,6 +1517,72 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + unsigned long long starttime = taosGetTimestampUs(); + + char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + exit(1); + } + + int id = 0; + for (int zz = 0; zz < 1; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname_tags(stmt, buf, tags); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); + } + + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + + ++id; + + unsigned long long endtime = taosGetTimestampUs(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); + + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); + + return 0; +} + + + + +//1 tables 10 records +int stmt_funcb_autoctb2(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; + } v = {0}; + + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); + + int *lb = taosMemoryMalloc(10 * sizeof(int)); + + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + // int one_null = 1; int one_not_null = 0; @@ -1537,7 +1732,7 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { unsigned long long starttime = taosGetTimestampUs(); - char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; + char *sql = "insert into ? using stb1 tags(1,true,2,3,4,5.0,6.0,'a','b') values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); @@ -1580,8 +1775,9 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { + //1 tables 10 records -int stmt_funcb_autoctb2(TAOS_STMT *stmt) { +int stmt_funcb_autoctb3(TAOS_STMT *stmt) { struct { int64_t *ts; int8_t b[10]; @@ -1701,56 +1897,31 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; + tags[i+0].buffer_type = TSDB_DATA_TYPE_BOOL; + tags[i+0].buffer = v.b; tags[i+0].is_null = &one_not_null; tags[i+0].length = NULL; - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; + tags[i+1].buffer_type = TSDB_DATA_TYPE_SMALLINT; + tags[i+1].buffer = v.v2; tags[i+1].is_null = &one_not_null; tags[i+1].length = NULL; - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; + tags[i+2].buffer_type = TSDB_DATA_TYPE_FLOAT; + tags[i+2].buffer = v.f4; tags[i+2].is_null = &one_not_null; tags[i+2].length = NULL; - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; + tags[i+3].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[i+3].buffer = v.bin; tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; + tags[i+3].length = (uintptr_t *)lb; } unsigned long long starttime = taosGetTimestampUs(); - char *sql = "insert into ? using stb1 tags(1,true,2,3,4,5.0,6.0,'a','b') values(?,?,?,?,?,?,?,?,?,?)"; + char *sql = "insert into ? using stb1 tags(1,?,2,?,4,?,6.0,?,'b') values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); @@ -1794,8 +1965,10 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { + + //1 tables 10 records -int stmt_funcb_autoctb3(TAOS_STMT *stmt) { +int stmt_funcb_autoctb4(TAOS_STMT *stmt) { struct { int64_t *ts; int8_t b[10]; @@ -1813,7 +1986,7 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { int *lb = taosMemoryMalloc(10 * sizeof(int)); TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*5); // int one_null = 1; int one_not_null = 0; @@ -1835,7 +2008,7 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { memset(v.bin[i], '0'+i%10, 40); } - for (int i = 0; i < 10; i+=10) { + for (int i = 0; i < 5; i+=5) { params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[i+0].buffer_length = sizeof(int64_t); params[i+0].buffer = &v.ts[10*i/10]; @@ -1850,200 +2023,9 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { params[i+1].is_null = is_null; params[i+1].num = 10; - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+0].buffer = v.b; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+1].buffer = v.v2; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+2].buffer = v.f4; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+3].buffer = v.bin; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = (uintptr_t *)lb; - } - - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? using stb1 tags(1,?,2,?,4,?,6.0,?,'b') values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - exit(1); - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, tags); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); - - return 0; -} - - - - - - - -//1 tables 10 records -int stmt_funcb_autoctb4(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*5); - -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 5; i+=5) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_INT; - params[i+2].buffer_length = sizeof(int32_t); - params[i+2].buffer = v.v4; + params[i+2].buffer_type = TSDB_DATA_TYPE_INT; + params[i+2].buffer_length = sizeof(int32_t); + params[i+2].buffer = v.v4; params[i+2].length = NULL; params[i+2].is_null = is_null; params[i+2].num = 10; @@ -2348,175 +2330,28 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); + unsigned long long starttime = taosGetTimestampUs(); - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); + char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + exit(1); } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; + int id = 0; + for (int zz = 0; zz < 1; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname_tags(stmt, buf, NULL); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname_tags. code:%s\n", taos_stmt_errstr(stmt)); + return -1; + } - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - exit(1); - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, NULL); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. code:%s\n", taos_stmt_errstr(stmt)); - return -1; - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + } if (taos_stmt_execute(stmt) != 0) { printf("failed to execute insert statement.\n"); @@ -2544,1678 +2379,205 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { -//1 tables 10 records -int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); - -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? using stb1 (id1,id2,id3,id4,id5,id6,id7,id8,id9) tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt)); - return -1; - //exit(1); - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, NULL); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); - return -1; - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); - - return 0; -} - - - - -//1 tables 10 records -int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); - -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, tags); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - - code = taos_stmt_bind_param_batch(stmt, params + id * 10); - if (code != 0) { - printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - - code = taos_stmt_bind_param_batch(stmt, params + id * 10); - if (code != 0) { - printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); - return -1; - } - - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); - - return 0; -} - - - - - - -//1 tables 10 records -int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); - -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(NULL, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(NULL)); - return -1; - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, tags); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - - code = taos_stmt_bind_param_batch(stmt, params + id * 10); - if (code != 0) { - printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - - code = taos_stmt_bind_param_batch(stmt, params + id * 10); - if (code != 0) { - printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); - return -1; - } - - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); - - return 0; -} - -//samets -int stmt_funcb4(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - - - -//1table 18000 reocrds -int stmt_funcb5(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[18000]; - int8_t v1[18000]; - int16_t v2[18000]; - int32_t v4[18000]; - int64_t v8[18000]; - float f4[18000]; - double f8[18000]; - char bin[18000][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(18000 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 18000); - char* no_null = taosMemoryMalloc(sizeof(char) * 18000); - - for (int i = 0; i < 18000; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 30000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[18000*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 18000; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 18000; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 18000; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 18000; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 18000; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 18000; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 18000; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 18000; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 18000; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 18000; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into m0 values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 10; l++) { - for (int zz = 0; zz < 1; zz++) { - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - ++id; - - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - -//1table 200000 reocrds -int stmt_funcb_ssz1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int b[30000]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 30000 * 3000); - - int *lb = taosMemoryMalloc(30000 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* no_null = taosMemoryMalloc(sizeof(int) * 200000); - - for (int i = 0; i < 30000; ++i) { - lb[i] = 40; - no_null[i] = 0; - v.b[i] = (int8_t)(i % 2); - } - - for (int i = 0; i < 30000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[30000*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 30000; - - params[i+1].buffer_type = TSDB_DATA_TYPE_INT; - params[i+1].buffer_length = sizeof(int); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = no_null; - params[i+1].num = 30000; - } - - int64_t tts = 0; - for (int64_t i = 0; i < 90000000LL; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 10; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - ++id; - - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(no_null); - - return 0; -} - - -//one table 60 records one time -int stmt_funcb_s1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - - - - - -//300 tables 60 records single column bind -int stmt_funcb_sc1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - for (int col=0; col < 10; ++col) { - taos_stmt_bind_single_param_batch(stmt, params + id++, col); - } - - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - -//1 tables 60 records single column bind -int stmt_funcb_sc2(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } +//1 tables 10 records +int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; + } v = {0}; + + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); + + int *lb = taosMemoryMalloc(10 * sizeof(int)); + + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } unsigned long long starttime = taosGetTimestampUs(); - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + char *sql = "insert into ? using stb1 (id1,id2,id3,id4,id5,id6,id7,id8,id9) tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; int code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt)); + return -1; + //exit(1); } int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - for (int col=0; col < 10; ++col) { - taos_stmt_bind_single_param_batch(stmt, params + id++, col); - } - - taos_stmt_add_batch(stmt); + for (int zz = 0; zz < 1; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname_tags(stmt, buf, NULL); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); + return -1; + } - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } + taos_stmt_bind_param_batch(stmt, params + id * 10); + taos_stmt_add_batch(stmt); + } - } - + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); } + ++id; + unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); taosMemoryFree(lb); taosMemoryFree(params); taosMemoryFree(is_null); taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } -//10 tables [1...10] records single column bind -int stmt_funcb_sc3(TAOS_STMT *stmt) { + + +//1 tables 10 records +int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { struct { int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; } v = {0}; - v.ts = taosMemoryMalloc(sizeof(int64_t) * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + + unsigned long long starttime = taosGetTimestampUs(); + + char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt)); + exit(1); } - int g = 0; - for (int i = 0; i < 600; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = g%10+1; + int id = 0; + for (int zz = 0; zz < 1; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname_tags(stmt, buf, tags); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname_tags. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + code = taos_stmt_bind_param_batch(stmt, params + id * 10); + if (code != 0) { + printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = g%10+1; + code = taos_stmt_bind_param_batch(stmt, params + id * 10); + if (code != 0) { + printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); + return -1; + } - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = g%10+1; + taos_stmt_add_batch(stmt); + } - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = g%10+1; + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = g%10+1; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = g%10+1; + ++id; - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = g%10+1; + unsigned long long endtime = taosGetTimestampUs(); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = g%10+1; + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = g%10+1; + return 0; +} - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = g%10+1; - ++g; - } - int64_t tts = 1591060628000; - for (int i = 0; i < 60; ++i) { - v.ts[i] = tts + i; - } + + + + +//1 tables 10 records +int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { + struct { + int64_t *ts; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; + } v = {0}; + + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); + + int *lb = taosMemoryMalloc(10 * sizeof(int)); + + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); unsigned long long starttime = taosGetTimestampUs(); - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); + char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; + int code = taos_stmt_prepare(NULL, sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(NULL)); + return -1; } int id = 0; - for (int zz = 0; zz < 10; zz++) { + for (int zz = 0; zz < 1; zz++) { char buf[32]; sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); + code = taos_stmt_set_tbname_tags(stmt, buf, tags); if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); + printf("failed to execute taos_stmt_set_tbname_tags. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); } - for (int col=0; col < 10; ++col) { - taos_stmt_bind_single_param_batch(stmt, params + id++, col); + code = taos_stmt_bind_param_batch(stmt, params + id * 10); + if (code != 0) { + printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + code = taos_stmt_bind_param_batch(stmt, params + id * 10); + if (code != 0) { + printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); + return -1; } taos_stmt_add_batch(stmt); @@ -4226,17 +2588,21 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { exit(1); } + ++id; + unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); + printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); taosMemoryFree(v.ts); taosMemoryFree(lb); taosMemoryFree(params); taosMemoryFree(is_null); taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } + #endif @@ -4249,10 +2615,10 @@ void prepareCheckResultImpl(TAOS * taos, char *tname, bool printr, int expec if (rows == expected) { if (!silent) { - printf("%d rows are fetched as expected from %s\n", rows, tname); + printf("***%d rows are fetched as expected from %s\n", rows, tname); } } else { - printf("!!!expect %d rows, but %d rows are fetched from %s\n", expected, rows, tname); + printf("!!!expect rows %d mis-match rows %d fetched from %s\n", expected, rows, tname); exit(1); } } @@ -4302,7 +2668,7 @@ int sql_perf1(TAOS *taos) { result = taos_query(taos, sql[i]); int code = taos_errno(result); if (code != 0) { - printf("failed to query table, reason:%s\n", taos_errstr(result)); + printf("%d failed to query table, reason:%s\n", taos_errstr(result)); taos_free_result(result); exit(1); } @@ -4539,7 +2905,7 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * blen += sprintf(buf + blen, ")"); if (gCaseCtrl.printCreateTblSql) { - printf("Create Table SQL:%s\n", buf); + printf("\tCreate Table SQL:%s\n", buf); } } @@ -4553,7 +2919,7 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int prepareStb) { result = taos_query(taos, "create database demo keep 36500"); code = taos_errno(result); if (code != 0) { - printf("failed to create database, reason:%s\n", taos_errstr(result)); + printf("!!!failed to create database, reason:%s\n", taos_errstr(result)); taos_free_result(result); exit(1); } @@ -4570,7 +2936,7 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int prepareStb) { result = taos_query(taos, buf); code = taos_errno(result); if (code != 0) { - printf("failed to create table, reason:%s\n", taos_errstr(result)); + printf("!!!failed to create table, reason:%s\n", taos_errstr(result)); taos_free_result(result); exit(1); } @@ -4583,7 +2949,7 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int prepareStb) { result = taos_query(taos, buf); code = taos_errno(result); if (code != 0) { - printf("failed to create table, reason:%s\n", taos_errstr(result)); + printf("!!!failed to create table, reason:%s\n", taos_errstr(result)); taos_free_result(result); exit(1); } @@ -4654,7 +3020,7 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { stmt = taos_stmt_init(taos); if (NULL == stmt) { - printf("taos_stmt_init failed, error:%s\n", taos_stmt_errstr(stmt)); + printf("!!!taos_stmt_init failed, error:%s\n", taos_stmt_errstr(stmt)); exit(1); } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index a8e2c22d42e5b6db5b9b4390b0109ea489729330..b95e822df3e959db53f7f0d09bd2b7b4ae2796b9 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -39,6 +39,7 @@ ./test.sh -f tsim/query/explain.sim ./test.sh -f tsim/query/session.sim ./test.sh -f tsim/query/scalarNull.sim +./test.sh -f tsim/query/udf.sim # ---- qnode ./test.sh -f tsim/qnode/basic1.sim @@ -69,7 +70,7 @@ ./test.sh -f tsim/stable/dnode3.sim ./test.sh -f tsim/stable/metrics.sim ./test.sh -f tsim/stable/refcount.sim -# ./test.sh -f tsim/stable/show.sim +#./test.sh -f tsim/stable/show.sim ./test.sh -f tsim/stable/values.sim ./test.sh -f tsim/stable/vnode3.sim diff --git a/tests/script/sh/copy_udf.sh b/tests/script/sh/copy_udf.sh new file mode 100755 index 0000000000000000000000000000000000000000..e1d9ff53d27462fcb05fa6902301faa4f625d44a --- /dev/null +++ b/tests/script/sh/copy_udf.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set +e +#set -x + +echo "Executing copy_udf.sh" + +SCRIPT_DIR=`dirname $0` +cd $SCRIPT_DIR/../ + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TAOS_DIR=`pwd` +UDF1_DIR=`find $TAOS_DIR -name "libudf1.so"|grep lib|head -n1` +UDF2_DIR=`find $TAOS_DIR -name "libudf2.so"|grep lib|head -n1` + +echo $UDF1_DIR +echo $UDF2_DIR + +UDF_TMP=/tmp/udf +mkdir $UDF_TMP +rm $UDF_TMP/libudf1.so +rm $UDF_TMP/libudf2.so + +echo "Copy udf shared library files to $UDF_TMP" + +cp $UDF1_DIR $UDF_TMP +cp $UDF2_DIR $UDF_TMP diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index b847149be3c4a189a87e8722d4b08a15ba7bba05..40882306c8aaf6477dc91c13b632a2cb9f8605e1 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -58,15 +58,15 @@ endi print ============= create database #database_option: { -# BLOCKS value [3~1000, default: 6] +# | BUFFER value [3~16384, default: 96] +# | PAGES value [64~16384, default: 256] # | CACHELAST value [0, 1, 2, 3] # | FSYNC value [0 ~ 180000 ms] # | KEEP value [days, 365000] -# | QUORUM value [1 | 2] # | REPLICA value [1 | 3] # | WAL value [1 | 2] -sql create database db CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 1 WAL 2 VGROUPS 6 SINGLE_STABLE 1 +sql create database db CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 3 WAL 2 VGROUPS 6 SINGLE_STABLE 1 sql show databases print rows: $rows print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -86,7 +86,7 @@ endi if $data3_db != 0 then # ntables return -1 endi -if $data4_db != 1 then # replica +if $data4_db != 3 then # replica return -1 endi if $data5_db != nostrict then # strict @@ -98,31 +98,34 @@ endi if $data7_db != 1440000,1440000,1440000 then # keep return -1 endi -#if $data8_db != 3 then # cache -# return -1 -#endi -#if $data9_db != 7 then # blocks -# return -1 -#endi -if $data10_db != 10 then # minrows +if $data8_db != 96 then # buffer + return -1 +endi +if $data9_db != 4 then # pagesize + return -1 +endi +if $data10_db != 256 then # pages + return -1 +endi +if $data11_db != 10 then # minrows return -1 endi -if $data11_db != 8000 then # maxrows +if $data12_db != 8000 then # maxrows return -1 endi -if $data12_db != 2 then # wal +if $data13_db != 2 then # wal return -1 endi -if $data13_db != 1000 then # fsync +if $data14_db != 1000 then # fsync return -1 endi -if $data14_db != 0 then # comp +if $data15_db != 0 then # comp return -1 endi -if $data15_db != 3 then # cachelast +if $data16_db != 3 then # cachelast return -1 endi -if $data16_db != ns then # precision +if $data17_db != ns then # precision return -1 endi @@ -302,14 +305,14 @@ sql_error alter database db maxrows 10 # little than minrows print ============== step wal sql alter database db wal 1 sql show databases -print wal $data12_db -if $data12_db != 1 then +print wal $data13_db +if $data13_db != 1 then return -1 endi sql alter database db wal 2 sql show databases -print wal $data12_db -if $data12_db != 2 then +print wal $data13_db +if $data13_db != 2 then return -1 endi @@ -321,20 +324,20 @@ sql_error alter database db wal -1 print ============== modify fsync sql alter database db fsync 2000 sql show databases -print fsync $data13_db -if $data13_db != 2000 then +print fsync $data14_db +if $data14_db != 2000 then return -1 endi sql alter database db fsync 500 sql show databases -print fsync $data13_db -if $data13_db != 500 then +print fsync $data14_db +if $data14_db != 500 then return -1 endi sql alter database db fsync 0 sql show databases -print fsync $data13_db -if $data13_db != 0 then +print fsync $data14_db +if $data14_db != 0 then return -1 endi sql_error alter database db fsync 180001 @@ -353,32 +356,32 @@ sql_error alter database db comp -1 print ============== modify cachelast [0, 1, 2, 3] sql alter database db cachelast 2 sql show databases -print cachelast $data15_db -if $data15_db != 2 then +print cachelast $data16_db +if $data16_db != 2 then return -1 endi sql alter database db cachelast 1 sql show databases -print cachelast $data15_db -if $data15_db != 1 then +print cachelast $data16_db +if $data16_db != 1 then return -1 endi sql alter database db cachelast 0 sql show databases -print cachelast $data15_db -if $data15_db != 0 then +print cachelast $data16_db +if $data16_db != 0 then return -1 endi sql alter database db cachelast 2 sql show databases -print cachelast $data15_db -if $data15_db != 2 then +print cachelast $data16_db +if $data16_db != 2 then return -1 endi sql alter database db cachelast 3 sql show databases -print cachelast $data15_db -if $data15_db != 3 then +print cachelast $data16_db +if $data16_db != 3 then return -1 endi diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index 8a3826fe21bfefb5afc3b4d0f096855fbc1f1e48..d7d72c4a1df1116d563c2f9283994e9807eac9d7 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -58,8 +58,9 @@ endi print ============= create database with all options #database_option: { -# | BLOCKS value [3~1000, default: 6] -# | CACHE value [default: 16] +# | BUFFER value [3~16384, default: 96] +# | PAGES value [64~16384, default: 256] +# | PAGESIZE value [1~16384, default: 4] # | CACHELAST value [0, 1, 2, 3, default: 0] # | COMP [0 | 1 | 2, default: 2] # | DAYS value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day] @@ -68,24 +69,18 @@ print ============= create database with all options # | MINROWS value [10~1000, default: 100] # | KEEP value [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day] # | PRECISION ['ms' | 'us' | 'ns', default: ms] -# | QUORUM value [1 | 2, default: 1] # | REPLICA value [1 | 3, default: 1] -# | TTL value [1d ~ , default: 1] # | WAL value [1 | 2, default: 1] # | VGROUPS value [default: 2] # | SINGLE_STABLE [0 | 1, default: ] -# | STREAM_MODE [0 | 1, default: ] # #$data0_db : name #$data1_db : create_time #$data2_db : vgroups #$data3_db : ntables #$data4_db : replica -#$data5_db : quorum #$data6_db : days #$data7_db : keep -#$data8_db : cache -#$data9_db : blocks #$data10_db : minrows #$data11_db : maxrows #$data12_db : wal @@ -124,31 +119,34 @@ endi if $data7_db != 5256000,5256000,5256000 then # keep return -1 endi -#if $data8_db != 16 then # cache -# return -1 -#endi -#if $data9_db != 6 then # blocks -# return -1 -#endi -if $data10_db != 100 then # minrows +if $data8_db != 96 then # buffer + return -1 +endi +if $data9_db != 4 then # pagesize + return -1 +endi +if $data10_db != 256 then # pages + return -1 +endi +if $data11_db != 100 then # minrows return -1 endi -if $data11_db != 4096 then # maxrows +if $data12_db != 4096 then # maxrows return -1 endi -if $data12_db != 1 then # wal +if $data13_db != 1 then # wal return -1 endi -if $data13_db != 3000 then # fsync +if $data14_db != 3000 then # fsync return -1 endi -if $data14_db != 2 then # comp +if $data15_db != 2 then # comp return -1 endi -if $data15_db != 0 then # cachelast +if $data16_db != 0 then # cachelast return -1 endi -if $data16_db != ms then # precision +if $data17_db != ms then # precision return -1 endi sql drop database db @@ -194,7 +192,7 @@ print ====> CACHELAST value [0, 1, 2, 3, default: 0] sql create database db CACHELAST 1 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data15_db != 1 then +if $data16_db != 1 then return -1 endi sql drop database db @@ -202,7 +200,7 @@ sql drop database db sql create database db CACHELAST 2 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data15_db != 2 then +if $data16_db != 2 then return -1 endi sql drop database db @@ -210,7 +208,7 @@ sql drop database db sql create database db CACHELAST 3 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data15_db != 3 then +if $data16_db != 3 then return -1 endi sql drop database db @@ -221,7 +219,7 @@ print ====> COMP [0 | 1 | 2, default: 2] sql create database db COMP 1 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data14_db != 1 then +if $data15_db != 1 then return -1 endi sql drop database db @@ -229,7 +227,7 @@ sql drop database db sql create database db COMP 0 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data14_db != 0 then +if $data15_db != 0 then return -1 endi sql drop database db @@ -280,7 +278,7 @@ print ====> FSYNC value [0 ~ 180000 ms, default: 3000] sql create database db FSYNC 0 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data13_db != 0 then +if $data14_db != 0 then return -1 endi sql drop database db @@ -288,7 +286,7 @@ sql drop database db sql create database db FSYNC 180000 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data13_db != 180000 then +if $data14_db != 180000 then return -1 endi sql drop database db @@ -299,10 +297,10 @@ print ====> MAXROWS value [200~10000, default: 4096], MINROWS value [10~1000, de sql create database db MAXROWS 10000 MINROWS 1000 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data10_db != 1000 then +if $data11_db != 1000 then return -1 endi -if $data11_db != 10000 then +if $data12_db != 10000 then return -1 endi sql drop database db @@ -310,10 +308,10 @@ sql drop database db sql create database db MAXROWS 200 MINROWS 10 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data10_db != 10 then +if $data11_db != 10 then return -1 endi -if $data11_db != 200 then +if $data12_db != 200 then return -1 endi sql drop database db @@ -331,7 +329,7 @@ print ====> PRECISION ['ms' | 'us' | 'ns', default: ms] sql create database db PRECISION 'us' sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data16_db != us then +if $data17_db != us then return -1 endi sql drop database db @@ -339,7 +337,7 @@ sql drop database db sql create database db PRECISION 'ns' sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data16_db != ns then +if $data17_db != ns then return -1 endi sql drop database db @@ -412,7 +410,7 @@ print ====> WAL value [1 | 2, default: 1] sql create database db WAL 2 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data12_db != 2 then +if $data13_db != 2 then return -1 endi sql drop database db @@ -420,7 +418,7 @@ sql drop database db sql create database db WAL 1 sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data12_db != 1 then +if $data13_db != 1 then return -1 endi sql drop database db diff --git a/tests/script/tsim/query/complex_having.sim b/tests/script/tsim/query/complex_having.sim index 86d4f7d4ca109035f1f9420debf449d18a5f89b9..ef58228f342f7f38460ebced4e9867dced0c9b44 100644 --- a/tests/script/tsim/query/complex_having.sim +++ b/tests/script/tsim/query/complex_having.sim @@ -215,6 +215,7 @@ sql select count(c1) from stb1 print ====> sql : select count(c1) from stb1 print ====> rows: $data00 if $data00 != 20 then + print expect 20, actual: $data00 return -1 endi diff --git a/tests/script/tsim/query/explain.sim b/tests/script/tsim/query/explain.sim index 66d3c48f5d0169f01413805234eb8dfa91f97a22..71f7969c837aac2126963786fdbce4303c2e620b 100644 --- a/tests/script/tsim/query/explain.sim +++ b/tests/script/tsim/query/explain.sim @@ -25,15 +25,15 @@ sql insert into tb3 values (now, 3, "Hash (cost=229.20..229.20 rows=101 width=2 sql create table tb4 using st1 tags(4); sql insert into tb4 values (now, 4, "Bitmap Heap Scan on tenk1 t1 (cost=5.07..229.20 rows=101 width=244) (actual time=0.080..0.526 rows=100 loops=1)"); -sql create table tb1 using st2 tags(1); -sql insert into tb1 values (now, 1, "Hash Join (cost=230.47..713.98 rows=101 width=488) (actual time=0.711..7.427 rows=100 loops=1)"); +#sql create table tb1 using st2 tags(1); +#sql insert into tb1 values (now, 1, "Hash Join (cost=230.47..713.98 rows=101 width=488) (actual time=0.711..7.427 rows=100 loops=1)"); -sql create table tb2 using st2 tags(2); -sql insert into tb2 values (now, 2, "Seq Scan on tenk2 t2 (cost=0.00..445.00 rows=10000 width=244) (actual time=0.007..2.583 rows=10000 loops=1)"); -sql create table tb3 using st2 tags(3); -sql insert into tb3 values (now, 3, "Hash (cost=229.20..229.20 rows=101 width=244) (actual time=0.659..0.659 rows=100 loops=1)"); -sql create table tb4 using st2 tags(4); -sql insert into tb4 values (now, 4, "Bitmap Heap Scan on tenk1 t1 (cost=5.07..229.20 rows=101 width=244) (actual time=0.080..0.526 rows=100 loops=1)"); +#sql create table tb2 using st2 tags(2); +#sql insert into tb2 values (now, 2, "Seq Scan on tenk2 t2 (cost=0.00..445.00 rows=10000 width=244) (actual time=0.007..2.583 rows=10000 loops=1)"); +#sql create table tb3 using st2 tags(3); +#sql insert into tb3 values (now, 3, "Hash (cost=229.20..229.20 rows=101 width=244) (actual time=0.659..0.659 rows=100 loops=1)"); +#sql create table tb4 using st2 tags(4); +#sql insert into tb4 values (now, 4, "Bitmap Heap Scan on tenk1 t1 (cost=5.07..229.20 rows=101 width=244) (actual time=0.080..0.526 rows=100 loops=1)"); print ======== step2 diff --git a/tests/script/tsim/query/session.sim b/tests/script/tsim/query/session.sim index a69b6249fc3f0389ca2982d11c9afc1435f7ced6..c39956c0df979a6c702d23782d0320223f4b8fad 100644 --- a/tests/script/tsim/query/session.sim +++ b/tests/script/tsim/query/session.sim @@ -65,6 +65,15 @@ sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.031', 5) sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.036', 6) sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.51', 7) +# vnode does not return the precision of the table +print ====> create database d1 precision 'us' +sql create database d1 precision 'us' +sql use d1 +sql create table dev_001 (ts timestamp ,i timestamp ,j int) +sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) +sql create table secondts(ts timestamp,t2 timestamp,i int) +sql insert into secondts values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) + $loop_test = 0 loop_test_pos: @@ -288,15 +297,6 @@ sql_error sql select count(*) from dev_001 session(ts,0s) sql_error select count(*) from dev_001 session(i,1y) sql_error select count(*) from dev_001 session(ts,1d) where ts <'2020-05-20 0:0:0' -# vnode does not return the precision of the table -print ====> create database d1 precision 'us' -sql create database d1 precision 'us' -sql use d1 -sql create table dev_001 (ts timestamp ,i timestamp ,j int) -sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) -sql create table secondts(ts timestamp,t2 timestamp,i int) -sql insert into secondts values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) - #print ====> select count(*) from dev_001 session(ts,1u) #sql select _wstartts, count(*) from dev_001 session(ts,1u) #print rows: $rows diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim new file mode 100644 index 0000000000000000000000000000000000000000..c76569b40fff3ee61ec0accdfeac7897f887c265 --- /dev/null +++ b/tests/script/tsim/query/udf.sim @@ -0,0 +1,69 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 2 +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 + +print ========= start dnode1 as LEADER +system sh/exec.sh -n dnode1 -s start +sleep 2000 +sql connect + +print ======== step1 udf +system sh/copy_udf.sh +sql create database udf vgroups 3; +sql use udf; +sql show databases; + +sql create table t (ts timestamp, f int); +sql insert into t values(now, 1)(now+1s, 2); + +sql create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8; +sql create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8; +sql show functions; +if $rows != 2 then + return -1 +endi +sql select udf1(f) from t; +if $rows != 2 then + return -1 +endi +if $data00 != 88 then + return -1 +endi +if $data10 != 88 then + return -1 +endi + +sql select udf2(f) from t; +if $rows != 1 then + return -1 +endi +if $data00 != 2.236067977 then + return -1 +endi + +sql create table t2 (ts timestamp, f1 int, f2 int); +sql insert into t2 values(now, 0, 0)(now+1s, 1, 1); +sql select udf1(f1, f2) from t2; +if $rows != 2 then + return -1 +endi +if $data00 != 88 then + return -1 +endi +if $data10 != 88 then + return -1 +endi + +sql select udf2(f1, f2) from t2; +if $rows != 1 then + return -1 +endi +if $data00 != 1.414213562 then + return -1 +endi + +#sql drop function udf1; +#sql drop function udf2; +system sh/exec.sh -n dnode1 -s stop -x SIGKILL diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index ca6cd1c11a7af848172c8e3c7f4e5e3947b897d8..e171d1abb9e3249c5b93ae01899b596e6238a4c4 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -3,9 +3,6 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start - -# after mnode support, del sleep 2000, and check dnode status -sleep 2000 sql connect #$loop_cnt = 0 @@ -101,7 +98,7 @@ if $rows != 1 then endi #sql select * from information_schema.user_streams sql select * from information_schema.user_tables -if $rows != 1 then +if $rows != 28 then return -1 endi #sql select * from information_schema.user_table_distributed @@ -199,7 +196,7 @@ if $rows != 1 then endi #sql select * from information_schema.user_streams sql select * from information_schema.user_tables -if $rows != 1 then +if $rows != 28 then return -1 endi #sql select * from information_schema.user_table_distributed diff --git a/tests/script/tsim/sync/oneReplica1VgElect.sim b/tests/script/tsim/sync/oneReplica1VgElect.sim index 8e99bc4b2eaee0c1343c29f0335ee2e46f51db64..bb9b3f449640818d888137721350b0cea90eebae 100644 --- a/tests/script/tsim/sync/oneReplica1VgElect.sim +++ b/tests/script/tsim/sync/oneReplica1VgElect.sim @@ -83,7 +83,7 @@ print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $dat if $rows != 3 then return -1 endi -if $data(db)[20] != ready then +if $data(db)[19] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim b/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim index e85d5ea4373d2e27ee219a6918fd859468c3d5da..7ceeb2806b320014c2b35ea5c640063e44793063 100644 --- a/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim +++ b/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim @@ -83,7 +83,7 @@ print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $dat if $rows != 3 then return -1 endi -if $data(db)[20] != ready then +if $data(db)[19] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/sync/threeReplica1VgElect.sim b/tests/script/tsim/sync/threeReplica1VgElect.sim index e06bd86daa0b8927db25d6dd706423a6385184e1..7f8c8339cbb44de39cb0da9dca12c19159209d3f 100644 --- a/tests/script/tsim/sync/threeReplica1VgElect.sim +++ b/tests/script/tsim/sync/threeReplica1VgElect.sim @@ -83,7 +83,7 @@ print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $dat if $rows != 3 then return -1 endi -if $data(db)[20] != ready then +if $data(db)[19] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim index 797baea811ab9519d3310c8a0be2bb9708bcceb5..1e12e8565feedc200151b64d37bc6663b4e09b53 100644 --- a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim +++ b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim @@ -83,7 +83,7 @@ print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $dat if $rows != 3 then return -1 endi -if $data(db)[20] != ready then +if $data(db)[19] != ready then goto check_db_ready endi diff --git a/tests/script/tsim/table/basic1.sim b/tests/script/tsim/table/basic1.sim index be3b718fae52c49440351bb594f73defab05130d..913ced74aa75d3df0df3e2ab1574e4d705872c18 100644 --- a/tests/script/tsim/table/basic1.sim +++ b/tests/script/tsim/table/basic1.sim @@ -91,9 +91,9 @@ print =============== create normal table sql create database ndb sql use ndb sql create table nt0 (ts timestamp, i int) -sql create table if not exists nt0 (ts timestamp, i int) +# sql create table if not exists nt0 (ts timestamp, i int) sql create table nt1 (ts timestamp, i int) -sql create table if not exists nt1 (ts timestamp, i int) +# sql create table if not exists nt1 (ts timestamp, i int) sql create table if not exists nt3 (ts timestamp, i int) sql show tables diff --git a/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim b/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim index db56bcf743dfa2cb10e1f3c8dca5213c6cc0d09a..e32a1df8025f1105ffcaee8e0f081149cb756f91 100644 --- a/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim +++ b/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim @@ -39,7 +39,7 @@ sql show databases print ==> rows: $rows print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] -if $data(db)[20] != nostrict then +if $data(db)[19] != nostrict then sleep 100 $loop_cnt = $loop_cnt + 1 goto check_db_ready diff --git a/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim b/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim index d7fa58558f97ae82ce4e643cbb259fbe815e9ccf..4750aab214e074408f3076cdd0533315fa436ea5 100644 --- a/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim +++ b/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim @@ -39,7 +39,7 @@ sql show databases print ==> rows: $rows print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] -if $data(db)[20] != nostrict then +if $data(db)[19] != nostrict then sleep 100 $loop_cnt = $loop_cnt + 1 goto check_db_ready diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index 4e9a1155002fa0decd1cb6fc114f8183ca766c94..f6dfe3f75c795ab8bd8eefc7b9d043d75854dc2e 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -73,17 +73,19 @@ class TDTestCase: hostname = socket.gethostname() serverPort = '7080' rpcDebugFlagVal = '143' - clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135'} + clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} clientCfgDict["serverPort"] = serverPort clientCfgDict["firstEp"] = hostname + ':' + serverPort clientCfgDict["secondEp"] = hostname + ':' + serverPort clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + clientCfgDict["fqdn"] = hostname - updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':''} + updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} updatecfgDict["clientCfg"] = clientCfgDict updatecfgDict["serverPort"] = serverPort updatecfgDict["firstEp"] = hostname + ':' + serverPort updatecfgDict["secondEp"] = hostname + ':' + serverPort + updatecfgDict["fqdn"] = hostname print ("===================: ", updatecfgDict) @@ -288,7 +290,7 @@ class TDTestCase: retCode = taos_command(buildPath, "f", keyDict['f'], 'performance_schema', keyDict['c'], '', '', '') print("============ ret code: ", retCode) if retCode != "TAOS_OK": - tdLog.exit("taos -s fail") + tdLog.exit("taos -f fail") print ("========== check new db ==========") tdSql.query("show databases") diff --git a/tests/system-test/0-others/taosShellError.py b/tests/system-test/0-others/taosShellError.py new file mode 100644 index 0000000000000000000000000000000000000000..5f2f79982a58fe33e361f7c05926fc7c276f84d7 --- /dev/null +++ b/tests/system-test/0-others/taosShellError.py @@ -0,0 +1,282 @@ + +import taos +import sys +import time +import socket +import pexpect +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +def taos_command (buildPath, key, value, expectString, cfgDir, sqlString='', key1='', value1=''): + if len(key) == 0: + tdLog.exit("taos test key is null!") + + taosCmd = buildPath + '/build/bin/taos ' + if len(cfgDir) != 0: + taosCmd = taosCmd + '-c ' + cfgDir + + taosCmd = taosCmd + ' -' + key + if len(value) != 0: + if key == 'p': + taosCmd = taosCmd + value + else: + taosCmd = taosCmd + ' ' + value + + if len(key1) != 0: + taosCmd = taosCmd + ' -' + key1 + if key1 == 'p': + taosCmd = taosCmd + value1 + else: + if len(value1) != 0: + taosCmd = taosCmd + ' ' + value1 + + tdLog.info ("taos cmd: %s" % taosCmd) + + child = pexpect.spawn(taosCmd, timeout=3) + #output = child.readline() + #print (output.decode()) + if len(expectString) != 0: + i = child.expect([expectString, pexpect.TIMEOUT, pexpect.EOF], timeout=6) + else: + i = child.expect([pexpect.TIMEOUT, pexpect.EOF], timeout=6) + + retResult = child.before.decode() + print("cmd return result:\n%s\n"%retResult) + #print(child.after.decode()) + if i == 0: + print ('taos login success! Here can run sql, taos> ') + if len(sqlString) != 0: + child.sendline (sqlString) + w = child.expect(["Query OK", pexpect.TIMEOUT, pexpect.EOF], timeout=1) + retResult = child.before.decode() + if w == 0: + return "TAOS_OK", retResult + else: + return "TAOS_FAIL", retResult + else: + if key == 'A' or key1 == 'A' or key == 'C' or key1 == 'C' or key == 'V' or key1 == 'V': + return "TAOS_OK", retResult + else: + return "TAOS_OK", retResult + else: + if key == 'A' or key1 == 'A' or key == 'C' or key1 == 'C' or key == 'V' or key1 == 'V': + return "TAOS_OK", retResult + else: + return "TAOS_FAIL", retResult + +class TDTestCase: + #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\ + # 'serverPort': 7080, 'firstEp': 'trd02:7080'} + hostname = socket.gethostname() + serverPort = '7080' + rpcDebugFlagVal = '143' + clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + clientCfgDict["serverPort"] = serverPort + clientCfgDict["firstEp"] = hostname + ':' + serverPort + clientCfgDict["secondEp"] = hostname + ':' + serverPort + clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + clientCfgDict["fqdn"] = hostname + + updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + updatecfgDict["clientCfg"] = clientCfgDict + updatecfgDict["serverPort"] = serverPort + updatecfgDict["firstEp"] = hostname + ':' + serverPort + updatecfgDict["secondEp"] = hostname + ':' + serverPort + updatecfgDict["fqdn"] = hostname + + print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + # time.sleep(2) + tdSql.query("create user testpy pass 'testpy'") + + #hostname = socket.gethostname() + #tdLog.info ("hostname: %s" % hostname) + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting'] + netrole = ['client', 'server'] + + keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ + 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''} + + keyDict['h'] = self.hostname + keyDict['c'] = cfgPath + keyDict['P'] = self.serverPort + + tdLog.printNoPrefix("================================ parameter: -h wiht error value") + #newDbName="dbh" + #sqlString = 'create database ' + newDbName + ';' + keyDict['h'] = 'abc' + retCode, retVal = taos_command(buildPath, "h", keyDict['h'], "taos>", keyDict['c'], '') + if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): + tdLog.info("taos -h %s test suceess"%keyDict['h']) + else: + tdLog.exit("taos -h %s fail"%keyDict['h']) + + keyDict['h'] = '\'abc\'' + retCode, retVal = taos_command(buildPath, "h", keyDict['h'], "taos>", keyDict['c'], '') + if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): + tdLog.info("taos -h %s test suceess"%keyDict['h']) + else: + tdLog.exit("taos -h %s fail"%keyDict['h']) + + keyDict['h'] = '3' + retCode, retVal = taos_command(buildPath, "h", keyDict['h'], "taos>", keyDict['c'], '') + if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): + tdLog.info("taos -h %s test suceess"%keyDict['h']) + else: + tdLog.exit("taos -h %s fail"%keyDict['h']) + + keyDict['h'] = '\'3\'' + retCode, retVal = taos_command(buildPath, "h", keyDict['h'], "taos>", keyDict['c'], '') + if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): + tdLog.info("taos -h %s test suceess"%keyDict['h']) + else: + tdLog.exit("taos -h %s fail"%keyDict['h']) + + tdLog.printNoPrefix("================================ parameter: -P wiht error value") + #newDbName="dbh" + #sqlString = 'create database ' + newDbName + ';' + keyDict['P'] = 'abc' + retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') + if (retCode == "TAOS_FAIL") and ("Invalid port" in retVal): + tdLog.info("taos -P %s test suceess"%keyDict['P']) + else: + tdLog.exit("taos -P %s fail"%keyDict['P']) + + keyDict['P'] = '\'abc\'' + retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') + if (retCode == "TAOS_FAIL") and ("Invalid port" in retVal): + tdLog.info("taos -P %s test suceess"%keyDict['P']) + else: + tdLog.exit("taos -P %s fail"%keyDict['P']) + + keyDict['P'] = '3' + retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') + if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): + tdLog.info("taos -P %s test suceess"%keyDict['P']) + else: + tdLog.exit("taos -P %s fail"%keyDict['P']) + + keyDict['P'] = '\'3\'' + retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') + if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): + tdLog.info("taos -P %s test suceess"%keyDict['P']) + else: + tdLog.exit("taos -P %s fail"%keyDict['P']) + + keyDict['P'] = '12ab' + retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') + if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): + tdLog.info("taos -P %s test suceess"%keyDict['P']) + else: + tdLog.exit("taos -P %s fail"%keyDict['P']) + + keyDict['P'] = '\'12ab\'' + retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') + if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): + tdLog.info("taos -P %s test suceess"%keyDict['P']) + else: + tdLog.exit("taos -P %s fail"%keyDict['P']) + + tdLog.printNoPrefix("================================ parameter: -f with error sql ") + pwd=os.getcwd() + newDbName="dbf" + sqlFile = pwd + "/0-others/sql.txt" + sql1 = "echo 'create database " + newDbName + "' > " + sqlFile + sql2 = "echo 'use " + newDbName + "' >> " + sqlFile + sql3 = "echo 'create table ntbf (ts timestamp, c binary(40)) no this item' >> " + sqlFile + sql4 = "echo 'insert into ntbf values (\"2021-04-01 08:00:00.000\", \"test taos -f1\")(\"2021-04-01 08:00:01.000\", \"test taos -f2\")' >> " + sqlFile + sql5 = "echo 'show databases' >> " + sqlFile + os.system(sql1) + os.system(sql2) + os.system(sql3) + os.system(sql4) + os.system(sql5) + + keyDict['f'] = pwd + "/0-others/sql.txt" + retCode, retVal = taos_command(buildPath, "f", keyDict['f'], 'performance_schema', keyDict['c'], '', '', '') + #print("============ ret code: ", retCode) + if retCode != "TAOS_OK": + tdLog.exit("taos -f fail") + + print ("========== check new db ==========") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + #print ("dbseq: %d, dbname: %s"%(i, tdSql.getData(i, 0))) + if tdSql.getData(i, 0) == newDbName: + break + else: + tdLog.exit("create db fail after taos -f fail") + + sqlString = "select * from " + newDbName + ".ntbf" + tdSql.error(sqlString) + + shellCmd = "rm -f " + sqlFile + os.system(shellCmd) + + keyDict['f'] = pwd + "/0-others/noexistfile.txt" + retCode, retVal = taos_command(buildPath, "f", keyDict['f'], 'failed to open file', keyDict['c'], '', '', '') + #print("============ ret code: ", retCode) + if retCode != "TAOS_OK": + tdLog.exit("taos -f fail") + + tdSql.query('drop database %s'%newDbName) + + tdLog.printNoPrefix("================================ parameter: -a with error value") + #newDbName="dba" + errorPassword = 'errorPassword' + sqlString = 'create database ' + newDbName + ';' + retCode, retVal = taos_command(buildPath, "u", keyDict['u'], "taos>", keyDict['c'], sqlString, 'a', errorPassword) + if retCode != "TAOS_FAIL": + tdLog.exit("taos -u %s -a %s"%(keyDict['u'], errorPassword)) + + tdLog.printNoPrefix("================================ parameter: -p with error value") + #newDbName="dba" + keyDict['p'] = 'errorPassword' + retCode, retVal = taos_command(buildPath, "u", keyDict['u'], "taos>", keyDict['c'], sqlString, 'p', keyDict['p']) + if retCode == "TAOS_FAIL" and "Authentication failure" in retVal: + tdLog.info("taos -p %s test suceess"%keyDict['p']) + else: + tdLog.exit("taos -u %s -p %s"%(keyDict['u'], keyDict['p'])) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/taosShellNetChk.py b/tests/system-test/0-others/taosShellNetChk.py new file mode 100644 index 0000000000000000000000000000000000000000..524268101afcca88e06bdbed4da0bdd55e6633b5 --- /dev/null +++ b/tests/system-test/0-others/taosShellNetChk.py @@ -0,0 +1,202 @@ + +import taos +import sys +import time +import socket +import pexpect +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +def taos_command (buildPath, key, value, expectString, cfgDir, sqlString='', key1='', value1=''): + if len(key) == 0: + tdLog.exit("taos test key is null!") + + taosCmd = buildPath + '/build/bin/taos ' + if len(cfgDir) != 0: + taosCmd = taosCmd + '-c ' + cfgDir + + taosCmd = taosCmd + ' -' + key + if len(value) != 0: + if key == 'p': + taosCmd = taosCmd + value + else: + taosCmd = taosCmd + ' ' + value + + if len(key1) != 0: + taosCmd = taosCmd + ' -' + key1 + if key1 == 'p': + taosCmd = taosCmd + value1 + else: + if len(value1) != 0: + taosCmd = taosCmd + ' ' + value1 + + tdLog.info ("taos cmd: %s" % taosCmd) + + child = pexpect.spawn(taosCmd, timeout=3) + #output = child.readline() + #print (output.decode()) + if len(expectString) != 0: + i = child.expect([expectString, pexpect.TIMEOUT, pexpect.EOF], timeout=6) + else: + i = child.expect([pexpect.TIMEOUT, pexpect.EOF], timeout=6) + + retResult = child.before.decode() + print("expect() return code: %d, content:\n %s\n"%(i, retResult)) + #print(child.after.decode()) + if i == 0: + print ('taos login success! Here can run sql, taos> ') + if len(sqlString) != 0: + child.sendline (sqlString) + w = child.expect(["Query OK", pexpect.TIMEOUT, pexpect.EOF], timeout=1) + retResult = child.before.decode() + if w == 0: + return "TAOS_OK", retResult + else: + return "TAOS_FAIL", retResult + else: + if key == 'A' or key1 == 'A' or key == 'C' or key1 == 'C' or key == 'V' or key1 == 'V': + return "TAOS_OK", retResult + else: + return "TAOS_OK", retResult + else: + if key == 'A' or key1 == 'A' or key == 'C' or key1 == 'C' or key == 'V' or key1 == 'V': + return "TAOS_OK", retResult + else: + return "TAOS_FAIL", retResult + +class TDTestCase: + #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\ + # 'serverPort': 7080, 'firstEp': 'trd02:7080'} + hostname = socket.gethostname() + serverPort = '7080' + rpcDebugFlagVal = '143' + clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + clientCfgDict["serverPort"] = serverPort + clientCfgDict["firstEp"] = hostname + ':' + serverPort + clientCfgDict["secondEp"] = hostname + ':' + serverPort + clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + clientCfgDict["fqdn"] = hostname + + updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + updatecfgDict["clientCfg"] = clientCfgDict + updatecfgDict["serverPort"] = serverPort + updatecfgDict["firstEp"] = hostname + ':' + serverPort + updatecfgDict["secondEp"] = hostname + ':' + serverPort + updatecfgDict["fqdn"] = hostname + + print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + tdSql.query("create user testpy pass 'testpy'") + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting'] + netrole = ['client', 'server'] + + keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ + 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''} + + keyDict['h'] = self.hostname + keyDict['c'] = cfgPath + keyDict['P'] = self.serverPort + + tdLog.printNoPrefix("================================ parameter: -k") + sqlString = '' + retCode, retVal = taos_command(buildPath, "k", '', "", keyDict['c'], sqlString) + if "2: service ok" in retVal: + tdLog.info("taos -k success") + else: + tdLog.exit("taos -k fail") + + # stop taosd + tdDnodes.stop(1) + #sleep(10) + #tdDnodes.start(1) + #sleep(5) + retCode, retVal = taos_command(buildPath, "k", '', "", keyDict['c'], sqlString) + if "0: unavailable" in retVal: + tdLog.info("taos -k success") + else: + tdLog.exit("taos -k fail") + + # restart taosd + tdDnodes.start(1) + #sleep(5) + retCode, retVal = taos_command(buildPath, "k", '', "", keyDict['c'], sqlString) + if "2: service ok" in retVal: + tdLog.info("taos -k success") + else: + tdLog.exit("taos -k fail") + + tdLog.printNoPrefix("================================ parameter: -n") + # stop taosd + tdDnodes.stop(1) + + role = 'server' + taosCmd = 'nohup ' + buildPath + '/build/bin/taos -c ' + keyDict['c'] + taosCmd = taosCmd + ' -n ' + role + ' > /dev/null 2>&1 &' + print (taosCmd) + os.system(taosCmd) + + pktLen = '2000' + pktNum = '10' + role = 'client' + taosCmd = buildPath + '/build/bin/taos -c ' + keyDict['c'] + taosCmd = taosCmd + ' -n ' + role + ' -l ' + pktLen + ' -N ' + pktNum + print (taosCmd) + child = pexpect.spawn(taosCmd, timeout=3) + i = child.expect([pexpect.TIMEOUT, pexpect.EOF], timeout=6) + + retResult = child.before.decode() + print("expect() return code: %d, content:\n %s\n"%(i, retResult)) + #print(child.after.decode()) + if i == 0: + tdLog.exit('taos -n server fail!') + + expectString1 = 'response is received, size:' + pktLen + expectSTring2 = pktNum + '/' + pktNum + if expectString1 in retResult and expectSTring2 in retResult: + tdLog.info("taos -n client success") + else: + tdLog.exit('taos -n client fail!') + + os.system('pkill taos') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py new file mode 100644 index 0000000000000000000000000000000000000000..a7d17bc41ebde9ed6da44c94ddd3265ec0777fa0 --- /dev/null +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -0,0 +1,291 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import threading +import multiprocessing as mp +from numpy.lib.function_base import insert +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np +import datetime as dt +import time +# constant define +WAITS = 5 # wait seconds + +class TDTestCase: + # + # --------------- main frame ------------------- + # + + def caseDescription(self): + ''' + limit and offset keyword function test cases; + case1: limit offset base function test + case2: offset return valid + ''' + return + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + # init + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + # tdSql.prepare() + # self.create_tables(); + self.ts = 1500000000000 + + + # run case + def run(self): + + # test base case + self.test_case1() + tdLog.debug(" LIMIT test_case1 ............ [OK]") + + # test advance case + # self.test_case2() + # tdLog.debug(" LIMIT test_case2 ............ [OK]") + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + # --------------- case ------------------- + + # create tables + def create_tables(self,dbname,stbname,count): + tdSql.execute("use %s" %dbname) + tdSql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname) + pre_create = "create table" + sql = pre_create + tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + # print(time.time()) + exeStartTime=time.time() + for i in range(count): + sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1) + if i >0 and i%3000 == 0: + tdSql.execute(sql) + sql = pre_create + # print(time.time()) + # end sql + if sql != pre_create: + tdSql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedCreate=count/spendTime + tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + return + + def newcur(self,host,cfg): + user = "root" + password = "taosdata" + port =6030 + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def new_create_tables(self,dbname,vgroups,stbname,tcountStart,tcountStop): + host = "chenhaoran02" + buildPath = self.getBuildPath() + config = buildPath+ "../sim/dnode1/cfg/" + + tsql=self.newcur(host,config) + tsql.execute("create database %s vgroups %d"%(dbname,vgroups)) + tsql.execute("use %s" %dbname) + tsql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname) + + pre_create = "create table" + sql = pre_create + tcountStop=int(tcountStop) + tcountStart=int(tcountStart) + count=tcountStop-tcountStart + + tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + # print(time.time()) + exeStartTime=time.time() + # print(type(tcountStop),type(tcountStart)) + for i in range(tcountStart,tcountStop): + sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1) + if i >0 and i%20000 == 0: + # print(sql) + tsql.execute(sql) + sql = pre_create + # print(time.time()) + # end sql + if sql != pre_create: + # print(sql) + tsql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedCreate=count/spendTime + # tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + return + + + + # insert data + def insert_data(self, dbname, stbname, ts_start, tcountStart,tcountStop,rowCount): + tdSql.execute("use %s" %dbname) + pre_insert = "insert into " + sql = pre_insert + tcount=tcountStop-tcountStart + allRows=tcount*rowCount + tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbname, allRows)) + exeStartTime=time.time() + for i in range(tcountStart,tcountStop): + sql += " %s_%d values "%(stbname,i) + for j in range(rowCount): + sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j) + if j >0 and j%5000 == 0: + # print(sql) + tdSql.execute(sql) + sql = "insert into %s_%d values " %(stbname,i) + # end sql + if sql != pre_insert: + # print(sql) + tdSql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedInsert=allRows/spendTime + # tdLog.debug("spent %.2fs to INSERT %d rows , insert rate is %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert)) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + + # test case1 base + def test_case1(self): + tdLog.debug("-----create database and tables test------- ") + tdSql.execute("drop database if exists db1") + tdSql.execute("drop database if exists db4") + tdSql.execute("drop database if exists db6") + tdSql.execute("drop database if exists db8") + tdSql.execute("drop database if exists db12") + tdSql.execute("drop database if exists db16") + + #create database and tables; + + # tdSql.execute("create database db11 vgroups 1") + # # self.create_tables("db1", "stb1", 30*10000) + # tdSql.execute("use db1") + # tdSql.execute("create stable stb1(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)") + + # tdSql.execute("create database db12 vgroups 1") + # # self.create_tables("db1", "stb1", 30*10000) + # tdSql.execute("use db1") + + # t1 = threading.Thread(target=self.new_create_tables("db1", "stb1", 15*10000), args=(1,)) + # t2 = threading.Thread(target=self.new_create_tables("db1", "stb1", 15*10000), args=(2,)) + # t1 = mp.Process(target=self.new_create_tables, args=("db1", "stb1", 0,count/2,)) + # t2 = mp.Process(target=self.new_create_tables, args=("db1", "stb1", count/2,count,)) + + count=50000 + vgroups=1 + threads = [] + threadNumbers=2 + for i in range(threadNumbers): + threads.append(mp.Process(target=self.new_create_tables, args=("db1%d"%i, vgroups, "stb1", 0,count,))) + start_time = time.time() + for tr in threads: + tr.start() + for tr in threads: + tr.join() + end_time = time.time() + spendTime=end_time-start_time + speedCreate=count/spendTime + tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + # self.new_create_tables("db1", "stb1", 15*10000) + # self.new_create_tables("db1", "stb1", 15*10000) + + # tdSql.execute("create database db4 vgroups 4") + # self.create_tables("db4", "stb4", 30*10000) + + # tdSql.execute("create database db6 vgroups 6") + # self.create_tables("db6", "stb6", 30*10000) + + # tdSql.execute("create database db8 vgroups 8") + # self.create_tables("db8", "stb8", 30*10000) + + # tdSql.execute("create database db12 vgroups 12") + # self.create_tables("db12", "stb12", 30*10000) + + # tdSql.execute("create database db16 vgroups 16") + # self.create_tables("db16", "stb16", 30*10000) + return + + # test case2 base:insert data + def test_case2(self): + + tdLog.debug("-----insert data test------- ") + # drop database + tdSql.execute("drop database if exists db1") + tdSql.execute("drop database if exists db4") + tdSql.execute("drop database if exists db6") + tdSql.execute("drop database if exists db8") + tdSql.execute("drop database if exists db12") + tdSql.execute("drop database if exists db16") + + #create database and tables; + + tdSql.execute("create database db1 vgroups 1") + self.create_tables("db1", "stb1", 1*100) + self.insert_data("db1", "stb1", self.ts, 1*50,1*10000) + + + tdSql.execute("create database db4 vgroups 4") + self.create_tables("db4", "stb4", 1*100) + self.insert_data("db4", "stb4", self.ts, 1*100,1*10000) + + tdSql.execute("create database db6 vgroups 6") + self.create_tables("db6", "stb6", 1*100) + self.insert_data("db6", "stb6", self.ts, 1*100,1*10000) + + tdSql.execute("create database db8 vgroups 8") + self.create_tables("db8", "stb8", 1*100) + self.insert_data("db8", "stb8", self.ts, 1*100,1*10000) + + tdSql.execute("create database db12 vgroups 12") + self.create_tables("db12", "stb12", 1*100) + self.insert_data("db12", "stb12", self.ts, 1*100,1*10000) + + tdSql.execute("create database db16 vgroups 16") + self.create_tables("db16", "stb16", 1*100) + self.insert_data("db16", "stb16", self.ts, 1*100,1*10000) + + return + +# +# add case with filename +# +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py new file mode 100644 index 0000000000000000000000000000000000000000..ccf83df952fadb274088fa2e9d296a6d955e455f --- /dev/null +++ b/tests/system-test/2-query/abs.py @@ -0,0 +1,483 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto(self ,origin_query , abs_query): + abs_result = tdSql.getResult(abs_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem >=0: + elem = elem + else: + elem = -elem + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(abs_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("abs function value has not as expected , sql is \"%s\" "%abs_query ) + sys.exit(1) + else: + tdLog.info("abs value check pass , it work as expected ,sql is \"%s\" "%abs_query ) + + def test_errors(self): + error_sql_lists = [ + "select abs from t1", + # "select abs(-+--+c1) from t1", + # "select +-abs(c1) from t1", + # "select ++-abs(c1) from t1", + # "select ++--abs(c1) from t1", + # "select - -abs(c1)*0 from t1", + # "select abs(tbname+1) from t1 ", + "select abs(123--123)==1 from t1", + "select abs(c1) as 'd1' from t1", + "select abs(c1 ,c2 ) from t1", + "select abs(c1 ,NULL) from t1", + "select abs(,) from t1;", + "select abs(abs(c1) ab from t1)", + "select abs(c1) as int from t1", + "select abs from stb1", + # "select abs(-+--+c1) from stb1", + # "select +-abs(c1) from stb1", + # "select ++-abs(c1) from stb1", + # "select ++--abs(c1) from stb1", + # "select - -abs(c1)*0 from stb1", + # "select abs(tbname+1) from stb1 ", + "select abs(123--123)==1 from stb1", + "select abs(c1) as 'd1' from stb1", + "select abs(c1 ,c2 ) from stb1", + "select abs(c1 ,NULL) from stb1", + "select abs(,) from stb1;", + "select abs(abs(c1) ab from stb1)", + "select abs(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select abs(ts) from t1" , + "select abs(c7) from t1", + "select abs(c8) from t1", + "select abs(c9) from t1", + "select abs(ts) from ct1" , + "select abs(c7) from ct1", + "select abs(c8) from ct1", + "select abs(c9) from ct1", + "select abs(ts) from ct3" , + "select abs(c7) from ct3", + "select abs(c8) from ct3", + "select abs(c9) from ct3", + "select abs(ts) from ct4" , + "select abs(c7) from ct4", + "select abs(c8) from ct4", + "select abs(c9) from ct4", + "select abs(ts) from stb1" , + "select abs(c7) from stb1", + "select abs(c8) from stb1", + "select abs(c9) from stb1" , + + "select abs(ts) from stbbb1" , + "select abs(c7) from stbbb1", + + "select abs(ts) from tbname", + "select abs(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select abs(c1) from t1", + "select abs(c2) from t1", + "select abs(c3) from t1", + "select abs(c4) from t1", + "select abs(c5) from t1", + "select abs(c6) from t1", + + "select abs(c1) from ct1", + "select abs(c2) from ct1", + "select abs(c3) from ct1", + "select abs(c4) from ct1", + "select abs(c5) from ct1", + "select abs(c6) from ct1", + + "select abs(c1) from ct3", + "select abs(c2) from ct3", + "select abs(c3) from ct3", + "select abs(c4) from ct3", + "select abs(c5) from ct3", + "select abs(c6) from ct3", + + "select abs(c1) from stb1", + "select abs(c2) from stb1", + "select abs(c3) from stb1", + "select abs(c4) from stb1", + "select abs(c5) from stb1", + "select abs(c6) from stb1", + + "select abs(c6) as alisb from stb1", + "select abs(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_abs_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select abs(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select abs(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select abs(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select abs(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select abs(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select abs(c6) from ct3") + + # used for regular table + tdSql.query("select abs(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1) + tdSql.checkData(3 , 0, 3) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto( "select c1, c2, c3 , c4, c5 from t1", "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from t1") + + # used for sub table + tdSql.query("select abs(c1) from ct1") + tdSql.checkData(0, 0, 8) + tdSql.checkData(1 , 0, 7) + tdSql.checkData(3 , 0, 5) + tdSql.checkData(5 , 0, 4) + + tdSql.query("select abs(c1) from ct1") + self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct1", "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from ct1") + self.check_result_auto("select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from ct1;","select c1 from ct1" ) + + # used for stable table + + tdSql.query("select abs(c1) from stb1") + tdSql.checkRows(25) + self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct4 ", "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from ct4") + self.check_result_auto("select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from ct4;" , "select c1 from ct4" ) + + + # used for not exists table + tdSql.error("select abs(c1) from stbbb1") + tdSql.error("select abs(c1) from tbname") + tdSql.error("select abs(c1) from ct5") + + # mix with common col + tdSql.query("select c1, abs(c1) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,8) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0) + tdSql.query("select c1, abs(c1) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,5) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + tdSql.query("select c1, abs(c1) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,5) + + # mix with common functions + tdSql.query("select c1, abs(c1),c5, floor(c5) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 , 6) + tdSql.checkData(3 , 2 ,6.66000) + tdSql.checkData(3 , 3 ,6.00000) + + tdSql.query("select c1, abs(c1),c5, floor(c5) from stb1 ") + + # mix with agg functions , not support + tdSql.error("select c1, abs(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, abs(c1),c5, count(c5) from ct1 ") + tdSql.error("select abs(c1), count(c5) from stb1 ") + tdSql.error("select abs(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # bug fix for compute + tdSql.query("select c1, abs(c1) -0 ,ceil(c1)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 8.000000000) + tdSql.checkData(1, 2, 8.000000000) + + tdSql.query(" select c1, abs(c1) -0 ,ceil(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 8.000000000) + tdSql.checkData(1, 2, 7.900000000) + + + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,3.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,88888) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,8.000000000) + tdSql.checkData(0,4,7.900000000) + tdSql.checkData(0,5,3.000000000) + + + def abs_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select abs(c1), abs(c2) ,abs(c3), abs(c4), abs(c5) ,abs(c6) from sub1_bound") + self.check_result_auto( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select abs(c1), abs(c2) ,abs(c3), abs(c3), abs(c2) ,abs(c1) from sub1_bound") + self.check_result_auto("select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from sub1_bound;" , "select abs(c1) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select abs(c1) ,abs(c2) , abs(c3) , abs(c4), abs(c5), abs(c6) from sub1_bound ") + tdSql.checkData(0,0,2147483647) + tdSql.checkData(0,1,9223372036854775807) + tdSql.checkData(0,2,32767) + tdSql.checkData(0,3,127) + tdSql.checkData(0,4,339999995214436424907732413799364296704.00000) + tdSql.checkData(0,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) + tdSql.checkData(1,0,2147483647) + tdSql.checkData(1,1,9223372036854775807) + tdSql.checkData(1,2,32767) + tdSql.checkData(1,3,127) + tdSql.checkData(1,4,339999995214436424907732413799364296704.00000) + tdSql.checkData(1,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) + tdSql.checkData(3,0,2147483646) + tdSql.checkData(3,1,9223372036854775806) + tdSql.checkData(3,2,32766) + tdSql.checkData(3,3,126) + tdSql.checkData(3,4,339999995214436424907732413799364296704.00000) + tdSql.checkData(3,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) + + # check + - * / in functions + tdSql.query("select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ") + tdSql.checkData(0,0,2147483648.000000000) + tdSql.checkData(0,1,9223372036854775807) + tdSql.checkData(0,2,32767.000000000) + tdSql.checkData(0,3,63.500000000) + tdSql.checkData(0,4,169999997607218212453866206899682148352.000000000) + tdSql.checkData(0,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) + + + tdSql.checkData(1,0,2147483646.000000000) + tdSql.checkData(1,1,9223372036854775808.000000000) + tdSql.checkData(1,2,32767.000000000) + tdSql.checkData(1,3,63.500000000) + tdSql.checkData(1,4,169999997607218212453866206899682148352.000000000) + + self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ") + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: abs basic query ============") + + self.basic_abs_function() + + tdLog.printNoPrefix("==========step5: abs boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step6: abs filter query ============") + + self.abs_func_filter() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py index fb0f22eaf8ba20b5229ec6f885c7af1202a85b0b..0e849410b7b6ca29a739c4309d6123d5c9508ede 100644 --- a/tests/system-test/2-query/cast.py +++ b/tests/system-test/2-query/cast.py @@ -35,9 +35,7 @@ class TDTestCase: for i in range(len(tdSql.queryRows)): if data_tb_col[i] is None: tdSql.checkData( i, 0 , None ) - if (col_name == "c2" or col_name == "double" ) and tbname == "t1" and i == 10: - continue - else: + if col_name not in ["c2", "double"] or tbname != "t1" or i != 10: utc_zone = datetime.timezone.utc utc_8 = datetime.timezone(datetime.timedelta(hours=8)) date_init_stamp = datetime.datetime.utcfromtimestamp(data_tb_col[i]/1000) @@ -597,7 +595,41 @@ class TDTestCase: time2str = str(int(datetime.datetime.timestamp(data_t1_c10[i])*1000)) tdSql.checkData( i, 0, time2str ) - + tdLog.printNoPrefix("==========step39: cast constant operation to bigint, expect change to int ") + tdSql.query("select cast(12121.23323131 as bigint) as b from ct4") + ( tdSql.checkData(i, 0, 12121) for i in range(len(tdSql.queryRows) ) ) + tdSql.query("select cast(12121.23323131 as binary(16)) as b from ct4") + ( tdSql.checkData(i, 0, '12121.233231') for i in range(len(tdSql.queryRows)) ) + tdSql.query("select cast(12121.23323131 as binary(2)) as b from ct4") + ( tdSql.checkData(i, 0, '12') for i in range(len(tdSql.queryRows) ) ) + tdSql.query("select cast(12121.23323131 as nchar(16)) as b from ct4") + ( tdSql.checkData(i, 0, '12121.233231') for i in range(len(tdSql.queryRows) ) ) + tdSql.query("select cast(12121.23323131 as nchar(2)) as b from ct4") + ( tdSql.checkData(i, 0, '12') for i in range(len(tdSql.queryRows) ) ) + + tdSql.query("select cast(12121.23323131 + 321.876897998 as bigint) as b from ct4") + ( tdSql.checkData(i, 0, 12443) for i in range(len(tdSql.queryRows) ) ) + tdSql.query("select cast(12121.23323131 + 321.876897998 as binary(16)) as b from ct4") + ( tdSql.checkData(i, 0, '12443.110129') for i in range(len(tdSql.queryRows)) ) + tdSql.query("select cast(12121.23323131 + 321.876897998 as binary(3)) as b from ct4") + ( tdSql.checkData(i, 0, '124') for i in range(len(tdSql.queryRows) ) ) + tdSql.query("select cast(12121.23323131 + 321.876897998 as nchar(16)) as b from ct4") + ( tdSql.checkData(i, 0, '12443.110129') for i in range(len(tdSql.queryRows)) ) + tdSql.query("select cast(12121.23323131 + 321.876897998 as nchar(3)) as b from ct4") + ( tdSql.checkData(i, 0, '124') for i in range(len(tdSql.queryRows) ) ) + + tdSql.query("select cast(12121.23323131 + 'test~!@`#$%^&*()}{][;><.,' as bigint) as b from ct4") + ( tdSql.checkData(i, 0, 12121) for i in range(len(tdSql.queryRows) ) ) + tdSql.query("select cast(12121.23323131 + 'test~!@`#$%^&*()}{][;><.,' as binary(16)) as b from ct4") + ( tdSql.checkData(i, 0, '12121.233231') for i in range(len(tdSql.queryRows)) ) + tdSql.query("select cast(12121.23323131 + 'test~!@`#$%^&*()}{][;><.,' as binary(2)) as b from ct4") + ( tdSql.checkData(i, 0, '12') for i in range(len(tdSql.queryRows) ) ) + tdSql.query("select cast(12121.23323131 + 'test~!@`#$%^&*()}{][;><.,' as nchar(16)) as b from ct4") + ( tdSql.checkData(i, 0, '12121.233231') for i in range(len(tdSql.queryRows) ) ) + tdSql.query("select cast(12121.23323131 + 'test~!@`#$%^&*()}{][;><.,' as nchar(2)) as b from ct4") + ( tdSql.checkData(i, 0, '12') for i in range(len(tdSql.queryRows) ) ) + + tdLog.printNoPrefix("==========step40: error cast condition, should return error ") tdSql.error("select cast(c1 as int) as b from ct4") tdSql.error("select cast(c1 as bool) as b from ct4") tdSql.error("select cast(c1 as tinyint) as b from ct4") diff --git a/tests/system-test/2-query/ceil.py b/tests/system-test/2-query/ceil.py new file mode 100644 index 0000000000000000000000000000000000000000..4196a5a8ce858bbc2d2f680e143cfb2488953dca --- /dev/null +++ b/tests/system-test/2-query/ceil.py @@ -0,0 +1,463 @@ +from math import ceil +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto(self ,origin_query , ceil_query): + pass + ceil_result = tdSql.getResult(ceil_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = ceil(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(ceil_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("ceil function value has not as expected , sql is \"%s\" "%ceil_query ) + sys.exit(1) + else: + tdLog.info("ceil value check pass , it work as expected ,sql is \"%s\" "%ceil_query ) + + def test_errors(self): + error_sql_lists = [ + "select ceil from t1", + # "select ceil(-+--+c1) from t1", + # "select +-ceil(c1) from t1", + # "select ++-ceil(c1) from t1", + # "select ++--ceil(c1) from t1", + # "select - -ceil(c1)*0 from t1", + # "select ceil(tbname+1) from t1 ", + "select ceil(123--123)==1 from t1", + "select ceil(c1) as 'd1' from t1", + "select ceil(c1 ,c2 ) from t1", + "select ceil(c1 ,NULL) from t1", + "select ceil(,) from t1;", + "select ceil(ceil(c1) ab from t1)", + "select ceil(c1) as int from t1", + "select ceil from stb1", + # "select ceil(-+--+c1) from stb1", + # "select +-ceil(c1) from stb1", + # "select ++-ceil(c1) from stb1", + # "select ++--ceil(c1) from stb1", + # "select - -ceil(c1)*0 from stb1", + # "select ceil(tbname+1) from stb1 ", + "select ceil(123--123)==1 from stb1", + "select ceil(c1) as 'd1' from stb1", + "select ceil(c1 ,c2 ) from stb1", + "select ceil(c1 ,NULL) from stb1", + "select ceil(,) from stb1;", + "select ceil(ceil(c1) ab from stb1)", + "select ceil(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select ceil(ts) from t1" , + "select ceil(c7) from t1", + "select ceil(c8) from t1", + "select ceil(c9) from t1", + "select ceil(ts) from ct1" , + "select ceil(c7) from ct1", + "select ceil(c8) from ct1", + "select ceil(c9) from ct1", + "select ceil(ts) from ct3" , + "select ceil(c7) from ct3", + "select ceil(c8) from ct3", + "select ceil(c9) from ct3", + "select ceil(ts) from ct4" , + "select ceil(c7) from ct4", + "select ceil(c8) from ct4", + "select ceil(c9) from ct4", + "select ceil(ts) from stb1" , + "select ceil(c7) from stb1", + "select ceil(c8) from stb1", + "select ceil(c9) from stb1" , + + "select ceil(ts) from stbbb1" , + "select ceil(c7) from stbbb1", + + "select ceil(ts) from tbname", + "select ceil(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select ceil(c1) from t1", + "select ceil(c2) from t1", + "select ceil(c3) from t1", + "select ceil(c4) from t1", + "select ceil(c5) from t1", + "select ceil(c6) from t1", + + "select ceil(c1) from ct1", + "select ceil(c2) from ct1", + "select ceil(c3) from ct1", + "select ceil(c4) from ct1", + "select ceil(c5) from ct1", + "select ceil(c6) from ct1", + + "select ceil(c1) from ct3", + "select ceil(c2) from ct3", + "select ceil(c3) from ct3", + "select ceil(c4) from ct3", + "select ceil(c5) from ct3", + "select ceil(c6) from ct3", + + "select ceil(c1) from stb1", + "select ceil(c2) from stb1", + "select ceil(c3) from stb1", + "select ceil(c4) from stb1", + "select ceil(c5) from stb1", + "select ceil(c6) from stb1", + + "select ceil(c6) as alisb from stb1", + "select ceil(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_ceil_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select ceil(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select ceil(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select ceil(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select ceil(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select ceil(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select ceil(c6) from ct3") + + # used for regular table + tdSql.query("select ceil(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1) + tdSql.checkData(3 , 0, 3) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto( "select c1, c2, c3 , c4, c5 from t1", "select (c1), ceil(c2) ,ceil(c3), ceil(c4), ceil(c5) from t1") + + # used for sub table + tdSql.query("select ceil(c1) from ct1") + tdSql.checkData(0, 0, 8) + tdSql.checkData(1 , 0, 7) + tdSql.checkData(3 , 0, 5) + tdSql.checkData(5 , 0, 4) + + tdSql.query("select ceil(c1) from ct1") + self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct1", "select (c1), ceil(c2) ,ceil(c3), ceil(c4), ceil(c5) from ct1") + self.check_result_auto("select ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(c1)))))))))) nest_col_func from ct1;","select c1 from ct1" ) + + # used for stable table + + tdSql.query("select ceil(c1) from stb1") + tdSql.checkRows(25) + self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct4 ", "select (c1), ceil(c2) ,ceil(c3), ceil(c4), ceil(c5) from ct4") + self.check_result_auto("select ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(c1)))))))))) nest_col_func from ct4;" , "select c1 from ct4" ) + + + # used for not exists table + tdSql.error("select ceil(c1) from stbbb1") + tdSql.error("select ceil(c1) from tbname") + tdSql.error("select ceil(c1) from ct5") + + # mix with common col + tdSql.query("select c1, ceil(c1) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,8) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0) + tdSql.query("select c1, ceil(c1) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,5) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + tdSql.query("select c1, ceil(c1) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,5) + + # mix with common functions + tdSql.query("select c1, ceil(c1),c5, ceil(c5) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 , 6) + tdSql.checkData(3 , 2 ,6.66000) + tdSql.checkData(3 , 3 ,7.00000) + + tdSql.query("select c1, ceil(c1),c5, floor(c5) from stb1 ") + + # mix with agg functions , not support + tdSql.error("select c1, ceil(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, ceil(c1),c5, count(c5) from ct1 ") + tdSql.error("select ceil(c1), count(c5) from stb1 ") + tdSql.error("select ceil(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # bug fix for compute + tdSql.query("select c1, abs(c1) -0 ,ceil(c1)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 8.000000000) + tdSql.checkData(1, 2, 8.000000000) + + tdSql.query(" select c1, abs(c1) -0 ,ceil(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 8.000000000) + tdSql.checkData(1, 2, 7.900000000) + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,3.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,88888) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,8.000000000) + tdSql.checkData(0,4,7.900000000) + tdSql.checkData(0,5,3.000000000) + + def ceil_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select ceil(c1), ceil(c2) ,ceil(c3), ceil(c4), ceil(c5) ,ceil(c6) from sub1_bound") + self.check_result_auto( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select ceil(c1), ceil(c2) ,ceil(c3), ceil(c3), ceil(c2) ,ceil(c1) from sub1_bound") + self.check_result_auto("select ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(ceil(c1)))))))))) nest_col_func from sub1_bound;" , "select ceil(c1) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select ceil(c1+0.2) ,ceil(c2) , ceil(c3+0.3) , ceil(c4-0.3), ceil(c5/2), ceil(c6/2) from sub1_bound ") + tdSql.checkData(0, 0, 2147483648.000000000) + tdSql.checkData(0, 2, 32768.000000000) + tdSql.checkData(0, 3, 127.000000000) + tdSql.checkData(0, 4, 169999997607218212453866206899682148352.000000000) + + tdSql.checkData(4, 0, -2147483642.000000000) + tdSql.checkData(4, 2, -32762.000000000) + tdSql.checkData(4, 3, -123.000000000) + tdSql.checkData(4, 4, -169499995645668991474575059260979281920.000000000) + + self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select ceil(c1+1) ,ceil(c2) , ceil(c3*1) , ceil(c4/2), ceil(c5)/2, ceil(c6) from sub1_bound ") + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: ceil basic query ============") + + self.basic_ceil_function() + + tdLog.printNoPrefix("==========step5: ceil boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step6: ceil filter query ============") + + self.abs_func_filter() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/char_length.py b/tests/system-test/2-query/char_length.py new file mode 100644 index 0000000000000000000000000000000000000000..e78db3b8b010a08329545af0e927fc946bb9ae33 --- /dev/null +++ b/tests/system-test/2-query/char_length.py @@ -0,0 +1,248 @@ +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + + +INT_COL = "c1" +BINT_COL = "c2" +SINT_COL = "c3" +TINT_COL = "c4" +FLOAT_COL = "c5" +DOUBLE_COL = "c6" +BOOL_COL = "c7" + +BINARY_COL = "c8" +NCHAR_COL = "c9" +TS_COL = "c10" + +UN_CHAR_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_COL, ] +CHAR_COL = [ BINARY_COL, NCHAR_COL, ] +TS_TYPE_COL = [TS_COL] + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def __char_length_condition(self): + char_length_condition = [] + for char_col in CHAR_COL: + char_length_condition.extend( + ( + char_col, + f"upper( {char_col} )", + ) + ) + char_length_condition.extend( f"cast( {un_char_col} as binary(16) ) " for un_char_col in UN_CHAR_COL) + char_length_condition.extend( f"cast( {char_col} + {char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL ) + char_length_condition.extend( f"cast( {char_col} + {un_char_col} as binary(32) ) " for un_char_col in UN_CHAR_COL ) + + char_length_condition.append('''"test1234!@#$%^&*():'> 0 " + return "" + + def __group_condition(self, col, having = ""): + return f" group by {col} having {having}" if having else f" group by {col} " + + def __char_length_current_check(self, tbname): + char_length_condition = self.__char_length_condition() + for condition in char_length_condition: + where_condition = self.__where_condition(condition) + group_having = self.__group_condition(condition, having=f"{condition} is not null " ) + group_no_having= self.__group_condition(condition ) + groups = ["", group_having, group_no_having] + + for group_condition in groups: + tdSql.query(f"select {condition}, char_length( {condition} ) from {tbname} {where_condition} {group_condition} ") + for i in range(tdSql.queryRows): + if not tdSql.getData(i,1): + tdSql.checkData(i, 1, None) + # elif "as nchar" in condition or (NCHAR_COL in condition and "as binary" not in condition): + # tdSql.checkData(i, 1, len(str(tdSql.getData(i,0) ) ) * 4 ) + else: + tdSql.checkData(i, 1, len(str(tdSql.getData(i,0) ) ) ) + + def __char_length_err_check(self,tbname): + sqls = [] + + for un_char_col in UN_CHAR_COL: + sqls.extend( + ( + f"select char_length( {un_char_col} ) from {tbname} ", + f"select char_length(ceil( {un_char_col} )) from {tbname} ", + f"select {un_char_col} from {tbname} group by char_length( {un_char_col} ) ", + ) + ) + + sqls.extend( f"select char_length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in UN_CHAR_COL ) + sqls.extend( f"select char_length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + + sqls.extend( f"select {char_col} from {tbname} group by char_length( {char_col} ) " for char_col in CHAR_COL) + sqls.extend( f"select char_length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + sqls.extend( f"select char_length( {char_col} + {ts_col} ) from {tbname} " for char_col in UN_CHAR_COL for ts_col in TS_TYPE_COL) + sqls.extend( f"select char_length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) + sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) + sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) + sqls.extend( + ( + f"select char_length() from {tbname} ", + f"select char_length(*) from {tbname} ", + f"select char_length(ccccccc) from {tbname} ", + f"select char_length(111) from {tbname} ", + f"select char_length(c8, 11) from {tbname} ", + ) + ) + + return sqls + + def __test_current(self): + tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") + tbname = ["ct1", "ct2", "ct4", "t1", "stb1"] + for tb in tbname: + self.__char_length_current_check(tb) + tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========") + + def __test_error(self): + tdLog.printNoPrefix("==========err sql condition check , must return error==========") + tbname = ["ct1", "ct2", "ct4", "t1", "stb1"] + + for tb in tbname: + for errsql in self.__char_length_err_check(tb): + tdSql.error(sql=errsql) + tdLog.printNoPrefix(f"==========err sql condition check in {tb} over==========") + + + def all_test(self): + self.__test_current() + self.__test_error() + + + def __create_tb(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + create_stb_sql = f'''create table stb1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + ) tags (t1 int) + ''' + create_ntb_sql = f'''create table t1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + ) + ''' + tdSql.execute(create_stb_sql) + tdSql.execute(create_ntb_sql) + + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + def __insert_data(self, rows): + now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + for i in range(rows): + tdSql.execute( + f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f'''insert into ct1 values + ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', { now_time + 8 } ) + ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', { now_time + 9 } ) + ''' + ) + + tdSql.execute( + f'''insert into ct4 values + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( + { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000} + ) + ( + { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000} + ) + ''' + ) + + tdSql.execute( + f'''insert into ct2 values + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( + { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, + { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + ) + ( + { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, + { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + ) + ''' + ) + + for i in range(rows): + insert_data = f'''insert into t1 values + ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, + "binary_{i}", "nchar_{i}", { now_time - 1000 * i } ) + ''' + tdSql.execute(insert_data) + tdSql.execute( + f'''insert into t1 values + ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, + "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + ) + ( + { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, + "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + ) + ''' + ) + + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + self.__create_tb() + + tdLog.printNoPrefix("==========step2:insert data") + self.__insert_data(10) + + tdLog.printNoPrefix("==========step3:all check") + self.all_test() + + # tdDnodes.stop(1) + # tdDnodes.start(1) + + # tdSql.execute("use db") + + # tdLog.printNoPrefix("==========step4:after wal, all check again ") + # self.all_test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/distinct.py b/tests/system-test/2-query/distinct.py index a82f3a6f59f243ffc6eafe6f36b1bdcdb9b9e12f..937ff78c71eba9ec617ddd9a05b04f22e2229820 100644 --- a/tests/system-test/2-query/distinct.py +++ b/tests/system-test/2-query/distinct.py @@ -101,7 +101,7 @@ class TDTestCase: # tdSql.error("select distinct c1, ts from stb1 group by c2") tdSql.error("select distinct c1, ts from t1 group by c2") # tdSql.error("select distinct c1, max(c2) from stb1 ") - tdSql.error("select distinct c1, max(c2) from t1 ") + # tdSql.error("select distinct c1, max(c2) from t1 ") # tdSql.error("select max(c2), distinct c1 from stb1 ") tdSql.error("select max(c2), distinct c1 from t1 ") # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 group by t0") diff --git a/tests/system-test/2-query/floor.py b/tests/system-test/2-query/floor.py new file mode 100644 index 0000000000000000000000000000000000000000..c955c24e05c37f51856a45956b3befd6552ac812 --- /dev/null +++ b/tests/system-test/2-query/floor.py @@ -0,0 +1,462 @@ +from math import floor +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto(self ,origin_query , floor_query): + pass + floor_result = tdSql.getResult(floor_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = floor(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(floor_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("floor function value has not as expected , sql is \"%s\" "%floor_query ) + sys.exit(1) + else: + tdLog.info("floor value check pass , it work as expected ,sql is \"%s\" "%floor_query ) + + def test_errors(self): + error_sql_lists = [ + "select floor from t1", + # "select floor(-+--+c1) from t1", + # "select +-floor(c1) from t1", + # "select ++-floor(c1) from t1", + # "select ++--floor(c1) from t1", + # "select - -floor(c1)*0 from t1", + # "select floor(tbname+1) from t1 ", + "select floor(123--123)==1 from t1", + "select floor(c1) as 'd1' from t1", + "select floor(c1 ,c2 ) from t1", + "select floor(c1 ,NULL) from t1", + "select floor(,) from t1;", + "select floor(floor(c1) ab from t1)", + "select floor(c1) as int from t1", + "select floor from stb1", + # "select floor(-+--+c1) from stb1", + # "select +-floor(c1) from stb1", + # "select ++-floor(c1) from stb1", + # "select ++--floor(c1) from stb1", + # "select - -floor(c1)*0 from stb1", + # "select floor(tbname+1) from stb1 ", + "select floor(123--123)==1 from stb1", + "select floor(c1) as 'd1' from stb1", + "select floor(c1 ,c2 ) from stb1", + "select floor(c1 ,NULL) from stb1", + "select floor(,) from stb1;", + "select floor(floor(c1) ab from stb1)", + "select floor(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select floor(ts) from t1" , + "select floor(c7) from t1", + "select floor(c8) from t1", + "select floor(c9) from t1", + "select floor(ts) from ct1" , + "select floor(c7) from ct1", + "select floor(c8) from ct1", + "select floor(c9) from ct1", + "select floor(ts) from ct3" , + "select floor(c7) from ct3", + "select floor(c8) from ct3", + "select floor(c9) from ct3", + "select floor(ts) from ct4" , + "select floor(c7) from ct4", + "select floor(c8) from ct4", + "select floor(c9) from ct4", + "select floor(ts) from stb1" , + "select floor(c7) from stb1", + "select floor(c8) from stb1", + "select floor(c9) from stb1" , + + "select floor(ts) from stbbb1" , + "select floor(c7) from stbbb1", + + "select floor(ts) from tbname", + "select floor(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select floor(c1) from t1", + "select floor(c2) from t1", + "select floor(c3) from t1", + "select floor(c4) from t1", + "select floor(c5) from t1", + "select floor(c6) from t1", + + "select floor(c1) from ct1", + "select floor(c2) from ct1", + "select floor(c3) from ct1", + "select floor(c4) from ct1", + "select floor(c5) from ct1", + "select floor(c6) from ct1", + + "select floor(c1) from ct3", + "select floor(c2) from ct3", + "select floor(c3) from ct3", + "select floor(c4) from ct3", + "select floor(c5) from ct3", + "select floor(c6) from ct3", + + "select floor(c1) from stb1", + "select floor(c2) from stb1", + "select floor(c3) from stb1", + "select floor(c4) from stb1", + "select floor(c5) from stb1", + "select floor(c6) from stb1", + + "select floor(c6) as alisb from stb1", + "select floor(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_floor_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select floor(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select floor(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select floor(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select floor(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select floor(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select floor(c6) from ct3") + + # used for regular table + tdSql.query("select floor(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1) + tdSql.checkData(3 , 0, 3) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto( "select c1, c2, c3 , c4, c5 from t1", "select (c1), floor(c2) ,floor(c3), floor(c4), floor(c5) from t1") + + # used for sub table + tdSql.query("select floor(c1) from ct1") + tdSql.checkData(0, 0, 8) + tdSql.checkData(1 , 0, 7) + tdSql.checkData(3 , 0, 5) + tdSql.checkData(5 , 0, 4) + + tdSql.query("select floor(c1) from ct1") + self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct1", "select (c1), floor(c2) ,floor(c3), floor(c4), floor(c5) from ct1") + self.check_result_auto("select floor(floor(floor(floor(floor(floor(floor(floor(floor(floor(c1)))))))))) nest_col_func from ct1;","select c1 from ct1" ) + + # used for stable table + + tdSql.query("select floor(c1) from stb1") + tdSql.checkRows(25) + self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct4 ", "select (c1), floor(c2) ,floor(c3), floor(c4), floor(c5) from ct4") + self.check_result_auto("select floor(floor(floor(floor(floor(floor(floor(floor(floor(floor(c1)))))))))) nest_col_func from ct4;" , "select c1 from ct4" ) + + + # used for not exists table + tdSql.error("select floor(c1) from stbbb1") + tdSql.error("select floor(c1) from tbname") + tdSql.error("select floor(c1) from ct5") + + # mix with common col + tdSql.query("select c1, floor(c1) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,8) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0) + tdSql.query("select c1, floor(c1) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,5) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + tdSql.query("select c1, floor(c1) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,5) + + # mix with common functions + tdSql.query("select c1, floor(c1),c5, floor(c5) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 , 6) + tdSql.checkData(3 , 2 ,6.66000) + tdSql.checkData(3 , 3 ,6.00000) + + tdSql.query("select c1, floor(c1),c5, floor(c5) from stb1 ") + + # mix with agg functions , not support + tdSql.error("select c1, floor(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, floor(c1),c5, count(c5) from ct1 ") + tdSql.error("select floor(c1), count(c5) from stb1 ") + tdSql.error("select floor(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # bug fix for compute + tdSql.query("select c1, abs(c1) -0 ,floor(c1)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 8.000000000) + tdSql.checkData(1, 2, 8.000000000) + + tdSql.query(" select c1, abs(c1) -0 ,floor(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 8.000000000) + tdSql.checkData(1, 2, 6.900000000) + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,3.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,88888) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,8.000000000) + tdSql.checkData(0,4,7.900000000) + tdSql.checkData(0,5,3.000000000) + + def floor_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select floor(c1), floor(c2) ,floor(c3), floor(c4), floor(c5) ,floor(c6) from sub1_bound") + self.check_result_auto( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select floor(c1), floor(c2) ,floor(c3), floor(c3), floor(c2) ,floor(c1) from sub1_bound") + self.check_result_auto("select floor(floor(floor(floor(floor(floor(floor(floor(floor(floor(c1)))))))))) nest_col_func from sub1_bound;" , "select floor(c1) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select floor(c1+0.2) ,floor(c2) , floor(c3+0.3) , floor(c4-0.3), floor(c5/2), floor(c6/2) from sub1_bound ") + tdSql.checkData(0, 0, 2147483647.000000000) + tdSql.checkData(0, 2, 32767.000000000) + tdSql.checkData(0, 3, 126.000000000) + tdSql.checkData(0, 4, 169999997607218212453866206899682148352.000000000) + + tdSql.checkData(4, 0, -2147483643.000000000) + tdSql.checkData(4, 2, -32763.000000000) + tdSql.checkData(4, 3, -124.000000000) + tdSql.checkData(4, 4, -169499995645668991474575059260979281920.000000000) + + self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select floor(c1+1) ,floor(c2) , floor(c3*1) , floor(c4/2), floor(c5)/2, floor(c6) from sub1_bound ") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: floor basic query ============") + + self.basic_floor_function() + + tdLog.printNoPrefix("==========step5: floor boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step6: floor filter query ============") + + self.abs_func_filter() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py new file mode 100644 index 0000000000000000000000000000000000000000..a39bc219466a38eb43290a2ee668a8b5c3e2d232 --- /dev/null +++ b/tests/system-test/2-query/join.py @@ -0,0 +1,335 @@ +import datetime + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +PRIMARY_COL = "ts" + +INT_COL = "c1" +BINT_COL = "c2" +SINT_COL = "c3" +TINT_COL = "c4" +FLOAT_COL = "c5" +DOUBLE_COL = "c6" +BOOL_COL = "c7" + +BINARY_COL = "c8" +NCHAR_COL = "c9" +TS_COL = "c10" + +NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] +CHAR_COL = [ BINARY_COL, NCHAR_COL, ] +BOOLEAN_COL = [ BOOL_COL, ] +TS_TYPE_COL = [ TS_COL, ] + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def __query_condition(self,tbname): + query_condition = [] + for char_col in CHAR_COL: + query_condition.extend( + ( + f"{tbname}.{char_col}", + f"upper( {tbname}.{char_col} )", + ) + ) + query_condition.extend( f"cast( {tbname}.{un_char_col} as binary(16) ) " for un_char_col in NUM_COL) + query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL ) + query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{un_char_col} as binary(32) ) " for un_char_col in NUM_COL ) + for num_col in NUM_COL: + query_condition.extend( + ( + f"{tbname}.{num_col}", + f"sin( {tbname}.{num_col} )" + ) + ) + query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_1} " for num_col_1 in NUM_COL ) + + query_condition.append(''' "test1234!@#$%^&*():'>= 0" + elif col in CHAR_COL: + return f" lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' " + elif col in BOOLEAN_COL: + return f" {tbname}.{col} in (false, true) " + elif col in TS_TYPE_COL or col in PRIMARY_COL: + return f" cast( {tbname}.{col} as binary(16) ) is not null " + else: + return "" + + def __group_condition(self, tbname, col, having = ""): + return f" group by {col} having {having}" if having else f" group by {col} " + + def __join_check(self, tblist, checkrows, join_flag=True): + query_conditions = self.__query_condition(tblist[0]) + join_condition = self.__join_condition(tb_list=tblist) if join_flag else " " + for condition in query_conditions: + where_condition = self.__where_condition(col=condition, tbname=tblist[0]) + group_having = self.__group_condition(tbname=tblist[0], col=condition, having=f"{condition} is not null " ) + group_no_having= self.__group_condition(tbname=tblist[0], col=condition ) + groups = ["", group_having, group_no_having] + for group_condition in groups: + if where_condition: + sql = f" select {condition} from {tblist[0]},{tblist[1]} where {join_condition} and {where_condition} {group_condition} " + else: + sql = f" select {condition} from {tblist[0]},{tblist[1]} where {join_condition} {group_condition} " + + if not join_flag : + tdSql.error(sql=sql) + if len(tblist) == 2: + if "ct1" in tblist or "t1" in tblist: + self.__join_current(sql, checkrows) + elif where_condition or "not null" in group_condition: + self.__join_current(sql, checkrows + 2 ) + elif group_condition: + self.__join_current(sql, checkrows + 3 ) + else: + self.__join_current(sql, checkrows + 5 ) + if len(tblist) > 2 or len(tblist) < 1: + tdSql.error(sql=sql) + + # def __join_err_check(self,tbname): + # sqls = [] + + # for un_char_col in NUM_COL: + # sqls.extend( + # ( + # f"select length( {un_char_col} ) from {tbname} ", + # f"select length(ceil( {un_char_col} )) from {tbname} ", + # f"select {un_char_col} from {tbname} group by length( {un_char_col} ) ", + # ) + # ) + + # sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in NUM_COL ) + # sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + + # sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL) + # sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + # sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in NUM_COL for ts_col in TS_TYPE_COL) + # sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) + # sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) + # sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) + # sqls.extend( + # ( + # f"select length() from {tbname} ", + # f"select length(*) from {tbname} ", + # f"select length(ccccccc) from {tbname} ", + # f"select length(111) from {tbname} ", + # f"select length(c8, 11) from {tbname} ", + # ) + # ) + + # return sqls + + def __join_current(self, sql, checkrows): + tdSql.query(sql=sql) + tdSql.checkRows(checkrows) + + + def __test_current(self): + # sourcery skip: extract-duplicate-method, inline-immediately-returned-variable + tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") + tblist_1 = ["ct1", "ct2"] + self.__join_check(tblist_1, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_1} over==========") + tblist_2 = ["ct2", "ct4"] + self.__join_check(tblist_2, self.rows) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_2} over==========") + tblist_3 = ["t1", "ct4"] + self.__join_check(tblist_3, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_3} over==========") + tblist_4 = ["t1", "ct1"] + self.__join_check(tblist_4, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_4} over==========") + + def __test_error(self): + # sourcery skip: extract-duplicate-method, move-assign-in-block + tdLog.printNoPrefix("==========err sql condition check , must return error==========") + err_list_1 = ["ct1","ct2", "ct4"] + err_list_2 = ["ct1","ct2", "t1"] + err_list_3 = ["ct1","ct4", "t1"] + err_list_4 = ["ct2","ct4", "t1"] + err_list_5 = ["ct1", "ct2","ct4", "t1"] + self.__join_check(err_list_1, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_1} over==========") + self.__join_check(err_list_2, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_2} over==========") + self.__join_check(err_list_3, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_3} over==========") + self.__join_check(err_list_4, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_4} over==========") + self.__join_check(err_list_5, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_5} over==========") + self.__join_check(["ct2", "ct4"], -1, join_flag=False) + tdLog.printNoPrefix("==========err sql condition check in has no join condition over==========") + + tdSql.error( f"select c1, c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{INT_COL}=ct4.{INT_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{TS_COL}=ct4.{TS_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{TS_COL}" ) + tdSql.error( f"select ct2.c1, ct1.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL}" ) + tdSql.error( f"select ct2.c1, ct4.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL} and c1 is not null " ) + tdSql.error( f"select ct2.c1, ct4.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL} and ct1.c1 is not null " ) + + + tbname = ["ct1", "ct2", "ct4", "t1"] + + for tb in tbname: + for errsql in self.__length_err_check(tb): + tdSql.error(sql=errsql) + tdLog.printNoPrefix(f"==========err sql condition check in {tb} over==========") + + + def all_test(self): + self.__test_current() + self.__test_error() + + + def __create_tb(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + create_stb_sql = f'''create table stb1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + ) tags (t1 int) + ''' + create_ntb_sql = f'''create table t1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + ) + ''' + tdSql.execute(create_stb_sql) + tdSql.execute(create_ntb_sql) + + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2} + + def __insert_data(self, rows): + now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + for i in range(rows): + tdSql.execute( + f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f'''insert into ct1 values + ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) + ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) + ''' + ) + + tdSql.execute( + f'''insert into ct4 values + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( + { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000} + ) + ( + { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000} + ) + ''' + ) + + tdSql.execute( + f'''insert into ct2 values + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( + { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, + { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } + ) + ( + { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, + { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } + ) + ''' + ) + + for i in range(rows): + insert_data = f'''insert into t1 values + ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, + "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) + ''' + tdSql.execute(insert_data) + tdSql.execute( + f'''insert into t1 values + ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, + "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } + ) + ( + { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, + "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } + ) + ''' + ) + + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + self.__create_tb() + + tdLog.printNoPrefix("==========step2:insert data") + self.rows = 10 + self.__insert_data(self.rows) + + tdLog.printNoPrefix("==========step3:all check") + self.all_test() + + # tdDnodes.stop(1) + # tdDnodes.start(1) + + # tdSql.execute("use db") + + # tdLog.printNoPrefix("==========step4:after wal, all check again ") + # self.all_test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/length.py b/tests/system-test/2-query/length.py new file mode 100644 index 0000000000000000000000000000000000000000..083bc62c9a8d0411b0db394f991e5fd01ad0474f --- /dev/null +++ b/tests/system-test/2-query/length.py @@ -0,0 +1,249 @@ +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + + +INT_COL = "c1" +BINT_COL = "c2" +SINT_COL = "c3" +TINT_COL = "c4" +FLOAT_COL = "c5" +DOUBLE_COL = "c6" +BOOL_COL = "c7" + +BINARY_COL = "c8" +NCHAR_COL = "c9" +TS_COL = "c10" + +UN_CHAR_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_COL, ] +CHAR_COL = [ BINARY_COL, NCHAR_COL, ] +TS_TYPE_COL = [TS_COL] + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def __length_condition(self): + length_condition = [] + for char_col in CHAR_COL: + length_condition.extend( + ( + char_col, + f"upper( {char_col} )", + ) + ) + length_condition.extend( f"cast( {un_char_col} as binary(16) ) " for un_char_col in UN_CHAR_COL) + length_condition.extend( f"cast( {char_col} + {char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL ) + length_condition.extend( f"cast( {char_col} + {un_char_col} as binary(32) ) " for un_char_col in UN_CHAR_COL ) + + length_condition.append('''"test1234!@#$%^&*():'> 0 " + return "" + + def __group_condition(self, col, having = ""): + return f" group by {col} having {having}" if having else f" group by {col} " + + def __length_current_check(self, tbname): + length_condition = self.__length_condition() + for condition in length_condition: + where_condition = self.__where_condition(condition) + group_having = self.__group_condition(condition, having=f"{condition} is not null " ) + group_no_having= self.__group_condition(condition ) + groups = ["", group_having, group_no_having] + + for group_condition in groups: + tdSql.query(f"select {condition}, length( {condition} ) from {tbname} {where_condition} {group_condition} ") + for i in range(tdSql.queryRows): + if not tdSql.getData(i,1): + tdSql.checkData(i, 1, None) + elif "as nchar" in condition or (NCHAR_COL in condition and "as binary" not in condition): + tdSql.checkData(i, 1, len(str(tdSql.getData(i,0) ) ) * 4 ) + else: + tdSql.checkData(i, 1, len(str(tdSql.getData(i,0) ) ) ) + + + def __length_err_check(self,tbname): + sqls = [] + + for un_char_col in UN_CHAR_COL: + sqls.extend( + ( + f"select length( {un_char_col} ) from {tbname} ", + f"select length(ceil( {un_char_col} )) from {tbname} ", + f"select {un_char_col} from {tbname} group by length( {un_char_col} ) ", + ) + ) + + sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in UN_CHAR_COL ) + sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + + sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL) + sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in UN_CHAR_COL for ts_col in TS_TYPE_COL) + sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) + sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) + sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) + sqls.extend( + ( + f"select length() from {tbname} ", + f"select length(*) from {tbname} ", + f"select length(ccccccc) from {tbname} ", + f"select length(111) from {tbname} ", + f"select length(c8, 11) from {tbname} ", + ) + ) + + return sqls + + def __test_current(self): + tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") + tbname = ["ct1", "ct2", "ct4", "t1", "stb1"] + for tb in tbname: + self.__length_current_check(tb) + tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========") + + def __test_error(self): + tdLog.printNoPrefix("==========err sql condition check , must return error==========") + tbname = ["ct1", "ct2", "ct4", "t1", "stb1"] + + for tb in tbname: + for errsql in self.__length_err_check(tb): + tdSql.error(sql=errsql) + tdLog.printNoPrefix(f"==========err sql condition check in {tb} over==========") + + + def all_test(self): + self.__test_current() + self.__test_error() + + + def __create_tb(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + create_stb_sql = f'''create table stb1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + ) tags (t1 int) + ''' + create_ntb_sql = f'''create table t1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + ) + ''' + tdSql.execute(create_stb_sql) + tdSql.execute(create_ntb_sql) + + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + def __insert_data(self, rows): + now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + for i in range(rows): + tdSql.execute( + f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f'''insert into ct1 values + ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', { now_time + 8 } ) + ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', { now_time + 9 } ) + ''' + ) + + tdSql.execute( + f'''insert into ct4 values + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( + { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000} + ) + ( + { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000} + ) + ''' + ) + + tdSql.execute( + f'''insert into ct2 values + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( + { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, + { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + ) + ( + { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, + { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + ) + ''' + ) + + for i in range(rows): + insert_data = f'''insert into t1 values + ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, + "binary_{i}", "nchar_{i}", { now_time - 1000 * i } ) + ''' + tdSql.execute(insert_data) + tdSql.execute( + f'''insert into t1 values + ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, + "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + ) + ( + { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, + "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + ) + ''' + ) + + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + self.__create_tb() + + tdLog.printNoPrefix("==========step2:insert data") + self.__insert_data(10) + + tdLog.printNoPrefix("==========step3:all check") + self.all_test() + + # tdDnodes.stop(1) + # tdDnodes.start(1) + + # tdSql.execute("use db") + + # tdLog.printNoPrefix("==========step4:after wal, all check again ") + # self.all_test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py new file mode 100644 index 0000000000000000000000000000000000000000..6e4c2921832d695a29c0f4bcf8baf4d79f6f2de8 --- /dev/null +++ b/tests/system-test/2-query/log.py @@ -0,0 +1,652 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_log2(self ,origin_query , log_query): + + log_result = tdSql.getResult(log_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem >0: + elem = math.log(elem,2) + elif elem <=0: + elem = None + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(log_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("log function value has not as expected , sql is \"%s\" "%log_query ) + sys.exit(1) + else: + tdLog.info("log value check pass , it work as expected ,sql is \"%s\" "%log_query ) + + def check_result_auto_log1(self ,origin_query , log_query): + log_result = tdSql.getResult(log_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem >0: + elem = None + elif elem <=0: + elem = None + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(log_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("log function value has not as expected , sql is \"%s\" "%log_query ) + sys.exit(1) + else: + tdLog.info("log value check pass , it work as expected ,sql is \"%s\" "%log_query ) + def check_result_auto_log__10(self ,origin_query , log_query): + log_result = tdSql.getResult(log_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem >0: + elem = None + elif elem <=0: + elem = None + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(log_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice("log function value has not as expected , sql is \"%s\" "%log_query ) + sys.exit(1) + else: + tdLog.info("log value check pass , it work as expected ,sql is \"%s\" "%log_query ) + + def test_errors(self): + error_sql_lists = [ + "select log from t1", + # "select log(-+--+c1 ,2) from t1", + # "select +-log(c1,2) from t1", + # "select ++-log(c1,2) from t1", + # "select ++--log(c1,2) from t1", + # "select - -log(c1,2)*0 from t1", + # "select log(tbname+1,2) from t1 ", + "select log(123--123,2)==1 from t1", + "select log(c1,2) as 'd1' from t1", + "select log(c1 ,c2 ,2) from t1", + "select log(c1 ,NULL ,2) from t1", + "select log(, 2) from t1;", + "select log(log(c1, 2) ab from t1)", + "select log(c1 ,2 ) as int from t1", + "select log from stb1", + # "select log(-+--+c1) from stb1", + # "select +-log(c1) from stb1", + # "select ++-log(c1) from stb1", + # "select ++--log(c1) from stb1", + # "select - -log(c1)*0 from stb1", + # "select log(tbname+1) from stb1 ", + "select log(123--123 ,2)==1 from stb1", + "select log(c1 ,2) as 'd1' from stb1", + "select log(c1 ,c2 ,2 ) from stb1", + "select log(c1 ,NULL,2) from stb1", + "select log(,) from stb1;", + "select log(log(c1 , 2) ab from stb1)", + "select log(c1 , 2) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select log(ts ,2 ) from t1" , + "select log(c7,c2 ) from t1", + "select log(c8,c1 ) from t1", + "select log(c9,c2 ) from t1", + "select log(ts,c7 ) from ct1" , + "select log(c7,c9 ) from ct1", + "select log(c8,c2 ) from ct1", + "select log(c9,c1 ) from ct1", + "select log(ts,2 ) from ct3" , + "select log(c7,2 ) from ct3", + "select log(c8,2 ) from ct3", + "select log(c9,2 ) from ct3", + "select log(ts,2 ) from ct4" , + "select log(c7,2 ) from ct4", + "select log(c8,2 ) from ct4", + "select log(c9,2 ) from ct4", + "select log(ts,2 ) from stb1" , + "select log(c7,2 ) from stb1", + "select log(c8,2 ) from stb1", + "select log(c9,2 ) from stb1" , + + "select log(ts,2 ) from stbbb1" , + "select log(c7,2 ) from stbbb1", + + "select log(ts,2 ) from tbname", + "select log(c9,2 ) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select log(c1,2 ) from t1", + "select log(c2,2 ) from t1", + "select log(c3,2 ) from t1", + "select log(c4,2 ) from t1", + "select log(c5,2 ) from t1", + "select log(c6,2 ) from t1", + + "select log(c1,2 ) from ct1", + "select log(c2,2 ) from ct1", + "select log(c3,2 ) from ct1", + "select log(c4,2 ) from ct1", + "select log(c5,2 ) from ct1", + "select log(c6,2 ) from ct1", + + "select log(c1,2 ) from ct3", + "select log(c2,2 ) from ct3", + "select log(c3,2 ) from ct3", + "select log(c4,2 ) from ct3", + "select log(c5,2 ) from ct3", + "select log(c6,2 ) from ct3", + + "select log(c1,2 ) from stb1", + "select log(c2,2 ) from stb1", + "select log(c3,2 ) from stb1", + "select log(c4,2 ) from stb1", + "select log(c5,2 ) from stb1", + "select log(c6,2 ) from stb1", + + "select log(c6,2) as alisb from stb1", + "select log(c6,2) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_log_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select log(c1 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select log(c2 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select log(c3 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select log(c4 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select log(c5 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select log(c6 ,2) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select log(c1 ,2) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 0.000000000) + tdSql.checkData(3 , 0, 1.584962501) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_log2( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,2), log(c2 ,2) ,log(c3, 2), log(c4 ,2), log(c5 ,2) from t1") + self.check_result_auto_log1( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,1), log(c2 ,1) ,log(c3, 1), log(c4 ,1), log(c5 ,1) from t1") + self.check_result_auto_log__10( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,-10), log(c2 ,-10) ,log(c3, -10), log(c4 ,-10), log(c5 ,-10) from t1") + + # used for sub table + tdSql.query("select c1 ,log(c1 ,3) from ct1") + tdSql.checkData(0, 1, 1.892789261) + tdSql.checkData(1 , 1, 1.771243749) + tdSql.checkData(3 , 1, 1.464973521) + tdSql.checkData(4 , 1, None) + + # test bug fix for log(c1,c2) + + tdSql.query("select c1, c2 ,log(c1,c2) from ct1") + tdSql.checkData(0 , 2, 0.182485070) + tdSql.checkData(1 , 2, 0.172791608) + tdSql.checkData(2 , 2, 0.161311499) + tdSql.checkData(3 , 2, 0.147315235) + tdSql.checkData(4 , 2, None) + + + self.check_result_auto_log2( "select c1, c2, c3 , c4, c5 from ct1", "select log(c1,2), log(c2,2) ,log(c3,2), log(c4,2), log(c5,2) from ct1") + self.check_result_auto_log__10( "select c1, c2, c3 , c4, c5 from ct1", "select log(c1,-10), log(c2,-10) ,log(c3,-10), log(c4,-10), log(c5,-10) from ct1") + + # nest query for log functions + tdSql.query("select c1 , log(c1,3) ,log(log(c1,3),3) , log(log(log(c1,3),3),3) from ct1;") + tdSql.checkData(0 , 0 , 8) + tdSql.checkData(0 , 1 , 1.892789261) + tdSql.checkData(0 , 2 , 0.580779541) + tdSql.checkData(0 , 3 , -0.494609470) + + tdSql.checkData(1 , 0 , 7) + tdSql.checkData(1 , 1 , 1.771243749) + tdSql.checkData(1 , 2 , 0.520367366) + tdSql.checkData(1 , 3 , -0.594586689) + + tdSql.checkData(4 , 0 , 0) + tdSql.checkData(4 , 1 , None) + tdSql.checkData(4 , 2 , None) + tdSql.checkData(4 , 3 , None) + + # # used for stable table + + tdSql.query("select log(c1, 2) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select log(c1, 2) from stbbb1") + tdSql.error("select log(c1, 2) from tbname") + tdSql.error("select log(c1, 2) from ct5") + + # mix with common col + tdSql.query("select c1, log(c1 ,2) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,3.000000000) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,None) + tdSql.query("select c1, log(c1,2) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,2.321928095) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + tdSql.query("select c1, log(c1 ,2 ) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,2.321928095) + + # mix with common functions + tdSql.query("select c1, log(c1 ,2),c5, log(c5 ,2) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 , 2.584962501) + tdSql.checkData(3 , 2 ,6.66000) + tdSql.checkData(3 , 3 ,2.735522144) + + tdSql.query("select c1, log(c1,1),c5, floor(c5 ) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, log(c1 ,2),c5, count(c5) from stb1 ") + tdSql.error("select c1, log(c1 ,2),c5, count(c5) from ct1 ") + tdSql.error("select log(c1 ,2), count(c5) from stb1 ") + tdSql.error("select log(c1 ,2), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # # bug fix for compute + tdSql.query("select c1, log(c1 ,2) -0 ,log(c1-4 ,2)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 3.000000000) + tdSql.checkData(1, 2, 2.000000000) + + tdSql.query(" select c1, log(c1 ,2) -0 ,log(c1-0.1 ,2)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 3.000000000) + tdSql.checkData(1, 2, 2.881852653) + + tdSql.query("select c1, log(c1, -10), c2, log(c2, -10), c3, log(c3, -10) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, log(c1, 100000000) from ct1") # bigint to double data overflow + tdSql.checkData(0, 1, 0.112886248) + tdSql.checkData(1, 1, 0.105637255) + tdSql.checkData(4, 1, None) + + + tdSql.query("select c1, log(c1, 10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(0, 1, 0.069468461) + tdSql.checkData(1, 1, 0.065007542) + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, log(c1, 10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, log(c1, 10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, 0.036123599) + tdSql.checkData(1, 1, 0.033803922) + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, log(c1, 10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, log(c1, 10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, 0.026561470) + tdSql.checkData(1, 1, 0.024855825) + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, log(c1, 10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, log(c1, 10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, 0.022577250) + tdSql.checkData(1, 1, 0.021127451) + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, log(c1, 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def log_base_test(self): + + # base is an regular number ,int or double + tdSql.query("select c1, log(c1, 2) from ct1") + tdSql.checkData(0, 1,3.000000000) + tdSql.query("select c1, log(c1, 2.0) from ct1") + tdSql.checkData(0, 1, 3.000000000) + + tdSql.query("select c1, log(1, 2.0) from ct1") + tdSql.checkData(0, 1, 0.000000000) + tdSql.checkRows(13) + + + # # bug for compute in functions + # tdSql.query("select c1, abs(1/0) from ct1") + # tdSql.checkData(0, 0, 8) + # tdSql.checkData(0, 1, 1) + + tdSql.query("select c1, log(1, 2.0) from ct1") + tdSql.checkData(0, 1, 0.000000000) + tdSql.checkRows(13) + + # two cols start log(x,y) + tdSql.query("select c1,c2, log(c1,c2) from ct1") + tdSql.checkData(0, 2, 0.182485070) + tdSql.checkData(1, 2, 0.172791608) + tdSql.checkData(4, 2, None) + + tdSql.query("select c1,c2, log(c2,c1) from ct1") + tdSql.checkData(0, 2, 5.479900349) + tdSql.checkData(1, 2, 5.787318105) + tdSql.checkData(4, 2, None) + + tdSql.query("select c1, log(2.0 , c1) from ct1") + tdSql.checkData(0, 1, 0.333333333) + tdSql.checkData(1, 1, 0.356207187) + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, log(2.0 , ceil(abs(c1))) from ct1") + tdSql.checkData(0, 1, 0.333333333) + tdSql.checkData(1, 1, 0.356207187) + tdSql.checkData(4, 1, None) + + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,3.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,88888) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,8.000000000) + tdSql.checkData(0,4,7.900000000) + tdSql.checkData(0,5,3.000000000) + + def log_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto_log2( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select log(c1,2), log(c2,2) ,log(c3,2), log(c4,2), log(c5,2) ,log(c6,2) from sub1_bound") + self.check_result_auto_log__10( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select log(c1,-10), log(c2,-10) ,log(c3,-10), log(c4,-10), log(c5,-10) ,log(c6,-10) from sub1_bound") + + self.check_result_auto_log2( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select log(c1,2), log(c2,2) ,log(c3,2), log(c3,2), log(c2,2) ,log(c1,2) from sub1_bound") + + + self.check_result_auto_log2("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select log(abs(c1) ,2) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select log(abs(c1),2) ,log(abs(c2),2) , log(abs(c3),2) , log(abs(c4),2), log(abs(c5),2), log(abs(c6),2) from sub1_bound ") + tdSql.checkData(0,0,math.log(2147483647,2)) + tdSql.checkData(0,1,math.log(9223372036854775807 ,2)) + tdSql.checkData(0,2,math.log(32767,2)) + tdSql.checkData(0,3,math.log(127 ,2)) + tdSql.checkData(0,4,math.log(339999995214436424907732413799364296704.00000,2)) + tdSql.checkData(0,5,math.log(169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000 ,2)) + tdSql.checkData(1,0,math.log(2147483647 ,2)) + tdSql.checkData(1,1,math.log(9223372036854775807 ,2)) + tdSql.checkData(1,2,math.log(32767 ,2)) + tdSql.checkData(1,3,math.log(127,2)) + tdSql.checkData(1,4,math.log(339999995214436424907732413799364296704.00000 ,2)) + tdSql.checkData(1,5,math.log(169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000 ,2)) + tdSql.checkData(3,0,math.log(2147483646,2)) + tdSql.checkData(3,1,math.log(9223372036854775806,2)) + tdSql.checkData(3,2,math.log(32766,2)) + tdSql.checkData(3,3,math.log(126 ,2)) + tdSql.checkData(3,4,math.log(339999995214436424907732413799364296704.00000,2)) + tdSql.checkData(3,5,math.log(169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000,2)) + + # check + - * / in functions + tdSql.query("select log(abs(c1+1) ,2) ,log(abs(c2),2) , log(abs(c3*1),2) , log(abs(c4/2),2), log(abs(c5) ,2)/2, log(abs(c6) ,2) from sub1_bound ") + tdSql.checkData(0,0,math.log(2147483648.000000000,2)) + tdSql.checkData(0,1,math.log(9223372036854775807,2)) + tdSql.checkData(0,2,math.log(32767.000000000,2)) + tdSql.checkData(0,3,math.log(63.500000000,2)) + tdSql.checkData(0,4,63.999401166) + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: log basic query ============") + + self.basic_log_function() + + tdLog.printNoPrefix("==========step5: big number log query ============") + + self.test_big_number() + + tdLog.printNoPrefix("==========step6: base number for log query ============") + + self.log_base_test() + + tdLog.printNoPrefix("==========step7: log boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step8: log filter query ============") + + self.abs_func_filter() + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/lower.py b/tests/system-test/2-query/lower.py new file mode 100644 index 0000000000000000000000000000000000000000..5445c37b8a0b5e08998371585f0e293c36378320 --- /dev/null +++ b/tests/system-test/2-query/lower.py @@ -0,0 +1,246 @@ +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + + +INT_COL = "c1" +BINT_COL = "c2" +SINT_COL = "c3" +TINT_COL = "c4" +FLOAT_COL = "c5" +DOUBLE_COL = "c6" +BOOL_COL = "c7" + +BINARY_COL = "c8" +NCHAR_COL = "c9" +TS_COL = "c10" + +UN_CHAR_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_COL, ] +CHAR_COL = [ BINARY_COL, NCHAR_COL, ] +TS_TYPE_COL = [TS_COL] + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def __lower_condition(self): + lower_condition = [] + for char_col in CHAR_COL: + lower_condition.extend( + ( + char_col, + f"upper( {char_col} )", + ) + ) + lower_condition.extend(f"cast( {un_char_col} as binary(16) ) " for un_char_col in UN_CHAR_COL) + lower_condition.extend( f"cast( {char_col} + {char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL ) + lower_condition.extend( f"cast( {char_col} + {un_char_col} as binary(32) ) " for un_char_col in UN_CHAR_COL ) + + lower_condition.append('''"test1234!@#$%^&*():'> 0 " + return "" + + def __group_condition(self, col, having = ""): + return f" group by {col} having {having}" if having else f" group by {col} " + + def __lower_current_check(self, tbname): + lower_condition = self.__lower_condition() + for condition in lower_condition: + where_condition = self.__where_condition(condition) + group_having = self.__group_condition(condition, having=f"{condition} is not null " ) + group_no_having= self.__group_condition(condition ) + groups = ["", group_having, group_no_having] + + for group_condition in groups: + tdSql.query(f"select {condition} from {tbname} {where_condition} {group_condition} ") + datas = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + lower_data = [ str(data).lower() if data else None for data in datas ] + tdSql.query(f"select lower( {condition} ) from {tbname} {where_condition} {group_condition}") + for i in range(len(lower_data)): + tdSql.checkData(i, 0, lower_data[i] ) if lower_data[i] else tdSql.checkData(i, 0, None) + + def __lower_err_check(self,tbname): + sqls = [] + + for un_char_col in UN_CHAR_COL: + sqls.extend( + ( + f"select lower( {un_char_col} ) from {tbname} ", + f"select lower(ceil( {un_char_col} )) from {tbname} ", + f"select {un_char_col} from {tbname} group by lower( {un_char_col} ) ", + ) + ) + + sqls.extend( f"select lower( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in UN_CHAR_COL ) + sqls.extend( f"select lower( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + + sqls.extend( f"select {char_col} from {tbname} group by lower( {char_col} ) " for char_col in CHAR_COL) + sqls.extend( f"select lower( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + sqls.extend( f"select lower( {char_col} + {ts_col} ) from {tbname} " for char_col in UN_CHAR_COL for ts_col in TS_TYPE_COL) + sqls.extend( f"select lower( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) + sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) + sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) + sqls.extend( + ( + f"select lower() from {tbname} ", + f"select lower(*) from {tbname} ", + f"select lower(ccccccc) from {tbname} ", + f"select lower(111) from {tbname} ", + f"select lower(c8, 11) from {tbname} ", + ) + ) + + return sqls + + def __test_current(self): + tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") + tbname = ["ct1", "ct2", "ct4", "t1", "stb1"] + for tb in tbname: + self.__lower_current_check(tb) + tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========") + + def __test_error(self): + tdLog.printNoPrefix("==========err sql condition check , must return error==========") + tbname = ["ct1", "ct2", "ct4", "t1", "stb1"] + + for tb in tbname: + for errsql in self.__lower_err_check(tb): + tdSql.error(sql=errsql) + tdLog.printNoPrefix(f"==========err sql condition check in {tb} over==========") + + + def all_test(self): + self.__test_current() + self.__test_error() + + + def __create_tb(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + create_stb_sql = f'''create table stb1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + ) tags (t1 int) + ''' + create_ntb_sql = f'''create table t1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + ) + ''' + tdSql.execute(create_stb_sql) + tdSql.execute(create_ntb_sql) + + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + def __insert_data(self, rows): + now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + for i in range(rows): + tdSql.execute( + f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f'''insert into ct1 values + ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', { now_time + 8 } ) + ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', { now_time + 9 } ) + ''' + ) + + tdSql.execute( + f'''insert into ct4 values + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( + { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000} + ) + ( + { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000} + ) + ''' + ) + + tdSql.execute( + f'''insert into ct2 values + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( + { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, + { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + ) + ( + { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, + { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + ) + ''' + ) + + for i in range(rows): + insert_data = f'''insert into t1 values + ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, + "binary_{i}", "nchar_{i}", { now_time - 1000 * i } ) + ''' + tdSql.execute(insert_data) + tdSql.execute( + f'''insert into t1 values + ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, + "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + ) + ( + { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, + "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + ) + ''' + ) + + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + self.__create_tb() + + tdLog.printNoPrefix("==========step2:insert data") + self.__insert_data(10) + + tdLog.printNoPrefix("==========step3:all check") + self.all_test() + + # tdDnodes.stop(1) + # tdDnodes.start(1) + + # tdSql.execute("use db") + + # tdLog.printNoPrefix("==========step4:after wal, all check again ") + # self.all_test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/pow.py b/tests/system-test/2-query/pow.py new file mode 100644 index 0000000000000000000000000000000000000000..8b0137b4118f5057a08392e27a33fbd7c82a484f --- /dev/null +++ b/tests/system-test/2-query/pow.py @@ -0,0 +1,652 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_pow2(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = math.pow(elem,2) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("pow function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("pow value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def check_result_auto_pow1(self ,origin_query , pow_query): + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else : + elem = pow(elem ,1) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("pow function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("pow value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def check_result_auto_pow__10(self ,origin_query , pow_query): + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem == 0: + elem = None + else: + elem = pow(elem ,-10) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("pow function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("pow value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select pow from t1", + # "select pow(-+--+c1 ,2) from t1", + # "select +-pow(c1,2) from t1", + # "select ++-pow(c1,2) from t1", + # "select ++--pow(c1,2) from t1", + # "select - -pow(c1,2)*0 from t1", + # "select pow(tbname+1,2) from t1 ", + "select pow(123--123,2)==1 from t1", + "select pow(c1,2) as 'd1' from t1", + "select pow(c1 ,c2 ,2) from t1", + "select pow(c1 ,NULL ,2) from t1", + "select pow(, 2) from t1;", + "select pow(pow(c1, 2) ab from t1)", + "select pow(c1 ,2 ) as int from t1", + "select pow from stb1", + # "select pow(-+--+c1) from stb1", + # "select +-pow(c1) from stb1", + # "select ++-pow(c1) from stb1", + # "select ++--pow(c1) from stb1", + # "select - -pow(c1)*0 from stb1", + # "select pow(tbname+1) from stb1 ", + "select pow(123--123 ,2)==1 from stb1", + "select pow(c1 ,2) as 'd1' from stb1", + "select pow(c1 ,c2 ,2 ) from stb1", + "select pow(c1 ,NULL,2) from stb1", + "select pow(,) from stb1;", + "select pow(pow(c1 , 2) ab from stb1)", + "select pow(c1 , 2) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select pow(ts ,2 ) from t1" , + "select pow(c7,c1 ) from t1", + "select pow(c8,c2) from t1", + "select pow(c9,c3 ) from t1", + "select pow(ts,c4 ) from ct1" , + "select pow(c7,c5 ) from ct1", + "select pow(c8,c6 ) from ct1", + "select pow(c9,c8 ) from ct1", + "select pow(ts,2 ) from ct3" , + "select pow(c7,2 ) from ct3", + "select pow(c8,2 ) from ct3", + "select pow(c9,2 ) from ct3", + "select pow(ts,2 ) from ct4" , + "select pow(c7,2 ) from ct4", + "select pow(c8,2 ) from ct4", + "select pow(c9,2 ) from ct4", + "select pow(ts,2 ) from stb1" , + "select pow(c7,2 ) from stb1", + "select pow(c8,2 ) from stb1", + "select pow(c9,2 ) from stb1" , + + "select pow(ts,2 ) from stbbb1" , + "select pow(c7,2 ) from stbbb1", + + "select pow(ts,2 ) from tbname", + "select pow(c9,2 ) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select pow(c1,2 ) from t1", + "select pow(c2,2 ) from t1", + "select pow(c3,2 ) from t1", + "select pow(c4,2 ) from t1", + "select pow(c5,2 ) from t1", + "select pow(c6,2 ) from t1", + + "select pow(c1,2 ) from ct1", + "select pow(c2,2 ) from ct1", + "select pow(c3,2 ) from ct1", + "select pow(c4,2 ) from ct1", + "select pow(c5,2 ) from ct1", + "select pow(c6,2 ) from ct1", + + "select pow(c1,2 ) from ct3", + "select pow(c2,2 ) from ct3", + "select pow(c3,2 ) from ct3", + "select pow(c4,2 ) from ct3", + "select pow(c5,2 ) from ct3", + "select pow(c6,2 ) from ct3", + + "select pow(c1,2 ) from stb1", + "select pow(c2,2 ) from stb1", + "select pow(c3,2 ) from stb1", + "select pow(c4,2 ) from stb1", + "select pow(c5,2 ) from stb1", + "select pow(c6,2 ) from stb1", + + "select pow(c6,2) as alisb from stb1", + "select pow(c6,2) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_pow_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select pow(c1 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c2 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c3 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c4 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c5 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c6 ,2) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select pow(c1 ,2) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1.000000000) + tdSql.checkData(3 , 0, 9.000000000) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_pow2( "select c1, c2, c3 , c4, c5 from t1", "select pow(c1 ,2), pow(c2 ,2) ,pow(c3, 2), pow(c4 ,2), pow(c5 ,2) from t1") + self.check_result_auto_pow1( "select c1, c2, c3 , c4, c5 from t1", "select pow(c1 ,1), pow(c2 ,1) ,pow(c3, 1), pow(c4 ,1), pow(c5 ,1) from t1") + self.check_result_auto_pow__10( "select c1, c2, c3 , c4, c5 from t1", "select pow(c1 ,-10), pow(c2 ,-10) ,pow(c3, -10), pow(c4 ,-10), pow(c5 ,-10) from t1") + + # used for sub table + tdSql.query("select c1 ,pow(c1 ,2) from ct1") + tdSql.checkData(0, 1, 64.000000000) + tdSql.checkData(1 , 1, 49.000000000) + tdSql.checkData(3 , 1, 25.000000000) + tdSql.checkData(4 , 1, 0) + + # # test bug fix for pow(c1,c2) + + tdSql.query("select c1, c5 ,pow(c1,c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, 104577724.506799981) + tdSql.checkData(2 , 2, 3684781.623933245) + tdSql.checkData(3 , 2, 152225.429759376) + tdSql.checkData(4 , 2, 7573.273783071) + + + self.check_result_auto_pow2( "select c1, c2, c3 , c4, c5 from ct1", "select pow(c1,2), pow(c2,2) ,pow(c3,2), pow(c4,2), pow(c5,2) from ct1") + self.check_result_auto_pow__10( "select c1, c2, c3 , c4, c5 from ct1", "select pow(c1,-10), pow(c2,-10) ,pow(c3,-10), pow(c4,-10), pow(c5,-10) from ct1") + + # nest query for pow functions + tdSql.query("select c1 , pow(c1,2) ,pow(pow(c1,2),2) , pow(pow(pow(c1,2),2),2) from ct1;") + tdSql.checkData(0 , 0 , 8) + tdSql.checkData(0 , 1 , 64.000000000) + tdSql.checkData(0 , 2 , 4096.000000000) + tdSql.checkData(0 , 3 , 16777216.000000000) + + tdSql.checkData(1 , 0 , 7) + tdSql.checkData(1 , 1 , 49.000000000) + tdSql.checkData(1 , 2 , 2401.000000000) + tdSql.checkData(1 , 3 , 5764801.000000000) + + tdSql.checkData(4 , 0 , 0) + tdSql.checkData(4 , 1 , 0.000000000) + tdSql.checkData(4 , 2 , 0.000000000) + tdSql.checkData(4 , 3 , 0.000000000) + + # # used for stable table + + tdSql.query("select pow(c1, 2) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select pow(c1, 2) from stbbb1") + tdSql.error("select pow(c1, 2) from tbname") + tdSql.error("select pow(c1, 2) from ct5") + + # mix with common col + tdSql.query("select c1, pow(c1 ,2) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,64.000000000) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0.000000000) + tdSql.query("select c1, pow(c1,2) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,25.000000000) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + + # mix with common functions + tdSql.query("select c1, pow(c1 ,2),pow(c1,2), log(pow(c1,2) ,2) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,36.000000000) + tdSql.checkData(3 , 2 ,36.000000000) + tdSql.checkData(3 , 3 ,5.169925001) + + tdSql.query("select c1, pow(c1,1),c5, floor(c5 ) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, pow(c1 ,2),c5, count(c5) from stb1 ") + tdSql.error("select c1, pow(c1 ,2),c5, count(c5) from ct1 ") + tdSql.error("select pow(c1 ,2), count(c5) from stb1 ") + tdSql.error("select pow(c1 ,2), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # # bug fix for compute + tdSql.query("select c1, pow(c1 ,2) -0 ,pow(c1-4 ,2)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 64.000000000) + tdSql.checkData(1, 2, 16.000000000) + + tdSql.query(" select c1, pow(c1 ,2) -0 ,pow(c1-0.1 ,2)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 64.000000000) + tdSql.checkData(1, 2, 62.310000000) + + tdSql.query("select c1, pow(c1, -10), c2, pow(c2, -10), c3, pow(c3, -10) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, pow(c1, 100000000) from ct1") # bigint to double data overflow + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + + tdSql.query("select c1, pow(c1, 10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, pow(c1, 10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def pow_base_test(self): + + # base is an regular number ,int or double + tdSql.query("select c1, pow(c1, 2) from ct1") + tdSql.checkData(0, 1,64.000000000) + tdSql.query("select c1, pow(c1, 2.0) from ct1") + tdSql.checkData(0, 1, 64.000000000) + + tdSql.query("select c1, pow(1, 2.0) from ct1") + tdSql.checkData(0, 1, 1.000000000) + tdSql.checkRows(13) + + + # # bug for compute in functions + # tdSql.query("select c1, abs(1/0) from ct1") + # tdSql.checkData(0, 0, 8) + # tdSql.checkData(0, 1, 1) + + tdSql.query("select c1, pow(1, 2.0) from ct1") + tdSql.checkData(0, 1, 1.000000000) + tdSql.checkRows(13) + + # two cols start pow(x,y) + tdSql.query("select c1,c2, pow(c1,c2) from ct1") + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + tdSql.checkData(4, 2, 1.000000000) + + tdSql.query("select c1,c2, pow(c2,c1) from ct1") + tdSql.checkData(0, 2, 3897131646727578700481513520437089271808.000000000) + tdSql.checkData(1, 2, 17217033054561120738612297152331776.000000000) + tdSql.checkData(4, 2, 1.000000000) + + tdSql.query("select c1, pow(2.0 , c1) from ct1") + tdSql.checkData(0, 1, 256.000000000) + tdSql.checkData(1, 1, 128.000000000) + tdSql.checkData(4, 1, 1.000000000) + + tdSql.query("select c1, pow(2.0 , c1) from ct1") + tdSql.checkData(0, 1, 256.000000000) + tdSql.checkData(1, 1, 128.000000000) + tdSql.checkData(4, 1, 1.000000000) + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,64.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,25.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,25.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c15 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,3.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) , round(abs(c1))-0.5 from ct4 where c1>log(c1,2) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,88888) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,8.000000000) + tdSql.checkData(0,4,7.900000000) + tdSql.checkData(0,5,3.000000000) + tdSql.checkData(0,6,7.500000000) + + def round_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select round(c1), round(c2) ,round(c3), round(c4), round(c5) ,round(c6) from sub1_bound") + self.check_result_auto( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select round(c1), round(c2) ,round(c3), round(c3), round(c2) ,round(c1) from sub1_bound") + self.check_result_auto("select round(round(round(round(round(round(round(round(round(round(c1)))))))))) nest_col_func from sub1_bound;" , "select round(c1) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select round(c1+0.2) ,round(c2) , round(c3+0.3) , round(c4-0.3), round(c5/2), round(c6/2) from sub1_bound ") + tdSql.checkData(0, 0, 2147483647.000000000) + tdSql.checkData(0, 2, 32767.000000000) + tdSql.checkData(0, 3, 127.000000000) + tdSql.checkData(0, 4, 169999997607218212453866206899682148352.000000000) + + tdSql.checkData(4, 0, -2147483643.000000000) + tdSql.checkData(4, 2, -32763.000000000) + tdSql.checkData(4, 3, -123.000000000) + tdSql.checkData(4, 4, -169499995645668991474575059260979281920.000000000) + + self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select round(c1+1) ,round(c2) , round(c3*1) , round(c4/2), round(c5)/2, round(c6) from sub1_bound ") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: round basic query ============") + + self.basic_round_function() + + tdLog.printNoPrefix("==========step5: round boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step6: round filter query ============") + + self.abs_func_filter() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/smaTest.py b/tests/system-test/2-query/smaTest.py new file mode 100644 index 0000000000000000000000000000000000000000..67824cc3a3c372821c5014d48f6d2dbbc9ee9066 --- /dev/null +++ b/tests/system-test/2-query/smaTest.py @@ -0,0 +1,131 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys + +from numpy.lib.function_base import insert +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + +# constant define +WAITS = 5 # wait seconds + +class TDTestCase: + # + # --------------- main frame ------------------- + # + # updatecfgDict = {'debugFlag': 135} + # updatecfgDict = {'fqdn': 135} + + def caseDescription(self): + ''' + limit and offset keyword function test cases; + case1: limit offset base function test + case2: offset return valid + ''' + return + + # init + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + tdSql.prepare() + self.create_tables(); + self.ts = 1500000000000 + + + # run case + def run(self): + # insert data + self.insert_data1("t1", self.ts, 1000*10000) + self.insert_data1("t4", self.ts, 1000*10000) + # test base case + # self.test_case1() + tdLog.debug(" LIMIT test_case1 ............ [OK]") + # test advance case + # self.test_case2() + tdLog.debug(" LIMIT test_case2 ............ [OK]") + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + # + # --------------- case ------------------- + # + + # create table + def create_tables(self): + # super table + tdSql.execute("create table st(ts timestamp, i1 int,i2 int) tags(area int)"); + # child table + tdSql.execute("create table t1 using st tags(1)"); + + tdSql.execute("create table st1(ts timestamp, i1 int ,i2 int) tags(area int) sma(i2) "); + tdSql.execute("create table t4 using st1 tags(1)"); + + return + + # insert data1 + def insert_data(self, tbname, ts_start, count): + pre_insert = "insert into %s values"%tbname + sql = pre_insert + tdLog.debug("doing insert table %s rows=%d ..."%(tbname, count)) + for i in range(count): + sql += " (%d,%d)"%(ts_start + i*1000, i ) + if i >0 and i%30000 == 0: + tdSql.execute(sql) + sql = pre_insert + # end sql + if sql != pre_insert: + tdSql.execute(sql) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + def insert_data1(self, tbname, ts_start, count): + pre_insert = "insert into %s values"%tbname + sql = pre_insert + tdLog.debug("doing insert table %s rows=%d ..."%(tbname, count)) + for i in range(count): + sql += " (%d,%d,%d)"%(ts_start + i*1000, i , i+1) + if i >0 and i%30000 == 0: + tdSql.execute(sql) + sql = pre_insert + # end sql + if sql != pre_insert: + tdSql.execute(sql) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + # test case1 base + # def test_case1(self): + # # + # # limit base function + # # + # # base no where + # sql = "select * from t1 limit 10" + # tdSql.waitedQuery(sql, 10, WAITS) + + +# +# add case with filename +# +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py new file mode 100644 index 0000000000000000000000000000000000000000..28e869e044ff57f113663e05c9c5267088084972 --- /dev/null +++ b/tests/system-test/2-query/sqrt.py @@ -0,0 +1,551 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_sqrt(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem < 0: + elem = None + else: + elem = math.sqrt(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("sqrt function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("sqrt value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select sqrt from t1", + # "select sqrt(-+--+c1 ) from t1", + # "select +-sqrt(c1) from t1", + # "select ++-sqrt(c1) from t1", + # "select ++--sqrt(c1) from t1", + # "select - -sqrt(c1)*0 from t1", + # "select sqrt(tbname+1) from t1 ", + "select sqrt(123--123)==1 from t1", + "select sqrt(c1) as 'd1' from t1", + "select sqrt(c1 ,c2) from t1", + "select sqrt(c1 ,NULL ) from t1", + "select sqrt(,) from t1;", + "select sqrt(sqrt(c1) ab from t1)", + "select sqrt(c1 ) as int from t1", + "select sqrt from stb1", + # "select sqrt(-+--+c1) from stb1", + # "select +-sqrt(c1) from stb1", + # "select ++-sqrt(c1) from stb1", + # "select ++--sqrt(c1) from stb1", + # "select - -sqrt(c1)*0 from stb1", + # "select sqrt(tbname+1) from stb1 ", + "select sqrt(123--123)==1 from stb1", + "select sqrt(c1) as 'd1' from stb1", + "select sqrt(c1 ,c2 ) from stb1", + "select sqrt(c1 ,NULL) from stb1", + "select sqrt(,) from stb1;", + "select sqrt(sqrt(c1) ab from stb1)", + "select sqrt(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select sqrt(ts) from t1" , + "select sqrt(c7) from t1", + "select sqrt(c8) from t1", + "select sqrt(c9) from t1", + "select sqrt(ts) from ct1" , + "select sqrt(c7) from ct1", + "select sqrt(c8) from ct1", + "select sqrt(c9) from ct1", + "select sqrt(ts) from ct3" , + "select sqrt(c7) from ct3", + "select sqrt(c8) from ct3", + "select sqrt(c9) from ct3", + "select sqrt(ts) from ct4" , + "select sqrt(c7) from ct4", + "select sqrt(c8) from ct4", + "select sqrt(c9) from ct4", + "select sqrt(ts) from stb1" , + "select sqrt(c7) from stb1", + "select sqrt(c8) from stb1", + "select sqrt(c9) from stb1" , + + "select sqrt(ts) from stbbb1" , + "select sqrt(c7) from stbbb1", + + "select sqrt(ts) from tbname", + "select sqrt(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select sqrt(c1) from t1", + "select sqrt(c2) from t1", + "select sqrt(c3) from t1", + "select sqrt(c4) from t1", + "select sqrt(c5) from t1", + "select sqrt(c6) from t1", + + "select sqrt(c1) from ct1", + "select sqrt(c2) from ct1", + "select sqrt(c3) from ct1", + "select sqrt(c4) from ct1", + "select sqrt(c5) from ct1", + "select sqrt(c6) from ct1", + + "select sqrt(c1) from ct3", + "select sqrt(c2) from ct3", + "select sqrt(c3) from ct3", + "select sqrt(c4) from ct3", + "select sqrt(c5) from ct3", + "select sqrt(c6) from ct3", + + "select sqrt(c1) from stb1", + "select sqrt(c2) from stb1", + "select sqrt(c3) from stb1", + "select sqrt(c4) from stb1", + "select sqrt(c5) from stb1", + "select sqrt(c6) from stb1", + + "select sqrt(c6) as alisb from stb1", + "select sqrt(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_sqrt_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select sqrt(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select sqrt(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1.000000000) + tdSql.checkData(3 , 0, 1.732050808) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_sqrt( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select sqrt(abs(c1)), sqrt(abs(c2)) ,sqrt(abs(c3)), sqrt(abs(c4)), sqrt(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,sqrt(c2) from ct1") + tdSql.checkData(0, 1, 298.140906284) + tdSql.checkData(1 , 1, 278.885281074) + tdSql.checkData(3 , 1, 235.701081881) + tdSql.checkData(4 , 1, 0.000000000) + + tdSql.query("select c1, c5 ,sqrt(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, 2.979932904) + tdSql.checkData(2 , 2, 2.787471970) + tdSql.checkData(3 , 2, 2.580697551) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_sqrt( "select c1, c2, c3 , c4, c5 from ct1", "select sqrt(c1), sqrt(c2) ,sqrt(c3), sqrt(c4), sqrt(c5) from ct1") + + # nest query for sqrt functions + tdSql.query("select c4 , sqrt(c4) ,sqrt(sqrt(c4)) , sqrt(sqrt(sqrt(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , 9.380831520) + tdSql.checkData(0 , 2 , 3.062814314) + tdSql.checkData(0 , 3 , 1.750089802) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , 8.774964387) + tdSql.checkData(1 , 2 , 2.962256638) + tdSql.checkData(1 , 3 , 1.721120750) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , None) + tdSql.checkData(11 , 2 , None) + tdSql.checkData(11 , 3 , None) + + # used for stable table + + tdSql.query("select sqrt(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select sqrt(c1) from stbbb1") + tdSql.error("select sqrt(c1) from tbname") + tdSql.error("select sqrt(c1) from ct5") + + # mix with common col + tdSql.query("select c1, sqrt(c1) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,2.828427125) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0.000000000) + tdSql.query("select c2, sqrt(c2) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,55555) + tdSql.checkData(4 , 1 ,235.701081881) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + + # mix with common functions + tdSql.query("select c1, sqrt(c1),sqrt(c1), sqrt(sqrt(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,2.449489743) + tdSql.checkData(3 , 2 ,2.449489743) + tdSql.checkData(3 , 3 ,1.565084580) + + tdSql.query("select c1, sqrt(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, sqrt(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, sqrt(c1),c5, count(c5) from ct1 ") + tdSql.error("select sqrt(c1), count(c5) from stb1 ") + tdSql.error("select sqrt(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # # bug fix for compute + tdSql.query("select c1, sqrt(c1) -0 ,sqrt(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 2.828427125) + tdSql.checkData(1, 2, 2.000000000) + + tdSql.query(" select c1, sqrt(c1) -0 ,sqrt(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 2.828427125) + tdSql.checkData(1, 2, 2.710693865) + + tdSql.query("select c1, sqrt(c1), c2, sqrt(c2), c3, sqrt(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, sqrt(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, 10000.000000000) + + + tdSql.query("select c1, sqrt(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, 3162277.660168380) + + tdSql.query("select c1, sqrt(c1) + sqrt(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sqrt(c1) + sqrt(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, 3162277660171.025390625) + + tdSql.query("select c1, sqrt(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sqrt(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, 100000000000000000.000000000) + + tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, 100000000000000000000.000000000) + + tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def pow_base_test(self): + + # base is an regular number ,int or double + tdSql.query("select c1, sqrt(c1) from ct1") + tdSql.checkData(0, 1,2.828427125) + tdSql.checkRows(13) + + # # bug for compute in functions + # tdSql.query("select c1, abs(1/0) from ct1") + # tdSql.checkData(0, 0, 8) + # tdSql.checkData(0, 1, 1) + + tdSql.query("select c1, sqrt(1) from ct1") + tdSql.checkData(0, 1, 1.000000000) + tdSql.checkRows(13) + + # two cols start sqrt(x,y) + tdSql.query("select c1,c2, sqrt(c2) from ct1") + tdSql.checkData(0, 2, 298.140906284) + tdSql.checkData(1, 2, 278.885281074) + tdSql.checkData(4, 2, 0.000000000) + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,3.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=sqrt(c1) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,11111) + tdSql.checkData(0,2,1.000000000) + tdSql.checkData(0,3,1.000000000) + tdSql.checkData(0,4,0.900000000) + tdSql.checkData(0,5,1.000000000) + + def pow_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto_sqrt( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select sqrt(abs(c1)), sqrt(abs(c2)) ,sqrt(abs(c3)), sqrt(abs(c4)), sqrt(abs(c5)) from sub1_bound") + + self.check_result_auto_sqrt( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select sqrt(c1), sqrt(c2) ,sqrt(c3), sqrt(c3), sqrt(c2) ,sqrt(c1) from sub1_bound") + + self.check_result_auto_sqrt("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select sqrt(abs(c1)) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select sqrt(abs(c1)) ,sqrt(abs(c2)) , sqrt(abs(c3)) , sqrt(abs(c4)), sqrt(abs(c5)), sqrt(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.sqrt(2147483647)) + tdSql.checkData(0,1,math.sqrt(9223372036854775807)) + tdSql.checkData(0,2,math.sqrt(32767)) + tdSql.checkData(0,3,math.sqrt(127)) + tdSql.checkData(0,4,math.sqrt(339999995214436424907732413799364296704.00000)) + tdSql.checkData(1,0,math.sqrt(2147483647)) + tdSql.checkData(1,1,math.sqrt(9223372036854775807)) + tdSql.checkData(1,2,math.sqrt(32767)) + tdSql.checkData(1,3,math.sqrt(127)) + tdSql.checkData(1,4,math.sqrt(339999995214436424907732413799364296704.00000)) + tdSql.checkData(3,0,math.sqrt(2147483646)) + tdSql.checkData(3,1,math.sqrt(9223372036854775806)) + tdSql.checkData(3,2,math.sqrt(32766)) + tdSql.checkData(3,3,math.sqrt(126)) + tdSql.checkData(3,4,math.sqrt(339999995214436424907732413799364296704.00000)) + + # check + - * / in functions + tdSql.query("select sqrt(abs(c1+1)) ,sqrt(abs(c2)) , sqrt(abs(c3*1)) , sqrt(abs(c4/2)), sqrt(abs(c5))/2, sqrt(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.sqrt(2147483648.000000000)) + tdSql.checkData(0,1,math.sqrt(9223372036854775807)) + tdSql.checkData(0,2,math.sqrt(32767.000000000)) + tdSql.checkData(0,3,math.sqrt(63.500000000)) + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: sqrt basic query ============") + + self.basic_sqrt_function() + + tdLog.printNoPrefix("==========step5: big number sqrt query ============") + + self.test_big_number() + + tdLog.printNoPrefix("==========step6: base number for sqrt query ============") + + self.pow_base_test() + + tdLog.printNoPrefix("==========step7: sqrt boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step8: sqrt filter query ============") + + self.abs_func_filter() + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/sum.py b/tests/system-test/2-query/sum.py index 757f5c3c90a072bbc5589a20768b7836c549bf80..9521b005ddcfebedc08c4b618c936726ec4e3c85 100644 --- a/tests/system-test/2-query/sum.py +++ b/tests/system-test/2-query/sum.py @@ -61,6 +61,7 @@ class TDTestCase: tdSql.checkData(0, 0, sum_data) tdSql.query(f"select {condition} from {tbname} {where_condition} {group_condition} ") + tdSql.query(f"select sum( {condition} ) from {tbname} {where_condition} {group_condition} ") def __sum_err_check(self,tbanme): sqls = [] @@ -131,74 +132,75 @@ class TDTestCase: tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') def __insert_data(self, rows): - for i in range(9): + now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + for i in range(rows): tdSql.execute( - f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct2 values ( now()-{i*90}d, {-1*i}, {-11111*i}, {-111*i}, {-11*i}, {-1.11*i}, {-11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" ) tdSql.execute( - '''insert into ct1 values - ( now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a ) - ( now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a ) + f'''insert into ct1 values + ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', { now_time + 8 } ) + ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', { now_time + 9 } ) ''' ) tdSql.execute( f'''insert into ct4 values - ( now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( now()+{rows * 9}d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( - now()+{rows * 9-10}d, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, - { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nachar_limit-1", now()-1d + { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000} ) ( - now()+{rows * 9-20}d, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, - { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nachar_limit-2", now()-2d + { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000} ) ''' ) tdSql.execute( f'''insert into ct2 values - ( now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( now()+{rows * 9}d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( - now()+{rows * 9-10}d, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, - { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nachar_limit-1", now()-1d + { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, + { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } ) ( - now()+{rows * 9-20}d, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, - { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nachar_limit-2", now()-2d + { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, + { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } ) ''' ) for i in range(rows): insert_data = f'''insert into t1 values - ( now()-{i}h, {i}, {i}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, - "binary_{i}", "nchar_{i}", now()-{i}s ) + ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, + "binary_{i}", "nchar_{i}", { now_time - 1000 * i } ) ''' tdSql.execute(insert_data) tdSql.execute( f'''insert into t1 values - ( now() + 3h, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( now()-{ ( rows // 2 ) * 60 + 30 }m, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( now()-{rows}h, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( now() + 2h, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, + ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, - "binary_limit-1", "nachar_limit-1", now()-1d + "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } ) ( - now() + 1h , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, + { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, - "binary_limit-2", "nachar_limit-2", now()-2d + "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } ) ''' ) @@ -211,7 +213,7 @@ class TDTestCase: self.__create_tb() tdLog.printNoPrefix("==========step2:insert data") - self.__insert_data(100) + self.__insert_data(10) tdLog.printNoPrefix("==========step3:all check") self.all_test() diff --git a/tests/system-test/2-query/upper.py b/tests/system-test/2-query/upper.py new file mode 100644 index 0000000000000000000000000000000000000000..3c3fddfb45b29dbc23ceabacae62717ee1155a52 --- /dev/null +++ b/tests/system-test/2-query/upper.py @@ -0,0 +1,245 @@ +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + + +INT_COL = "c1" +BINT_COL = "c2" +SINT_COL = "c3" +TINT_COL = "c4" +FLOAT_COL = "c5" +DOUBLE_COL = "c6" +BOOL_COL = "c7" + +BINARY_COL = "c8" +NCHAR_COL = "c9" +TS_COL = "c10" + +UN_CHAR_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_COL, ] +CHAR_COL = [ BINARY_COL, NCHAR_COL, ] +TS_TYPE_COL = [TS_COL] + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def __upper_condition(self): + upper_condition = [] + for char_col in CHAR_COL: + upper_condition.extend( + ( + char_col, + f"lower( {char_col} )", + ) + ) + upper_condition.extend(f"cast( {un_char_col} as binary(16) ) " for un_char_col in UN_CHAR_COL) + upper_condition.extend( f"cast( {char_col} + {char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL ) + upper_condition.extend( f"cast( {char_col} + {un_char_col} as binary(32) ) " for un_char_col in UN_CHAR_COL ) + + upper_condition.append('''"test1234!@#$%^&*():'> 0 " + return "" + + def __group_condition(self, col, having = ""): + return f" group by {col} having {having}" if having else f" group by {col} " + + def __upper_current_check(self, tbname): + upper_condition = self.__upper_condition() + for condition in upper_condition: + where_condition = self.__where_condition(condition) + group_having = self.__group_condition(condition, having=f"{condition} is not null " ) + group_no_having= self.__group_condition(condition ) + groups = ["", group_having, group_no_having] + + for group_condition in groups: + tdSql.query(f"select {condition} from {tbname} {where_condition} {group_condition} ") + datas = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + upper_data = [ str(data).upper() if data else None for data in datas ] + tdSql.query(f"select upper( {condition} ) from {tbname} {where_condition} {group_condition}") + for i in range(len(upper_data)): + tdSql.checkData(i, 0, upper_data[i] ) if upper_data[i] else tdSql.checkData(i, 0, None) + + def __upper_err_check(self,tbname): + sqls = [] + + for un_char_col in UN_CHAR_COL: + sqls.extend( + ( + f"select upper( {un_char_col} ) from {tbname} ", + f"select upper(ceil( {un_char_col} )) from {tbname} ", + f"select {un_char_col} from {tbname} group by upper( {un_char_col} ) ", + ) + ) + + sqls.extend( f"select upper( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in UN_CHAR_COL ) + sqls.extend( f"select upper( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + + sqls.extend( f"select {char_col} from {tbname} group by upper( {char_col} ) " for char_col in CHAR_COL) + sqls.extend( f"select upper( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + sqls.extend( f"select upper( {char_col} + {ts_col} ) from {tbname} " for char_col in UN_CHAR_COL for ts_col in TS_TYPE_COL) + sqls.extend( f"select upper( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) + sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) + sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) + sqls.extend( + ( + f"select upper() from {tbname} ", + f"select upper(*) from {tbname} ", + f"select upper(ccccccc) from {tbname} ", + f"select upper(111) from {tbname} ", + ) + ) + + return sqls + + def __test_current(self): + tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") + tbname = ["ct1", "ct2", "ct4", "t1", "stb1"] + for tb in tbname: + self.__upper_current_check(tb) + tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========") + + def __test_error(self): + tdLog.printNoPrefix("==========err sql condition check , must return error==========") + tbname = ["ct1", "ct2", "ct4", "t1", "stb1"] + + for tb in tbname: + for errsql in self.__upper_err_check(tb): + tdSql.error(sql=errsql) + tdLog.printNoPrefix(f"==========err sql condition check in {tb} over==========") + + + def all_test(self): + self.__test_current() + self.__test_error() + + + def __create_tb(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + create_stb_sql = f'''create table stb1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + ) tags (t1 int) + ''' + create_ntb_sql = f'''create table t1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + ) + ''' + tdSql.execute(create_stb_sql) + tdSql.execute(create_ntb_sql) + + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + def __insert_data(self, rows): + now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + for i in range(rows): + tdSql.execute( + f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + ) + tdSql.execute( + f'''insert into ct1 values + ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', { now_time + 8 } ) + ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', { now_time + 9 } ) + ''' + ) + + tdSql.execute( + f'''insert into ct4 values + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( + { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000} + ) + ( + { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000} + ) + ''' + ) + + tdSql.execute( + f'''insert into ct2 values + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( + { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, + { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + ) + ( + { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, + { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + ) + ''' + ) + + for i in range(rows): + insert_data = f'''insert into t1 values + ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, + "binary_{i}", "nchar_{i}", { now_time - 1000 * i } ) + ''' + tdSql.execute(insert_data) + tdSql.execute( + f'''insert into t1 values + ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, + "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + ) + ( + { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, + "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + ) + ''' + ) + + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + self.__create_tb() + + tdLog.printNoPrefix("==========step2:insert data") + self.__insert_data(10) + + tdLog.printNoPrefix("==========step3:all check") + self.all_test() + + # tdDnodes.stop(1) + # tdDnodes.start(1) + + # tdSql.execute("use db") + + # tdLog.printNoPrefix("==========step4:after wal, all check again ") + # self.all_test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 83f185ae97e589c89c262efea671fa9e692715e2..51f1649cc862920c534a5b00f2ab383b80cefbc1 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -1,11 +1,14 @@ #!/bin/bash set -e +set -x python3 ./test.py -f 0-others/taosShell.py +python3 ./test.py -f 0-others/taosShellError.py +python3 ./test.py -f 0-others/taosShellNetChk.py #python3 ./test.py -f 2-query/between.py -#python3 ./test.py -f 2-query/distinct.py +python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py #python3 ./test.py -f 2-query/timezone.py @@ -14,3 +17,11 @@ python3 ./test.py -f 2-query/Today.py #python3 ./test.py -f 2-query/cast.py + +python3 ./test.py -f 2-query/abs.py +python3 ./test.py -f 2-query/ceil.py +python3 ./test.py -f 2-query/floor.py +python3 ./test.py -f 2-query/round.py +python3 ./test.py -f 2-query/log.py +python3 ./test.py -f 2-query/pow.py +python3 ./test.py -f 2-query/sqrt.py diff --git a/tests/test-all.sh b/tests/test-all.sh index ff9905e2097a41f11875cda7c34a6dcd2c28f69c..67bf66ee40892aadcff1e26f494bb0650657dd5e 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -93,6 +93,7 @@ function runSimCaseOneByOnefq { if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then #case=`echo $line | grep sim$ |awk '{print $NF}'` case=`echo $line | grep -o ".*\.sim" |awk '{print $NF}'` + echo "$line running ..." start_time=`date +%s` date +%F\ %T | tee -a out.log diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index db01b84d0e57766f170e8c805804caf9358a85c6..73cdf7f59ce9333f41e9dd2cfd1b7347949790de 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -29,49 +29,49 @@ #define NC "\033[0m" #define min(a, b) (((a) < (b)) ? (a) : (b)) -#define MAX_SQL_STR_LEN (1024 * 1024) -#define MAX_ROW_STR_LEN (16 * 1024) -#define MAX_CONSUMER_THREAD_CNT (16) +#define MAX_SQL_STR_LEN (1024 * 1024) +#define MAX_ROW_STR_LEN (16 * 1024) +#define MAX_CONSUMER_THREAD_CNT (16) typedef struct { - TdThread thread; - int32_t consumerId; + TdThread thread; + int32_t consumerId; - int32_t ifCheckData; - int64_t expectMsgCnt; - - int64_t consumeMsgCnt; - int64_t consumeRowCnt; - int32_t checkresult; + int32_t ifCheckData; + int64_t expectMsgCnt; - char topicString[1024]; - char keyString[1024]; + int64_t consumeMsgCnt; + int64_t consumeRowCnt; + int32_t checkresult; - int32_t numOfTopic; - char topics[32][64]; + char topicString[1024]; + char keyString[1024]; - int32_t numOfKey; - char key[32][64]; - char value[32][64]; + int32_t numOfTopic; + char topics[32][64]; + + int32_t numOfKey; + char key[32][64]; + char value[32][64]; tmq_t* tmq; tmq_list_t* topicList; - + } SThreadInfo; typedef struct { // input from argvs - char cdbName[32]; - char dbName[32]; - int32_t showMsgFlag; - int32_t showRowFlag; - int32_t consumeDelay; // unit s - int32_t numOfThread; - SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; + char cdbName[32]; + char dbName[32]; + int32_t showMsgFlag; + int32_t showRowFlag; + int32_t consumeDelay; // unit s + int32_t numOfThread; + SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; } SConfInfo; static SConfInfo g_stConfInfo; -TdFilePtr g_fp = NULL; +TdFilePtr g_fp = NULL; // char* g_pRowValue = NULL; // TdFilePtr g_fp = NULL; @@ -93,7 +93,6 @@ static void printHelp() { exit(EXIT_SUCCESS); } - void initLogFile() { // FILE *fp = fopen(g_stConfInfo.resultFileName, "a"); char file[256]; @@ -106,36 +105,35 @@ void initLogFile() { g_fp = pFile; } - void saveConfigToLogFile() { time_t tTime = taosGetTimestampSec(); struct tm tm = *taosLocalTime(&tTime, NULL); taosFprintfFile(g_fp, "###################################################################\n"); - taosFprintfFile(g_fp, "# configDir: %s\n", configDir); - taosFprintfFile(g_fp, "# dbName: %s\n", g_stConfInfo.dbName); - taosFprintfFile(g_fp, "# cdbName: %s\n", g_stConfInfo.cdbName); - taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); - taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); - taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); - taosFprintfFile(g_fp, "# numOfThread: %d\n", g_stConfInfo.numOfThread); - - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + taosFprintfFile(g_fp, "# configDir: %s\n", configDir); + taosFprintfFile(g_fp, "# dbName: %s\n", g_stConfInfo.dbName); + taosFprintfFile(g_fp, "# cdbName: %s\n", g_stConfInfo.cdbName); + taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); + taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); + taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); + taosFprintfFile(g_fp, "# numOfThread: %d\n", g_stConfInfo.numOfThread); + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); - taosFprintfFile(g_fp, " Topics: "); - for (int j = 0 ; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { - taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); + taosFprintfFile(g_fp, " Topics: "); + for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { + taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); } - taosFprintfFile(g_fp, "\n"); + taosFprintfFile(g_fp, "\n"); taosFprintfFile(g_fp, " Key: "); - for (int k = 0 ; k < g_stConfInfo.stThreads[i].numOfKey; k++) { - taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); + for (int k = 0; k < g_stConfInfo.stThreads[i].numOfKey; k++) { + taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); } taosFprintfFile(g_fp, "\n"); } - + taosFprintfFile(g_fp, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); taosFprintfFile(g_fp, "###################################################################\n"); } @@ -168,7 +166,7 @@ void parseArgument(int32_t argc, char* argv[]) { } initLogFile(); - + taosFprintfFile(g_fp, "====parseArgument() success\n"); #if 1 @@ -203,26 +201,26 @@ void ltrim(char* str) { // return str; } -static int running = 1; +static int running = 1; static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) { - char buf[1024]; + char buf[1024]; int32_t totalRows = 0; - //printf("topic: %s\n", tmq_get_topic_name(msg)); - //printf("vg:%d\n", tmq_get_vgroup_id(msg)); - taosFprintfFile(g_fp, "msg index:%" PRId64 ", threadLable: %d\n", msgIndex, threadLable); - taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg)); - + // printf("topic: %s\n", tmq_get_topic_name(msg)); + // printf("vg:%d\n", tmq_get_vgroup_id(msg)); + taosFprintfFile(g_fp, "msg index:%" PRId64 ", threadLable: %d\n", msgIndex, threadLable); + taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg)); + while (1) { TAOS_ROW row = taos_fetch_row(msg); if (row == NULL) break; - if (0 != g_stConfInfo.showRowFlag) { - TAOS_FIELD* fields = taos_fetch_fields(msg); + if (0 != g_stConfInfo.showRowFlag) { + TAOS_FIELD* fields = taos_fetch_fields(msg); int32_t numOfFields = taos_field_count(msg); taos_print_row(buf, row, fields, numOfFields); - taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); - } - totalRows++; + taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); + } + totalRows++; } return totalRows; @@ -241,43 +239,43 @@ int queryDB(TAOS* taos, char* command) { return 0; } -static void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets) { +static void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { printf("tmq_commit_cb_print() commit %d\n", resp); } -void build_consumer(SThreadInfo *pInfo) { +void build_consumer(SThreadInfo* pInfo) { tmq_conf_t* conf = tmq_conf_new(); - //tmq_conf_set(conf, "td.connect.ip", "localhost"); - //tmq_conf_set(conf, "td.connect.port", "6030"); + // tmq_conf_set(conf, "td.connect.ip", "localhost"); + // tmq_conf_set(conf, "td.connect.port", "6030"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); - tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); + tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL); // tmq_conf_set(conf, "group.id", "cgrp1"); for (int32_t i = 0; i < pInfo->numOfKey; i++) { tmq_conf_set(conf, pInfo->key[i], pInfo->value[i]); } - //tmq_conf_set(conf, "client.id", "c-001"); + // tmq_conf_set(conf, "client.id", "c-001"); + + // tmq_conf_set(conf, "enable.auto.commit", "true"); + // tmq_conf_set(conf, "enable.auto.commit", "false"); + + // tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - //tmq_conf_set(conf, "enable.auto.commit", "true"); - //tmq_conf_set(conf, "enable.auto.commit", "false"); + // tmq_conf_set(conf, "auto.offset.reset", "none"); + // tmq_conf_set(conf, "auto.offset.reset", "earliest"); + // tmq_conf_set(conf, "auto.offset.reset", "latest"); - //tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - - //tmq_conf_set(conf, "auto.offset.reset", "none"); - //tmq_conf_set(conf, "auto.offset.reset", "earliest"); - //tmq_conf_set(conf, "auto.offset.reset", "latest"); - pInfo->tmq = tmq_consumer_new(conf, NULL, 0); return; } -void build_topic_list(SThreadInfo *pInfo) { +void build_topic_list(SThreadInfo* pInfo) { pInfo->topicList = tmq_list_new(); // tmq_list_append(topic_list, "test_stb_topic_1"); for (int32_t i = 0; i < pInfo->numOfTopic; i++) { @@ -286,41 +284,37 @@ void build_topic_list(SThreadInfo *pInfo) { return; } -int32_t saveConsumeResult(SThreadInfo *pInfo) { +int32_t saveConsumeResult(SThreadInfo* pInfo) { char sqlStr[1024] = {0}; - + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); assert(pConn != NULL); - + // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int - sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", - g_stConfInfo.cdbName, - pInfo->consumerId, - pInfo->consumeMsgCnt, - pInfo->consumeRowCnt, - pInfo->checkresult); - + sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName, + pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult); + TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { printf("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); exit(-1); } - + taos_free_result(pRes); return 0; } -void loop_consume(SThreadInfo *pInfo) { +void loop_consume(SThreadInfo* pInfo) { tmq_resp_err_t err; - + int64_t totalMsgs = 0; int64_t totalRows = 0; while (running) { TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000); - if (tmqMsg) { + if (tmqMsg) { if (0 != g_stConfInfo.showMsgFlag) { totalRows += msg_process(tmqMsg, totalMsgs, pInfo->consumerId); } @@ -328,7 +322,7 @@ void loop_consume(SThreadInfo *pInfo) { taos_free_result(tmqMsg); totalMsgs++; - + if (totalMsgs >= pInfo->expectMsgCnt) { break; } @@ -336,7 +330,7 @@ void loop_consume(SThreadInfo *pInfo) { break; } } - + err = tmq_consumer_close(pInfo->tmq); if (err) { printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); @@ -346,27 +340,27 @@ void loop_consume(SThreadInfo *pInfo) { pInfo->consumeMsgCnt = totalMsgs; pInfo->consumeRowCnt = totalRows; - taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %"PRId64", consumeRowCnt: %"PRId64"\n", pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt); - + taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %" PRId64 ", consumeRowCnt: %" PRId64 "\n", + pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt); } -void *consumeThreadFunc(void *param) { +void* consumeThreadFunc(void* param) { int32_t totalMsgs = 0; - SThreadInfo *pInfo = (SThreadInfo *)param; + SThreadInfo* pInfo = (SThreadInfo*)param; build_consumer(pInfo); build_topic_list(pInfo); - if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)){ + if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { return NULL; } - + tmq_resp_err_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); if (err) { printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } - + loop_consume(pInfo); tmq_commit(pInfo->tmq, NULL, 0); @@ -374,10 +368,10 @@ void *consumeThreadFunc(void *param) { err = tmq_unsubscribe(pInfo->tmq); if (err) { printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); - pInfo->consumeMsgCnt = -1; + pInfo->consumeMsgCnt = -1; return NULL; - } - + } + // save consume result into consumeresult table saveConsumeResult(pInfo); @@ -389,7 +383,7 @@ void parseConsumeInfo() { const char delim[2] = ","; const char ch = ':'; - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { token = strtok(g_stConfInfo.stThreads[i].topicString, delim); while (token != NULL) { // printf("%s\n", token ); @@ -397,10 +391,10 @@ void parseConsumeInfo() { ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]); // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); g_stConfInfo.stThreads[i].numOfTopic++; - + token = strtok(NULL, delim); } - + token = strtok(g_stConfInfo.stThreads[i].keyString, delim); while (token != NULL) { // printf("%s\n", token ); @@ -414,7 +408,7 @@ void parseConsumeInfo() { // g_stConfInfo.value[g_stConfInfo.numOfKey]); g_stConfInfo.stThreads[i].numOfKey++; } - + token = strtok(NULL, delim); } } @@ -422,48 +416,49 @@ void parseConsumeInfo() { int32_t getConsumeInfo() { char sqlStr[1024] = {0}; - + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); assert(pConn != NULL); - + sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName); TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { printf("error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); taosFprintfFile(g_fp, "error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); - taosCloseFile(&g_fp); + taosCloseFile(&g_fp); taos_free_result(pRes); exit(-1); - } - - TAOS_ROW row = NULL; - int num_fields = taos_num_fields(pRes); - TAOS_FIELD* fields = taos_fetch_fields(pRes); - - // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int - + } + + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(pRes); + TAOS_FIELD* fields = taos_fetch_fields(pRes); + + // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, + // ifcheckdata int + int32_t numOfThread = 0; while ((row = taos_fetch_row(pRes))) { - int32_t* lengths = taos_fetch_lengths(pRes); - - for (int i = 0; i < num_fields; ++i) { + int32_t* lengths = taos_fetch_lengths(pRes); + + for (int i = 0; i < num_fields; ++i) { if (row[i] == NULL || 0 == i) { continue; } - + if ((1 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t *)row[i]); + g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t*)row[i]); } else if ((2 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { memcpy(g_stConfInfo.stThreads[numOfThread].topicString, row[i], lengths[i]); } else if ((3 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { memcpy(g_stConfInfo.stThreads[numOfThread].keyString, row[i], lengths[i]); } else if ((4 == i) && (fields[i].type == TSDB_DATA_TYPE_BIGINT)) { - g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t *)row[i]); + g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]); } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t *)row[i]); + g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]); } } - numOfThread ++; + numOfThread++; } g_stConfInfo.numOfThread = numOfThread; @@ -474,7 +469,6 @@ int32_t getConsumeInfo() { return 0; } - int main(int32_t argc, char* argv[]) { parseArgument(argc, argv); getConsumeInfo(); @@ -485,20 +479,21 @@ int main(int32_t argc, char* argv[]) { taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); // pthread_create one thread to consume - taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); + taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { - taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, (void *)(&(g_stConfInfo.stThreads[i]))); + taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, + (void*)(&(g_stConfInfo.stThreads[i]))); } for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); } - //printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); - - taosFprintfFile(g_fp, "==== close tmqlog ====\n"); - taosCloseFile(&g_fp); - + // printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); + + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); + taosCloseFile(&g_fp); + return 0; } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index befcfc0731fec30dd3f8266415398840fe9b15ad..08fe2cbc32f080ca83fbe952924b69e5e2505902 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -141,6 +141,9 @@ int32_t shellRunCommand(char *command) { *p++ = '\\'; } break; + default: + *p++ = '\\'; + break; } *p++ = c; esc = false; diff --git a/tools/shell/src/shellNettest.c b/tools/shell/src/shellNettest.c index c8ec31c48bb29c56215c1bba4090eb8f4cddc7b4..345b85d896c92c3b28a0e6cc78f9ce775a03bd4a 100644 --- a/tools/shell/src/shellNettest.c +++ b/tools/shell/src/shellNettest.c @@ -21,7 +21,7 @@ static void shellWorkAsClient() { SRpcInit rpcInit = {0}; SEpSet epSet = {.inUse = 0, .numOfEps = 1}; SRpcMsg rpcRsp = {0}; - void *clientRpc = NULL; + void * clientRpc = NULL; char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)("_pwd"), strlen("_pwd"), pass); @@ -116,6 +116,7 @@ static void shellWorkAsServer() { } SRpcInit rpcInit = {0}; + memcpy(rpcInit.localFqdn, tsLocalFqdn, strlen(tsLocalFqdn)); rpcInit.localPort = pArgs->port; rpcInit.label = "CHK"; rpcInit.numOfThreads = tsNumOfRpcThreads; @@ -126,7 +127,7 @@ static void shellWorkAsServer() { void *serverRpc = rpcOpen(&rpcInit); if (serverRpc == NULL) { - printf("failed to init net test server since %s", terrstr()); + printf("failed to init net test server since %s\n", terrstr()); } else { printf("network test server is initialized, port:%u\n", pArgs->port); taosSetSignal(SIGTERM, shellNettestHandler); diff --git a/tools/taos-tools b/tools/taos-tools index bf6c766986c61ff4fc80421fdea682a8fd4b5b32..59e0ebaf4905e4cb6d95a01c58b3fa507abc5a20 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit bf6c766986c61ff4fc80421fdea682a8fd4b5b32 +Subproject commit 59e0ebaf4905e4cb6d95a01c58b3fa507abc5a20